There are several improvements and features added.

1. Comments queue
2. Voting queue
3. Switch from filesystem streams to in-memory buffers

Comments and Voting queues

---

In other projects, I have noticed significant efficiency improvements from switching to queues that spread voting/commenting out instead of processing all votes and comments simultaneously. The queue implementations are fairly simple. Just pushing/shifting off an array. It is also managed through a scheduler that shifts items off the array in periodic increments. Ideally, we would want to manage this with an event. That may come in as a future change.

Switching from Filesystem Streams to in-memory Buffers

Prior to this change, JPG data was read via HTTP using streams. The data would be streamed and then written to a file. This file would then be read back in as a buffer. We are essentially skipping a step. Rather than blocking on filesystem I/O for streams, I switched to in-memory buffers. The data is converted from a stream to a buffer and loaded into memory rather than writing to disk first.

Changes

diff --git a/steem-exif-spider-bot/helpers/bot/comment.js b/steem-exif-spider-bot/helpers/bot/comment.js
new file mode 100644
index 0000000..10c2093
--- /dev/null
+++ b/steem-exif-spider-bot/helpers/bot/comment.js
@@ -0,0 +1,61 @@
+const Promise = require('bluebird')
+const steem = require('steem')
+const { user, wif, weight } = require('../../config')
+const schedule = require('node-schedule')
+const Handlebars = require('handlebars')
+const fs = Promise.promisifyAll(require('fs'))
+const path = require('path')
+
+const MINUTE = new schedule.RecurrenceRule();
+MINUTE.second = 1
+
+function loadTemplate(template) {
+    return fs.readFileAsync(template, 'utf8')
+}
+
+
+function execute(comments) {
+
+    if (comments.length() < 1) {
+        return {};
+    }
+
+    const { author, permlink } = comments.shift();
+
+    var context = {
+    }
+
+    return loadTemplate(path.join(__dirname, '..', 'templates', "exif.hb"))
+        .then((template) => {
+            var templateSpec = Handlebars.compile(template)
+            return templateSpec(context)
+        })
+        .then((message) => {
+            var new_permlink = 're-' + author 
+                + '-' + permlink 
+                + '-' + new Date().toISOString().replace(/[^a-zA-Z0-9]+/g, '').toLowerCase();
+            console.log("Commenting on ", author, permlink, type)
+
+            return steem.broadcast.commentAsync(
+                wif,
+                author, // Leave parent author empty
+                permlink, // Main tag
+                user, // Author
+                new_permlink, // Permlink
+                new_permlink,
+                message, // Body
+                { tags: [], app: "steemit-exif-spider-bot/0.1.0" }
+            ).then((results) => {
+                console.log(results)
+                return results
+            })
+            .catch((err) => {
+                console.log("Error ", err.message)
+            })
+        })
+}
+
+module.exports = {
+    execute
+}

Comment queue implementation. Pulls comments off the queue and posts them to the steem blockchain.

diff --git a/steem-exif-spider-bot/helpers/bot/exif.js b/steem-exif-spider-bot/helpers/bot/exif.js
index bc49a21..738c48a 100644
--- a/steem-exif-spider-bot/helpers/bot/exif.js
+++ b/steem-exif-spider-bot/helpers/bot/exif.js
@@ -17,6 +17,9 @@ module.exports = {
     execute
 }
 
+let VOTING = {}
+let COMMENTS = {}
+
 function loadTemplate(template) {
     return fs.readFileAsync(template, 'utf8')
 }
@@ -35,60 +38,32 @@ function processComment(comment) {
             }
             return [];
         })
-        .each((image) => {
-            if (image.indexOf(".jpg") > -1|| image.indexOf(".JPG") > -1) {
-                const dest = tempfile('.jpg');
-                try {
-                    got.stream(image).pipe(fs.createWriteStream(dest))
-                        .on('close', () => {
-                            try {
-                                const input = ExifReader.load(fs.readFileSync(dest));
-                                const tags = []
-                                for (let key in input) {
-                                    const value = input[key];
-                                    if (key != "MakerNote"
-                                        && key.indexOf("undefined") < 0
-                                        && key.indexOf("omment") < 0
-                                        && key.indexOf("ersion") < 0) {
-                                        tags.push({ name: key, value: value.value, description: value.description })
-                                    }
-                                }
-
-                                reply(comment, tags)
-                            }
-                            catch(err) {
-                                if (err.message == "No Exif data") {
-
-                                }
-                            }
-                        })
-                }
-                catch (err) {
-                    console.log("Error ", err)
-                }
-                finally {
-                    fs.unlink(dest, (err) => {
-                        // file deleted
+        .map((image) => {
+            if (image.indexOf(".jpg") > -1 || image.indexOf(".JPG") > -1) {
+                const buffers = [];
+                return got(image, {encoding: null })
+                    .then((response) => {
+                        console.log("Loading ", image);
+                        return ExifReader.load(response.body);
+                    })
+                    .catch((error) => {
+                        console.log("Error ", error);
                     });
+            }
+        })
+        .filter((tags) => tags ? true : false)
+        .each(input => {
+            const tags = []
+            for (let key in input) {
+                const value = input[key];
+                if (key != "MakerNote"
+                    && key.indexOf("undefined") < 0
+                    && key.indexOf("omment") < 0
+                    && key.indexOf("ersion") < 0) {
+                    tags.push({ name: key, value: value.value, description: value.description })
                 }
             }
+            reply(comment, tags)
         })
         .catch((error) => {
             console.log("Error ", error)
@@ -101,51 +76,23 @@ function reply(comment, tags) {
         tags: tags
     }
 
-
-    return loadTemplate(path.join(__dirname, '..', 'templates', 'exif.hb'))
-    .then((template) => {
-        var templateSpec = Handlebars.compile(template)
-        return templateSpec(context)
-    })
-    .then((body) => {
-        console.log("Body ", body)
-        return body;
-    })
-    .then((body) => {
-        var permlink = 're-' + comment.author 
-            + '-' + comment.permlink 
-            + '-' + new Date().toISOString().replace(/[^a-zA-Z0-9]+/g, '').toLowerCase();
-
+    return new Promise((resolve, reject) => {
         console.log("Replying to ", {author: comment.author, permlink: comment.permlink})
-        return steem.broadcast.commentAsync(
-            wif,
-            comment.author, // Leave parent author empty
-            comment.permlink,
-            user, // Author
-            permlink, // Permlink
-            permlink, // Title
-            body, // Body
-            { "app": "steem-exif-spider-bot/0.1.0" }
-        )
-        .catch((err) => {
-            console.log("Unable to process comment. ", err)
-        })
+        COMMENTS.push({ author: comment.author, permlink: comment.permlink })
+
+        return [ comment.author, comment.permlink]
     })
-    .then((response) => {
-        return steem.broadcast.voteAsync(wif, user, comment.author, comment.permlink, weight)
-            .then((results) =>  {
-                console.log(results)
-            })
-            .catch((err) => {
-                console.log("Vote failed: ", err)
-            })
+    .spread((author, permlink) => {
+        VOTING.push({ author: author, permlink: permlink, weight: weight });
     })
     .catch((err) => {
         console.log("Error loading template ", err)
     })
 }
 
-function execute() {
+function execute(voting, comments) {
+    VOTING = voting
+    COMMENTS = comments
 
     steem.api.streamOperations((err, results) => {
         return new Promise((resolve, reject) => {

Moving functionality that will be implemented in comment and voting queues. Switching from filesystem streams to in-memory buffers.

diff --git a/steem-exif-spider-bot/helpers/bot/index.js b/steem-exif-spider-bot/helpers/bot/index.js
index 083546c..64b61ac 100644
--- a/steem-exif-spider-bot/helpers/bot/index.js
+++ b/steem-exif-spider-bot/helpers/bot/index.js
@@ -1,6 +1,27 @@
+const voting_queue = [];
+const comment_queue = [];
+
+const voting = {
+    length: () => { return voting_queue.length },
+    push: (obj) => { return voting_queue.push(obj) },
+    pop: () => { return voting_queue.pop() },
+    shift: () => { return voting_queue.shift() },
+    unshift: (obj) => { return voting_queue.unshift(obj) }
+}
+
+const comments = {
+    length: () => { return comment_queue.length },
+    push: (obj) => { return comment_queue.push(obj) },
+    pop: () => { return comment_queue.pop() },
+    shift: () => { return comment_queue.shift() },
+    unshift: (obj) => { return comment_queue.unshift(obj) }
+}
+
 
 function run() {
-    return require("./exif").execute();
+    require('./comment').execute(comments)
+    require('./vote').execute(voting)
+    require('./exif').execute(voting, comments)
 }

The model for the queue management

diff --git a/steem-exif-spider-bot/helpers/bot/vote.js b/steem-exif-spider-bot/helpers/bot/vote.js
new file mode 100644
index 0000000..58714c8
--- /dev/null
+++ b/steem-exif-spider-bot/helpers/bot/vote.js
@@ -0,0 +1,84 @@
+const Promise = require('bluebird')
+const steem = require('steem')
+const { user, wif, weight } = require('../../config')
+const schedule = require('node-schedule')
+const moment = require('moment');
+
+const MINUTE = new schedule.RecurrenceRule();
+MINUTE.second = 1
+
+const SECONDS_PER_HOUR = 3600
+const PERCENT_PER_DAY = 20
+const HOURS_PER_DAY = 24
+const MAX_VOTING_POWER = 10000
+const DAYS_TO_100_PERCENT = 100 / PERCENT_PER_DAY
+const SECONDS_FOR_100_PERCENT = DAYS_TO_100_PERCENT * HOURS_PER_DAY * SECONDS_PER_HOUR
+const RECOVERY_RATE = MAX_VOTING_POWER / SECONDS_FOR_100_PERCENT
+const DEFAULT_THRESHOLD = 9500
+
+
+function current_voting_power(vp_last, last_vote) {
+    console.log("Comparing %s to %s ", moment().utc().add(7, 'hours').local().toISOString(), moment(last_vote).utc().local().toISOString())
+
+    var seconds_since_vote = moment().utc().add(7, 'hours').local().diff(moment(last_vote).utc().local(), 'seconds')
+    return (RECOVERY_RATE * seconds_since_vote) + vp_last
+}
+
+function time_needed_to_recover(voting_power, threshold) {
+    return (threshold - voting_power) / RECOVERY_RATE
+}
+
+function check_can_vote() {
+    return steem.api.getAccountsAsync([ user]).then((accounts) => {
+        if (accounts && accounts.length > 0) {
+            const account = accounts[0];
+            console.log("Voting threshold for %s: %s", user, DEFAULT_THRESHOLD)
+            console.log("Getting voting power for %d %s", account.voting_power, account.last_vote_time)
+            var voting_power = current_voting_power(account.voting_power, account.last_vote_time)
+            if (voting_power > DEFAULT_THRESHOLD) {
+                return true;
+            }
+        }
+        return false;
+    })
+}
+
+function vote(author, permlink, weight) {
+    return steem.broadcast.voteAsync(
+        wif, 
+        user, 
+        author,
+        permlink,
+        weight
+    )
+    .then((results) =>  {
+        console.log("Vote results: ", results)
+        return results;
+    },
+    (err) => {
+        console.log("Vote failed for %s: %s", user, err.message)
+    })
+}
+
+function execute(voting) {
+    schedule.scheduleJob(MINUTE, function() {
+        if (voting.length() < 1) {
+            return {};
+        }
+               
+        const { author, permlink, weight } = voting.shift();
+
+        return check_can_vote().then((can_vote) => {
+            if (can_vote) {
+                vote(author, permlink, weight)
+            }
+            else {
+                voting.push({ author, permlink, weight })
+            }
+        })
+    })
+}
+
+module.exports = {
+    execute
+}

This is the module for the voting queue implementation. It periodically pulls votes off a queue and votes them.

diff --git a/steem-exif-spider-bot/package.json b/steem-exif-spider-bot/package.json
index 1abbdbe..8024fd2 100644
--- a/steem-exif-spider-bot/package.json
+++ b/steem-exif-spider-bot/package.json
@@ -31,6 +31,7 @@
     "got": "^8.3.0",
     "handlebars": "^4.0.11",
     "jdataview": "^2.5.0",
+    "node-schedule": "^1.3.0",
     "request": "^2.85.0",
     "steem": "^0.7.1",
     "tempfile": "^2.0.0",

Adding node schedule dependency so the queue can be run using node-schedule

---

Posted on Utopian.io - Rewarding Open Source Contributors

---