first commit

batch/job_queue.js

@@ -0,0 +1,165 @@
+'use strict';
+
+var debug = require('./util/debug')('queue');
+var queueAsync = require('queue-async');
+
+function JobQueue(metadataBackend, jobPublisher) {
+    this.metadataBackend = metadataBackend;
+    this.jobPublisher = jobPublisher;
+}
+
+module.exports = JobQueue;
+
+var QUEUE = {
+    DB: 5,
+    PREFIX: 'batch:queue:',
+    INDEX: 'batch:indexes:queue'
+};
+
+module.exports.QUEUE = QUEUE;
+
+JobQueue.prototype.enqueue = function (user, jobId, callback) {
+    debug('JobQueue.enqueue user=%s, jobId=%s', user, jobId);
+
+    this.metadataBackend.redisMultiCmd(QUEUE.DB, [
+        [ 'LPUSH', QUEUE.PREFIX + user, jobId ],
+        [ 'SADD', QUEUE.INDEX, user ]
+    ], function (err) {
+        if (err) {
+            return callback(err);
+        }
+
+        this.jobPublisher.publish(user);
+        callback();
+    }.bind(this));
+};
+
+JobQueue.prototype.size = function (user, callback) {
+    this.metadataBackend.redisCmd(QUEUE.DB, 'LLEN', [ QUEUE.PREFIX + user ], callback);
+};
+
+JobQueue.prototype.dequeue = function (user, callback) {
+    var dequeueScript = [
+        'local job_id = redis.call("RPOP", KEYS[1])',
+        'if redis.call("LLEN", KEYS[1]) == 0 then',
+        '   redis.call("SREM", KEYS[2], ARGV[1])',
+        'end',
+        'return job_id'
+    ].join('\n');
+
+    var redisParams = [
+        dequeueScript, //lua source code
+        2, // Two "keys" to pass
+        QUEUE.PREFIX + user, //KEYS[1], the key of the queue
+        QUEUE.INDEX, //KEYS[2], the key of the index
+        user // ARGV[1] - value of the element to remove from the index
+    ];
+
+    this.metadataBackend.redisCmd(QUEUE.DB, 'EVAL', redisParams, function (err, jobId) {
+        debug('JobQueue.dequeued user=%s, jobId=%s', user, jobId);
+        return callback(err, jobId);
+    });
+};
+
+JobQueue.prototype.enqueueFirst = function (user, jobId, callback) {
+    debug('JobQueue.enqueueFirst user=%s, jobId=%s', user, jobId);
+    this.metadataBackend.redisMultiCmd(QUEUE.DB, [
+        [ 'RPUSH', QUEUE.PREFIX + user, jobId ],
+        [ 'SADD', QUEUE.INDEX, user ]
+    ], function (err) {
+        if (err) {
+            return callback(err);
+        }
+
+        this.jobPublisher.publish(user);
+        callback();
+    }.bind(this));
+};
+
+
+JobQueue.prototype.getQueues = function (callback) {
+    this.metadataBackend.redisCmd(QUEUE.DB, 'SMEMBERS', [ QUEUE.INDEX ], function (err, queues) {
+        if (err) {
+            return callback(err);
+        }
+
+        callback(null, queues);
+    });
+};
+
+JobQueue.prototype.scanQueues = function (callback) {
+    var self = this;
+
+    self.scan(function (err, queues) {
+        if (err) {
+            return callback(err);
+        }
+
+        self.addToQueueIndex(queues, function (err) {
+            if (err) {
+                return callback(err);
+            }
+
+            callback(null, queues);
+        });
+    });
+};
+
+JobQueue.prototype.scan = function (callback) {
+    var self = this;
+    var initialCursor = ['0'];
+    var users = {};
+
+    self._scan(initialCursor, users, function(err, users) {
+        if (err) {
+            return callback(err);
+        }
+
+        callback(null, Object.keys(users));
+    });
+};
+
+JobQueue.prototype._scan = function (cursor, users, callback) {
+    var self = this;
+    var redisParams = [cursor[0], 'MATCH', QUEUE.PREFIX + '*'];
+
+    self.metadataBackend.redisCmd(QUEUE.DB, 'SCAN', redisParams, function (err, currentCursor) {
+        if (err) {
+            return callback(null, users);
+        }
+
+        var queues = currentCursor[1];
+        if (queues) {
+            queues.forEach(function (queue) {
+                var user = queue.substr(QUEUE.PREFIX.length);
+                users[user] = true;
+            });
+        }
+
+        var hasMore = currentCursor[0] !== '0';
+        if (!hasMore) {
+            return callback(null, users);
+        }
+
+        self._scan(currentCursor, users, callback);
+    });
+};
+
+JobQueue.prototype.addToQueueIndex = function (users, callback) {
+    var self = this;
+    var usersQueues = queueAsync(users.length);
+
+    users.forEach(function (user) {
+        usersQueues.defer(function (user, callback) {
+            self.metadataBackend.redisCmd(QUEUE.DB, 'SADD', [ QUEUE.INDEX, user], callback);
+        }, user);
+    });
+
+    usersQueues.awaitAll(function (err) {
+        if (err) {
+            return callback(err);
+        }
+
+        callback(null);
+    });
+};