first commit

2023-05-19 00:42:48 +08:00
commit 53de9c6c51
243 changed files with 39485 additions and 0 deletions
--- a/batch/maintenance/host-user-queue-mover.js
+++ b/batch/maintenance/host-user-queue-mover.js
@@ -0,0 +1,144 @@
+'use strict';
+
+var asyncQ = require('queue-async');
+var debug = require('../util/debug')('queue-mover');
+var forever = require('../util/forever');
+
+var QUEUE = {
+    OLD: {
+        DB: 5,
+        PREFIX: 'batch:queues:' // host
+    },
+    NEW: {
+        DB: 5,
+        PREFIX: 'batch:queue:' // user
+    }
+};
+
+function HostUserQueueMover(jobQueue, jobService, locker, redisPool) {
+    this.jobQueue = jobQueue;
+    this.jobService = jobService;
+    this.locker = locker;
+    this.pool = redisPool;
+}
+
+module.exports = HostUserQueueMover;
+
+HostUserQueueMover.prototype.moveOldJobs = function(callback) {
+    var self = this;
+    this.getOldQueues(function(err, hosts) {
+        var async = asyncQ(4);
+        hosts.forEach(function(host) {
+            async.defer(self.moveOldQueueJobs.bind(self), host);
+        });
+
+        async.awaitAll(function (err) {
+            if (err) {
+                debug('Something went wrong moving jobs', err);
+            } else {
+                debug('Finished moving all jobs');
+            }
+
+            callback();
+        });
+    });
+};
+
+HostUserQueueMover.prototype.moveOldQueueJobs = function(host, callback) {
+    var self = this;
+    // do forever, it does not throw a stack overflow
+    forever(
+        function (next) {
+            self.locker.lock(host, function(err) {
+                // we didn't get the lock for the host
+                if (err) {
+                    debug('Could not lock host=%s. Reason: %s', host, err.message);
+                    return next(err);
+                }
+                debug('Locked host=%s', host);
+                self.processNextJob(host, next);
+            });
+        },
+        function (err) {
+            if (err) {
+                debug(err.name === 'EmptyQueue' ? err.message : err);
+            }
+            self.locker.unlock(host, callback);
+        }
+    );
+};
+
+//this.metadataBackend.redisCmd(QUEUE.DB, 'RPOP', [ QUEUE.PREFIX + user ], callback);
+
+HostUserQueueMover.prototype.processNextJob = function (host, callback) {
+    var self = this;
+    this.pool.acquire(QUEUE.OLD.DB, function(err, client) {
+        if (err) {
+            return callback(err);
+        }
+
+        client.lpop(QUEUE.OLD.PREFIX + host, function(err, jobId) {
+            self.pool.release(QUEUE.OLD.DB, client);
+            debug('Found jobId=%s at queue=%s', jobId, host);
+            if (!jobId) {
+                var emptyQueueError = new Error('Empty queue');
+                emptyQueueError.name = 'EmptyQueue';
+                return callback(emptyQueueError);
+            }
+            self.jobService.get(jobId, function(err, job) {
+                if (err) {
+                    debug(err);
+                    return callback();
+                }
+                if (job) {
+                    return self.jobQueue.enqueueFirst(job.data.user, jobId, function() {
+                        return callback();
+                    });
+                }
+                return callback();
+            });
+        });
+    });
+};
+
+HostUserQueueMover.prototype.getOldQueues = function(callback) {
+    var initialCursor = ['0'];
+    var hosts = {};
+    var self = this;
+
+    this.pool.acquire(QUEUE.OLD.DB, function(err, client) {
+        if (err) {
+            return callback(err);
+        }
+        self._getOldQueues(client, initialCursor, hosts, function(err, hosts) {
+            self.pool.release(QUEUE.DB, client);
+            return callback(err, Object.keys(hosts));
+        });
+    });
+};
+
+HostUserQueueMover.prototype._getOldQueues = function (client, cursor, hosts, callback) {
+    var self = this;
+    var redisParams = [cursor[0], 'MATCH', QUEUE.OLD.PREFIX + '*'];
+
+    client.scan(redisParams, function(err, currentCursor) {
+        if (err) {
+            return callback(null, hosts);
+        }
+
+        var queues = currentCursor[1];
+        if (queues) {
+            queues.forEach(function (queue) {
+                var user = queue.substr(QUEUE.OLD.PREFIX.length);
+                hosts[user] = true;
+            });
+        }
+
+        var hasMore = currentCursor[0] !== '0';
+        if (!hasMore) {
+            return callback(null, hosts);
+        }
+
+        self._getOldQueues(client, currentCursor, hosts, callback);
+    });
+};