first commit

batch/scheduler/capacity/fixed.js

@@ -0,0 +1,11 @@
+'use strict';
+
+function FixedCapacity(capacity) {
+    this.capacity = Math.max(1, capacity);
+}
+
+module.exports = FixedCapacity;
+
+FixedCapacity.prototype.getCapacity = function(callback) {
+    return callback(null, this.capacity);
+};

batch/scheduler/capacity/http-load.js

@@ -0,0 +1,32 @@
+'use strict';
+
+var util = require('util');
+var debug = require('../../util/debug')('capacity-http-load');
+var HttpSimpleCapacity = require('./http-simple');
+
+function HttpLoadCapacity(host, capacityEndpoint) {
+    HttpSimpleCapacity.call(this, host, capacityEndpoint);
+}
+util.inherits(HttpLoadCapacity, HttpSimpleCapacity);
+
+module.exports = HttpLoadCapacity;
+
+HttpLoadCapacity.prototype.getCapacity = function(callback) {
+    this.getResponse(function(err, values) {
+        var capacity = 1;
+
+        if (err) {
+            return callback(null, capacity);
+        }
+
+        var cores = parseInt(values.cores, 10);
+        var relativeLoad = parseFloat(values.relative_load);
+
+        capacity = Math.max(1, Math.floor(((1 - relativeLoad) * cores) - 1));
+
+        capacity = Number.isFinite(capacity) ? capacity : 1;
+
+        debug('host=%s, capacity=%s', this.host, capacity);
+        return callback(null, capacity);
+    }.bind(this));
+};

batch/scheduler/capacity/http-simple.js

@@ -0,0 +1,62 @@
+'use strict';
+
+var request = require('request');
+var debug = require('../../util/debug')('capacity-http-simple');
+
+function HttpSimpleCapacity(host, capacityEndpoint) {
+    this.host = host;
+    this.capacityEndpoint = capacityEndpoint;
+
+    this.lastResponse = null;
+    this.lastResponseTime = 0;
+}
+
+module.exports = HttpSimpleCapacity;
+
+HttpSimpleCapacity.prototype.getCapacity = function(callback) {
+    this.getResponse(function(err, values) {
+        var capacity = 1;
+
+        if (err) {
+            return callback(null, capacity);
+        }
+
+        var availableCores = parseInt(values.available_cores, 10);
+
+        capacity = Math.max(availableCores, 1);
+        capacity = Number.isFinite(capacity) ? capacity : 1;
+
+        debug('host=%s, capacity=%s', this.host, capacity);
+        return callback(null, capacity);
+    }.bind(this));
+};
+
+HttpSimpleCapacity.prototype.getResponse = function(callback) {
+    var requestParams = {
+        method: 'POST',
+        url: this.capacityEndpoint,
+        timeout: 2000,
+        json: true
+    };
+    debug('getCapacity(%s)', this.host);
+
+    // throttle requests for 500 ms
+    var now = Date.now();
+    if (this.lastResponse !== null && ((now - this.lastResponseTime) < 500)) {
+        return callback(null, this.lastResponse);
+    }
+
+    request.post(requestParams, function(err, res, jsonRes) {
+        if (err) {
+            return callback(err);
+        }
+        if (jsonRes && jsonRes.retcode === 0) {
+            this.lastResponse = jsonRes.return_values || {};
+            // We could go more aggressive by updating lastResponseTime on failures.
+            this.lastResponseTime = now;
+
+            return callback(null, this.lastResponse);
+        }
+        return callback(new Error('Could not retrieve information from endpoint'));
+    }.bind(this));
+};

batch/scheduler/host-scheduler.js

@@ -0,0 +1,85 @@
+'use strict';
+
+var _ = require('underscore');
+var debug = require('../util/debug')('host-scheduler');
+var Scheduler = require('./scheduler');
+var Locker = require('../leader/locker');
+var FixedCapacity = require('./capacity/fixed');
+var HttpSimpleCapacity = require('./capacity/http-simple');
+var HttpLoadCapacity = require('./capacity/http-load');
+
+function HostScheduler(name, taskRunner, redisPool) {
+    this.name = name || 'scheduler';
+    this.taskRunner = taskRunner;
+    this.locker = Locker.create('redis-distlock', { pool: redisPool });
+    this.locker.on('error', function(err, host) {
+        debug('[%s] Locker.error %s', this.name, err.message);
+        this.unlock(host);
+    }.bind(this));
+    // host => Scheduler
+    this.schedulers = {};
+}
+
+module.exports = HostScheduler;
+
+HostScheduler.prototype.add = function(host, user, callback) {
+    this.lock(host, function(err, scheduler) {
+        if (err) {
+            debug('[%s] Could not lock host=%s', this.name, host);
+            return callback(err);
+        }
+        scheduler.add(user);
+        var wasRunning = scheduler.schedule();
+        debug('[%s] Scheduler host=%s was running=%s', this.name, host, wasRunning);
+        return callback(err, wasRunning);
+    }.bind(this));
+};
+
+HostScheduler.prototype.getCapacityProvider = function(host) {
+    var strategy = global.settings.batch_capacity_strategy;
+
+    if (strategy === 'http-simple' || strategy === 'http-load') {
+        if (global.settings.batch_capacity_http_url_template) {
+            var endpoint = _.template(global.settings.batch_capacity_http_url_template, { dbhost: host });
+            debug('Using strategy=%s capacity. Endpoint=%s', strategy, endpoint);
+
+            if (strategy === 'http-simple') {
+                return new HttpSimpleCapacity(host, endpoint);
+            }
+            return new HttpLoadCapacity(host, endpoint);
+        }
+    }
+
+    var fixedCapacity = global.settings.batch_capacity_fixed_amount || 4;
+    debug('Using strategy=fixed capacity=%d', fixedCapacity);
+    return new FixedCapacity(fixedCapacity);
+};
+
+HostScheduler.prototype.lock = function(host, callback) {
+    debug('[%s] lock(%s)', this.name, host);
+    var self = this;
+    this.locker.lock(host, function(err) {
+        if (err) {
+            debug('[%s] Could not lock host=%s. Reason: %s', self.name, host, err.message);
+            return callback(err);
+        }
+
+        if (!self.schedulers.hasOwnProperty(host)) {
+            var scheduler = new Scheduler(self.getCapacityProvider(host), self.taskRunner);
+            scheduler.on('done', self.unlock.bind(self, host));
+            self.schedulers[host] = scheduler;
+        }
+
+        debug('[%s] Locked host=%s', self.name, host);
+        return callback(null, self.schedulers[host]);
+    });
+};
+
+HostScheduler.prototype.unlock = function(host) {
+    debug('[%s] unlock(%s)', this.name, host);
+    if (this.schedulers.hasOwnProperty(host)) {
+        // TODO stop scheduler?
+        delete this.schedulers[host];
+    }
+    this.locker.unlock(host, debug);
+};

batch/scheduler/scheduler.js

@@ -0,0 +1,201 @@
+'use strict';
+
+// Inspiration from:
+// - https://www.kernel.org/doc/Documentation/scheduler/sched-design-CFS.txt
+// - https://www.kernel.org/doc/Documentation/rbtree.txt
+// - http://www.ibm.com/developerworks/linux/library/l-completely-fair-scheduler/
+
+var util = require('util');
+var EventEmitter = require('events').EventEmitter;
+var RBTree = require('bintrees').RBTree;
+
+var debug = require('../util/debug')('scheduler');
+
+var forever = require('../util/forever');
+
+function Scheduler(capacity, taskRunner) {
+    EventEmitter.call(this);
+    debug('new Scheduler');
+    this.taskRunner = taskRunner;
+    this.capacity = capacity;
+    this.tasks = [];
+    this.users = {};
+    this.tasksTree = new RBTree(function(taskEntityA, taskEntityB) {
+        // if the user is the same it's the same entity
+        if (taskEntityA.user === taskEntityB.user) {
+            return 0;
+        }
+
+        // priority for entity with less executed jobs
+        if (taskEntityA.jobs !== taskEntityB.jobs) {
+            return taskEntityA.jobs - taskEntityB.jobs;
+        }
+
+        // priority for oldest job
+        if (taskEntityA.createdAt !== taskEntityB.createdAt) {
+            return taskEntityA.createdAt - taskEntityB.createdAt;
+        }
+
+        // we don't care if we arrive here
+        return -1;
+    });
+}
+util.inherits(Scheduler, EventEmitter);
+
+module.exports = Scheduler;
+
+Scheduler.prototype.add = function(user) {
+    debug('add(%s)', user);
+    var taskEntity = this.users[user];
+    if (taskEntity) {
+        if (taskEntity.status === STATUS.DONE) {
+            taskEntity.status = STATUS.PENDING;
+            this.tasksTree.insert(taskEntity);
+            this.emit('add');
+        }
+
+        return true;
+    } else {
+        taskEntity = new TaskEntity(user, this.tasks.length);
+        this.tasks.push(taskEntity);
+        this.users[user] = taskEntity;
+        this.tasksTree.insert(taskEntity);
+
+        this.emit('add');
+
+        return false;
+    }
+};
+
+Scheduler.prototype.schedule = function() {
+    if (this.running) {
+        return true;
+    }
+    this.running = true;
+
+    var self = this;
+    forever(
+        function (next) {
+            debug('Waiting for task');
+            self.acquire(function(err, taskEntity) {
+                debug('Acquired user=%j', taskEntity);
+
+                if (!taskEntity) {
+                    return next(new Error('all users finished'));
+                }
+
+                self.tasksTree.remove(taskEntity);
+                taskEntity.running();
+
+                debug('Running task for user=%s', taskEntity.user);
+                self.taskRunner.run(taskEntity.user, function(err, userQueueIsEmpty) {
+                    debug('Run task=%j, done=%s', taskEntity, userQueueIsEmpty);
+                    taskEntity.ran(userQueueIsEmpty);
+                    self.release(err, taskEntity);
+                });
+
+                // try to acquire next user
+                // will block until capacity slot is available
+                next();
+            });
+        },
+        function (err) {
+            debug('done: %s', err.message);
+            self.running = false;
+            self.emit('done');
+            self.removeAllListeners();
+        }
+    );
+
+    return false;
+};
+
+Scheduler.prototype.acquire = function(callback) {
+    this.removeAllListeners('add');
+    this.removeAllListeners('release');
+
+    if (this.tasks.every(is(STATUS.DONE))) {
+        return callback(null, null);
+    }
+
+    var self = this;
+    this.capacity.getCapacity(function(err, capacity) {
+        if (err) {
+            return callback(err);
+        }
+
+        debug('Trying to acquire task');
+        var running = self.tasks.filter(is(STATUS.RUNNING));
+        debug('[capacity=%d, running=%d] candidates=%j', capacity, running.length, self.tasks);
+
+        self.once('add', function() {
+            debug('Got a new task');
+            self.acquire(callback);
+        });
+        self.once('release', function() {
+            debug('Slot was released');
+            self.acquire(callback);
+        });
+
+        if (running.length >= capacity) {
+            debug('Not enough capacity');
+            return null;
+        }
+
+        var isRunningAny = self.tasks.some(is(STATUS.RUNNING));
+        var candidate = self.tasksTree.min();
+        if (isRunningAny && candidate === null) {
+            debug('Waiting for last task to finish');
+            return null;
+        }
+
+        if (candidate) {
+            self.emit('acquired', candidate.user);
+        }
+
+        return callback(null, candidate);
+    });
+};
+
+Scheduler.prototype.release = function(err, taskEntity) {
+    debug('Released %j', taskEntity);
+    if (taskEntity.is(STATUS.PENDING)) {
+        this.tasksTree.insert(taskEntity);
+    }
+    this.emit('release');
+};
+
+
+/* Task entities */
+
+var STATUS = {
+    PENDING: 'pending',
+    RUNNING: 'running',
+    DONE: 'done'
+};
+
+function TaskEntity(user, createdAt) {
+    this.user = user;
+    this.createdAt = createdAt;
+    this.status = STATUS.PENDING;
+    this.jobs = 0;
+}
+
+TaskEntity.prototype.is = function(status) {
+    return this.status === status;
+};
+
+TaskEntity.prototype.running = function() {
+    this.status = STATUS.RUNNING;
+};
+
+TaskEntity.prototype.ran = function(userQueueIsEmpty) {
+    this.jobs++;
+    this.status = userQueueIsEmpty ? STATUS.DONE : STATUS.PENDING;
+};
+
+function is(status) {
+    return function(taskEntity) {
+        return taskEntity.is(status);
+    };
+}