From 5084e69aff0bcb6fb6c979c2e76b4ec4b4939561 Mon Sep 17 00:00:00 2001 From: javi Date: Wed, 7 Dec 2011 11:54:24 +0100 Subject: [PATCH 1/2] added varnish tests --- lib/cartodb/varnish.js | 69 +++++++++++++++--- test/acceptance/varnish.js | 141 +++++++++++++++++++++++++++++++++++++ 2 files changed, 202 insertions(+), 8 deletions(-) create mode 100644 test/acceptance/varnish.js diff --git a/lib/cartodb/varnish.js b/lib/cartodb/varnish.js index 0f2f94a1..3e5f0c6f 100644 --- a/lib/cartodb/varnish.js +++ b/lib/cartodb/varnish.js @@ -5,6 +5,7 @@ */ var net = require('net') +var EventEmitter = require('events').EventEmitter; function VarnishClient(host, port, ready_callback) { @@ -15,22 +16,28 @@ function VarnishClient(host, port, ready_callback) { var connected = false; var connecting = false; + function log() { + console.log.apply(console, arguments); + } + function connect() { if(connecting) return; connecting = true; - console.log("VARNISH: connection"); + log("VARNISH: connection"); ready = false; if(!client) { client = net.createConnection(port, host); client.on('connect', function () { - console.log("VARNISH: connected"); + log("VARNISH: connected"); connected = true; connecting = false; + self.emit('connect'); }); } else { client.connect(port, host); } } + self.connect = connect; connect(); @@ -45,22 +52,25 @@ function VarnishClient(host, port, ready_callback) { var body = lines[1]; if(!ready) { ready = true; - ready_callback(); + ready_callback && ready_callback(); + self.emit('ready'); } else if(cmd_callback) { var c = cmd_callback cmd_callback = null; c(null, code, body); + self.emit('response', code, body) } } }); client.on('error', function(err) { - console.log("[ERROR] some problem in varnish connection"); + log("[ERROR] some problem in varnish connection", err); + self.emit('error', err); }); - client.on('close', function() { - console.log("[INFO] closed varnish connection"); + client.on('close', function(e) { + log("[INFO] closed varnish connection"); self.close(); connected = false; connecting = false; @@ -80,11 +90,17 @@ function VarnishClient(host, port, ready_callback) { // fist param of the callback are the error, null // if all went ok this.run_cmd = function(cmd, callback) { - if(!connected) connect(); + if(!connected) { + connect(); + } if(!cmd_callback) { _send(cmd, callback); } else { callback('response pending'); + self.emit('error', { + code: 'RESPONSE_PENDING', + message: 'there is a response pending' + }); } } @@ -92,10 +108,13 @@ function VarnishClient(host, port, ready_callback) { this.close = function() { client.end(); ready = false; + self.emit('close'); } } +VarnishClient.prototype = new EventEmitter(); + function VarnishPool(opts, ready) { var resources = []; @@ -130,15 +149,45 @@ function VarnishPool(opts, ready) { function VarnishQueue(host, port) { + var self = this; var MAX_QUEUE = 2000; var queue = []; var ready = false; + var reconnectTimer = null; + var reconnectTries = 0; + var MAX_RECONNECT_TRIES = 50; - var client = new VarnishClient(host, port, function(err) { + var client = new VarnishClient(host, port); + + function log() { + console.log.apply(console, arguments); + } + + client.on('ready', function() { ready = true; + log('sending pending'); _send_pending(); + reconnectTries = 0; + clearInterval(reconnectTimer); }); + function reconnect() { + clearInterval(reconnectTimer); + reconnectTimer = setInterval(function() { + client.connect(); + ++reconnectTries; + if(reconnectTries >= MAX_RECONNECT_TRIES) { + self.emit('error', { + code: 'ABORT_RECONNECT', + message: 'max reconnect tries, abouting' + }); + clearInterval(reconnectTimer); + } + }, 500); + } + client.on('close', reconnect); + client.on('error', reconnect); + function _send_pending(empty_callback) { var c = queue.pop(); if(!c) return; @@ -149,6 +198,7 @@ function VarnishQueue(host, port) { if(empty_callback) { empty_callback(); } + self.emit('empty'); } }); } @@ -157,6 +207,7 @@ function VarnishQueue(host, port) { queue.push(cmd); if(queue.length > MAX_QUEUE) { console.log("varnish command queue too long, removing commands"); + self.emit('error', {code: 'TOO_LONG', message: "varnish command queue too long, removing commands"}); queue.pop(); } if(ready) { @@ -172,6 +223,8 @@ function VarnishQueue(host, port) { } +VarnishQueue.prototype = new EventEmitter(); + /* var queue = new VarnishQueue('localhost', 6082) setInterval(function() { diff --git a/test/acceptance/varnish.js b/test/acceptance/varnish.js new file mode 100644 index 00000000..3a34534b --- /dev/null +++ b/test/acceptance/varnish.js @@ -0,0 +1,141 @@ +var assert = require('assert'); +var net = require('net'); +require(__dirname + '/../test_helper'); +var varnish = require(__dirname + '/../../lib/cartodb/varnish'); +var tests = module.exports = {}; + +function VarnishEmu(on_cmd_recieved, port) { + var self = this; + var welcome_msg = 'hi, im a varnish emu, right?'; + + self.commands_recieved = []; + + var sockets = []; + var server = net.createServer(function (socket) { + var command = ''; + socket.write("200 " + welcome_msg.length + "\n"); + socket.write(welcome_msg); + socket.on('data', function(data) { + self.commands_recieved.push(data); + server.commands++; + on_cmd_recieved && on_cmd_recieved(self.commands_recieved); + socket.write('200 0\n'); + }); + sockets.push(socket); + }); + server.commands = 0; + server.listen(port || 0, "127.0.0.1"); + server.close_connections = function() { + for(var s in sockets) { + sockets[s].end(); + } + }; + return server; +} + +tests['ok'] = function() { + assert.ok(true); +}; + +tests['should connect'] = function() { + var ok = false; + var server = VarnishEmu(); + server.on('listening', function() { + var client = new varnish.VarnishClient('127.0.0.1', server.address().port); + client.on('connect', function() { + ok = true; + }); + }); + setTimeout(function() { assert.ok(ok); + server.close(); + }, 200); +}; + +tests['should send a command'] = function() { + var ok = false; + var server = VarnishEmu(function() { + ok = true; + }); + server.on('listening', function() { + var client = new varnish.VarnishClient('127.0.0.1', server.address().port); + client.on('ready', function() { + client.run_cmd('purge obj.http.X == test', function(){}); + }); + }); + setTimeout(function() { assert.ok(ok); }, 100); +} + +tests['should emit close on server disconect'] = function() { + var ok = false; + var server = VarnishEmu(); + server.on('listening', function() { + var client = new varnish.VarnishClient('127.0.0.1', server.address().port); + client.on('ready', function() { + client.on('close', function() { ok = true; }); + server.close_connections(); + server.close(); + }); + }); + setTimeout(function() { assert.ok(ok); }, 300); +} + +tests['should emit response on command'] = function() { + var ok = false; + var server = VarnishEmu() + server.on('listening', function() { + var client = new varnish.VarnishClient('127.0.0.1', server.address().port); + client.on('ready', function() { + client.run_cmd('purge obj.http.X == test', function(){}); + client.on('response', function(code, body) { + ok = true; + assert.equal(200, code); + }); + }); + }); + setTimeout(function() { assert.ok(ok); }, 100); +} + +tests['should emit error when the user tries to send when thereis a pending command'] = function() { + var ok = false; + var server = VarnishEmu() + server.on('listening', function() { + var client = new varnish.VarnishClient('127.0.0.1', server.address().port); + client.on('ready', function() { + client.run_cmd('purge obj.http.X == test', function(){}); + client.on('error', function(e) { + ok = true; + assert.equal('RESPONSE_PENDING', e.code); + }); + client.run_cmd('purge obj.http.X == test', function(){}); + }); + }); + setTimeout(function() { assert.ok(ok); }, 100); +}; + + +// +// queue +// + +tests['should send command'] = function() { + var server = VarnishEmu() + server.on('listening', function() { + var queue = new varnish.VarnishQueue('127.0.0.1', server.address().port); + for(var i = 0; i < 5; ++i) { + queue.run_cmd('purge simon_is == gay'); + } + }); + setTimeout(function() { assert.equal(5, server.commands); }, 100); +} + +tests['should send commands on connect'] = function() { + // first create queue + var queue = new varnish.VarnishQueue('127.0.0.1', 1234) + for(var i = 0; i < 5; ++i) { + queue.run_cmd('purge simon_is == gay'); + } + // then server + var server = VarnishEmu(null, 1234) + setTimeout(function() { assert.equal(5, server.commands); }, 1000); +} + From b53bcceb6dfae055f409cc349b96f5121b3ef41b Mon Sep 17 00:00:00 2001 From: javi Date: Wed, 7 Dec 2011 13:06:23 +0100 Subject: [PATCH 2/2] lovely fixes --- lib/cartodb/varnish.js | 22 ++++++++++++++++------ test/acceptance/varnish.js | 5 +++-- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/lib/cartodb/varnish.js b/lib/cartodb/varnish.js index 3e5f0c6f..a60d119f 100644 --- a/lib/cartodb/varnish.js +++ b/lib/cartodb/varnish.js @@ -21,7 +21,7 @@ function VarnishClient(host, port, ready_callback) { } function connect() { - if(connecting) return; + if(connecting || connected ) return; connecting = true; log("VARNISH: connection"); ready = false; @@ -30,8 +30,8 @@ function VarnishClient(host, port, ready_callback) { client.on('connect', function () { log("VARNISH: connected"); connected = true; - connecting = false; self.emit('connect'); + connecting = false; }); } else { client.connect(port, host); @@ -155,7 +155,7 @@ function VarnishQueue(host, port) { var ready = false; var reconnectTimer = null; var reconnectTries = 0; - var MAX_RECONNECT_TRIES = 50; + var MAX_RECONNECT_TRIES = 120; // 2 minutes var client = new VarnishClient(host, port); @@ -163,15 +163,24 @@ function VarnishQueue(host, port) { console.log.apply(console, arguments); } + // attach a dummy callback to error event to avoid nodejs throws an exception and closes the process + self.on('error', function(e) { + log("error", e); + }); + + client.on('connect', function() { + clearInterval(reconnectTimer); + reconnectTries = 0; + }); + client.on('ready', function() { ready = true; log('sending pending'); _send_pending(); - reconnectTries = 0; - clearInterval(reconnectTimer); }); function reconnect() { + ready = false; clearInterval(reconnectTimer); reconnectTimer = setInterval(function() { client.connect(); @@ -183,12 +192,13 @@ function VarnishQueue(host, port) { }); clearInterval(reconnectTimer); } - }, 500); + }, 1000); } client.on('close', reconnect); client.on('error', reconnect); function _send_pending(empty_callback) { + if(!ready) return; var c = queue.pop(); if(!c) return; client.run_cmd(c, function() { diff --git a/test/acceptance/varnish.js b/test/acceptance/varnish.js index 3a34534b..5a4dbd17 100644 --- a/test/acceptance/varnish.js +++ b/test/acceptance/varnish.js @@ -131,11 +131,12 @@ tests['should send command'] = function() { tests['should send commands on connect'] = function() { // first create queue var queue = new varnish.VarnishQueue('127.0.0.1', 1234) - for(var i = 0; i < 5; ++i) { + for(var i = 0; i < 10; ++i) { queue.run_cmd('purge simon_is == gay'); } // then server var server = VarnishEmu(null, 1234) - setTimeout(function() { assert.equal(5, server.commands); }, 1000); + //wait 2 seconds because the client tries every second the reconnection + setTimeout(function() { assert.equal(10, server.commands); }, 2000); }