diff --git a/NEWS.md b/NEWS.md index 7a10b7a8..439acd10 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,12 @@ 1.19.1 -- 2014-mm-dd -------------------- +New features: + - Report to statsd the status of redis pools + +Enhancements: + - Share one redis-mpool across the application + 1.19.0 -- 2014-10-14 -------------------- diff --git a/app.js b/app.js index 0d946375..e8d5ea44 100755 --- a/app.js +++ b/app.js @@ -8,7 +8,8 @@ */ var path = require('path'), - fs = require('fs') + fs = require('fs'), + RedisPool = require('redis-mpool') ; @@ -31,7 +32,7 @@ var _ = require('underscore'); global.environment = require(__dirname + '/config/environments/' + ENV); global.environment.api_hostname = require('os').hostname().split('.')[0]; -global.log4js = require('log4js') +global.log4js = require('log4js'); log4js_config = { appenders: [], replaceConsole:true @@ -69,13 +70,25 @@ if ( global.environment.rollbar ) { log4js.configure(log4js_config, { cwd: __dirname }); global.logger = log4js.getLogger(); +var redisOpts = _.extend(global.environment.redis, { name: 'windshaft' }), + redisPool = new RedisPool(redisOpts); + // Include cartodb_windshaft only _after_ the "global" variable is set // See https://github.com/Vizzuality/Windshaft-cartodb/issues/28 -var CartodbWindshaft = require('./lib/cartodb/cartodb_windshaft'); -var serverOptions = require('./lib/cartodb/server_options')(); +var CartodbWindshaft = require('./lib/cartodb/cartodb_windshaft'), + serverOptions = require('./lib/cartodb/server_options')(redisPool); ws = CartodbWindshaft(serverOptions); +if (global.statsClient) { + redisPool.on('status', function(status) { + var keyPrefix = status.name + '.db' + status.db + '.'; + global.statsClient.gauge(keyPrefix + 'count', status.count); + global.statsClient.gauge(keyPrefix + 'unused', status.unused); + global.statsClient.gauge(keyPrefix + 'waiting', status.waiting); + }); +} + // Maximum number of connections for one process // 128 is a good number if you have up to 1024 filedescriptors // 4 is good if you have max 32 filedescriptors diff --git a/config/environments/development.js.example b/config/environments/development.js.example index 0abc4637..fafb283f 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -112,6 +112,9 @@ var config = { slowPool: { log: true, // whether a slow acquire must be logged or not elapsedThreshold: 25 // the threshold to determine an slow acquire must be reported or not + }, + emitter: { + statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd } } ,sqlapi: { diff --git a/config/environments/production.js.example b/config/environments/production.js.example index 76fc3d50..c4fb2707 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -106,6 +106,9 @@ var config = { slowPool: { log: true, // whether a slow acquire must be logged or not elapsedThreshold: 25 // the threshold to determine an slow acquire must be reported or not + }, + emitter: { + statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd } } ,sqlapi: { diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 98562d64..d0e93555 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -106,6 +106,9 @@ var config = { slowPool: { log: true, // whether a slow acquire must be logged or not elapsedThreshold: 25 // the threshold to determine an slow acquire must be reported or not + }, + emitter: { + statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd } } ,sqlapi: { diff --git a/config/environments/test.js.example b/config/environments/test.js.example index be080e82..b2ea2736 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -106,6 +106,9 @@ var config = { slowPool: { log: true, // whether a slow acquire must be logged or not elapsedThreshold: 25 // the threshold to determine an slow acquire must be reported or not + }, + emitter: { + statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd } } ,sqlapi: { diff --git a/lib/cartodb/cartodb_windshaft.js b/lib/cartodb/cartodb_windshaft.js index cb6db192..216d8682 100644 --- a/lib/cartodb/cartodb_windshaft.js +++ b/lib/cartodb/cartodb_windshaft.js @@ -1,10 +1,6 @@ - var _ = require('underscore') , Step = require('step') , Windshaft = require('windshaft') - , redisPool = require('redis-mpool')(_.extend(global.environment.redis, {name: 'windshaft:cartodb'})) - // TODO: instanciate cartoData with redisPool - , cartoData = require('cartodb-redis')(global.environment.redis) , SignedMaps = require('./signed_maps.js') , TemplateMaps = require('./template_maps.js') , Cache = require('./cache_validator') @@ -26,6 +22,11 @@ var CartodbWindshaft = function(serverOptions) { } } + var redisPool = serverOptions.redis.pool + || require('redis-mpool')(_.extend(global.environment.redis, {name: 'windshaft:cartodb'})); + + var cartoData = require('cartodb-redis')({pool: redisPool}); + if(serverOptions.cache_enabled) { console.log("cache invalidation enabled, varnish on ", serverOptions.varnish_host, ' ', serverOptions.varnish_port); Cache.init(serverOptions.varnish_host, serverOptions.varnish_port, serverOptions.varnish_secret); @@ -41,7 +42,7 @@ var CartodbWindshaft = function(serverOptions) { err = new Error("map state cannot be changed by unauthenticated request!"); } callback(err, req); - } + }; // This is for Templated maps // @@ -52,7 +53,7 @@ var CartodbWindshaft = function(serverOptions) { serverOptions.signedMaps = new SignedMaps(redisPool); var templateMapsOpts = { max_user_templates: global.environment.maxUserTemplates - } + }; var templateMaps = new TemplateMaps(redisPool, serverOptions.signedMaps, templateMapsOpts); // boot @@ -64,7 +65,7 @@ var CartodbWindshaft = function(serverOptions) { var version = wsversion(); version.windshaft_cartodb = require('../../package.json').version; return version; - } + }; var ws_sendResponse = ws.sendResponse; // GET routes for which we don't want to request any caching. @@ -202,7 +203,7 @@ var CartodbWindshaft = function(serverOptions) { ws.userByReq = function(req) { return serverOptions.userByReq(req); - } + }; // Add a template ws.post(template_baseurl, function(req, res) { @@ -445,7 +446,7 @@ var CartodbWindshaft = function(serverOptions) { function prepareResponse(err, tpl_ids){ if ( err ) throw err; // NOTE: might omit "cbduser" if == dbowner ... - var ids = _.map(tpl_ids, function(id) { return cdbuser + '@' + id; }) + var ids = _.map(tpl_ids, function(id) { return cdbuser + '@' + id; }); return { template_ids: ids }; }, function finish(err, response){ @@ -500,7 +501,7 @@ var CartodbWindshaft = function(serverOptions) { if ( tpl_id[0] && tpl_id[0] != cdbuser ) { var err = new Error('Cannot instanciate map of user "' + tpl_id[0] + '" on database of user "' - + cdbuser + '"') + + cdbuser + '"'); err.http_status = 403; callback(err); return; @@ -665,6 +666,6 @@ var CartodbWindshaft = function(serverOptions) { // ---- Template maps interface ends @} return ws; -} +}; module.exports = CartodbWindshaft; diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index 2fa27c8c..c8c9dbcc 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -1,10 +1,9 @@ var _ = require('underscore') , Step = require('step') - , cartoData = require('cartodb-redis')(global.environment.redis) , Cache = require('./cache_validator') , QueryTablesApi = require('./api/query_tables_api') , crypto = require('crypto') - , LZMA = require('lzma').LZMA; + , LZMA = require('lzma').LZMA ; // This is for backward compatibility with 1.3.3 @@ -16,11 +15,11 @@ if ( _.isUndefined(global.environment.sqlapi.domain) ) { } } -module.exports = function(){ - - var lzmaWorker = new LZMA(); - - var queryTablesApi = new QueryTablesApi(); +module.exports = function(redisPool) { + var redisOpts = redisPool ? {pool: redisPool} : global.environment.redis; + var cartoData = require('cartodb-redis')(redisOpts), + lzmaWorker = new LZMA(), + queryTablesApi = new QueryTablesApi(); var rendererConfig = _.defaults(global.environment.renderer || {}, { cache_ttl: 60000, // milliseconds @@ -77,6 +76,9 @@ module.exports = function(){ // See http://github.com/CartoDB/Windshaft-cartodb/issues/161 me.redis.unwatchOnRelease = false; + // Re-use redisPool + me.redis.pool = redisPool; + /* This whole block is about generating X-Cache-Channel { */ // TODO: review lifetime of elements of this cache diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index e72b9e40..fc38898e 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -70,7 +70,8 @@ "dependencies": { "async": { "version": "0.2.10", - "from": "async@~0.2.0" + "from": "async@~0.2.0", + "resolved": "https://registry.npmjs.org/async/-/async-0.2.10.tgz" }, "readable-stream": { "version": "1.0.33-1", @@ -90,7 +91,7 @@ }, "inherits": { "version": "2.0.1", - "from": "inherits@2" + "from": "inherits@~2.0.1" } } } @@ -106,9 +107,9 @@ "resolved": "https://github.com/Vizzuality/node-varnish/tarball/0.3.0" }, "redis-mpool": { - "version": "0.1.0", - "from": "https://github.com/CartoDB/node-redis-mpool/tarball/0.1.0", - "resolved": "https://github.com/CartoDB/node-redis-mpool/tarball/0.1.0", + "version": "0.2.0", + "from": "https://github.com/CartoDB/node-redis-mpool/tarball/0.2.0", + "resolved": "https://github.com/CartoDB/node-redis-mpool/tarball/0.2.0", "dependencies": { "generic-pool": { "version": "2.1.1", @@ -162,12 +163,13 @@ }, "underscore": { "version": "1.6.0", - "from": "underscore@~1.6.0" + "from": "underscore@~1.6.0", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz" }, "windshaft": { - "version": "0.28.2", - "from": "https://github.com/CartoDB/Windshaft/tarball/0.28.2", - "resolved": "https://github.com/CartoDB/Windshaft/tarball/0.28.2", + "version": "0.29.0", + "from": "https://github.com/CartoDB/Windshaft/tarball/0.29.0", + "resolved": "https://github.com/CartoDB/Windshaft/tarball/0.29.0", "dependencies": { "chronograph": { "version": "0.1.0", @@ -272,7 +274,8 @@ }, "async": { "version": "0.9.0", - "from": "async@~0.9.0" + "from": "async@~0.9.0", + "resolved": "https://registry.npmjs.org/async/-/async-0.9.0.tgz" } } }, @@ -1462,6 +1465,7 @@ "express": { "version": "2.5.11", "from": "express@~2.5.11", + "resolved": "https://registry.npmjs.org/express/-/express-2.5.11.tgz", "dependencies": { "connect": { "version": "1.9.2", @@ -1469,7 +1473,8 @@ "dependencies": { "formidable": { "version": "1.0.15", - "from": "formidable@1.0.x" + "from": "formidable@1.0.x", + "resolved": "https://registry.npmjs.org/formidable/-/formidable-1.0.15.tgz" } } }, diff --git a/package.json b/package.json index 1a30463e..6f77fcd6 100644 --- a/package.json +++ b/package.json @@ -25,12 +25,12 @@ "node-varnish": "https://github.com/Vizzuality/node-varnish/tarball/0.3.0", "underscore" : "~1.6.0", "dot": "~1.0.2", - "windshaft": "https://github.com/CartoDB/Windshaft/tarball/0.28.2", + "windshaft": "https://github.com/CartoDB/Windshaft/tarball/0.29.0", "step": "~0.0.5", "request": "~2.9.203", "cartodb-redis": "https://github.com/CartoDB/node-cartodb-redis/tarball/0.11.0", "cartodb-psql": "https://github.com/CartoDB/node-cartodb-psql/tarball/0.4.0", - "redis-mpool": "https://github.com/CartoDB/node-redis-mpool/tarball/0.1.0", + "redis-mpool": "https://github.com/CartoDB/node-redis-mpool/tarball/0.2.0", "lzma": "~1.3.7", "log4js": "~0.6.17", "rollbar": "~0.3.13"