Release 1.20.2

Upgrades Windshaft to report redis/renderers/mapnik pool metrics

.gitignore

@@ -5,3 +5,5 @@ config/environments/*.js
 tools/munin/windshaft.conf
 logs/
 pids/
+redis.pid
+test.log

.travis.yml

@@ -1,27 +1,10 @@
+addons:
+  postgresql: "9.3"
+
 before_install:
-  - sudo mv /etc/apt/sources.list.d/pgdg-source.list* /tmp
-  - sudo apt-get -qq purge postgis* postgresql*
-  - sudo apt-add-repository --yes ppa:cartodb/postgresql-9.3
-  - sudo apt-add-repository --yes ppa:cartodb/gis
-  - sudo rm -Rf /var/lib/postgresql /etc/postgresql
-  - sudo apt-add-repository --yes ppa:mapnik/nightly-2.3
-  - sudo apt-get update
-  - sudo apt-get install -y postgresql-9.3-postgis-2.1
-  - sudo apt-get install -y postgresql-contrib-9.3
-  - sudo apt-get install -q libprotobuf-dev protobuf-compiler
-  - sudo apt-get install -q libmapnik-dev
-  - sudo apt-get install -q mapnik-input-plugin-gdal mapnik-input-plugin-ogr mapnik-input-plugin-postgis
-  - sudo apt-get install -y gdal-bin
-  - echo -e "local\tall\tall\ttrust\nhost\tall\tall\t127.0.0.1/32\ttrust\nhost\tall\tall\t::1/128\ttrust" |sudo tee /etc/postgresql/9.3/main/pg_hba.conf
-  - sudo service postgresql restart
   - createdb template_postgis
   - psql -c "CREATE EXTENSION postgis" template_postgis
 
-before_script:
-  # Tell npm to use known registrars:
-  # see http://blog.npmjs.org/post/78085451721/npms-self-signed-certificate-is-no-more
-  - npm config set ca ""
-
 env:
   - NPROCS=1 JOBS=1 PGUSER=postgres
 

NEWS.md

@@ -1,3 +1,62 @@
+1.20.2 -- 2014-10-20
+--------------------
+
+Announcements:
+ - Upgrades windshaft to 0.31.0
+
+
+1.20.1 -- 2014-10-17
+--------------------
+
+Announcements:
+ - Upgrades redis-mpool to 0.3.0
+
+
+1.20.0 -- 2014-10-15
+--------------------
+
+New features:
+ - Report to statsd the status of redis pools
+ - Upgrades Windshaft to start reporting redis/renderers/mapnik pool metrics
+
+Enhancements:
+ - Share one redis-mpool across the application
+
+
+1.19.0 -- 2014-10-14
+--------------------
+
+Announcements:
+ - Dropping support for npm <1.2.1
+   npm-shrinkwrap.json is incompatible when generated with npm >=1.2.1 and consumed by npm <1.2.1
+ - Upgrades windshaft to 0.28.2
+ - Generates npm-shrinkwrap.json with npm >1.2.0
+
+
+1.18.2 -- 2014-10-13
+--------------------
+
+Bug fixes:
+ - Defaults resultSet to object if undefined in QueryTablesApi
+
+Announcements:
+ - Upgrades windshaft to 0.28.1
+
+
+1.18.1 -- 2014-10-13
+--------------------
+
+New features:
+ - Allow to add more node.js' threadpool workers via process.env.UV_THREADPOOL_SIZE
+
+
+1.18.0 -- 2014-10-03
+--------------------
+
+Announcements:
+ - Comes back to use mapnik 2.3.x based on cartodb/node-mapnik@1.4.15-cdb from windshaft@0.28.0
+
+
 1.17.2 -- 2014-10-01
 --------------------
 

app.js

@@ -8,7 +8,8 @@
  */
 
 var path = require('path'),
-    fs = require('fs')
+    fs = require('fs'),
+    RedisPool = require('redis-mpool')
 ;
 
 
@@ -31,12 +32,16 @@ var _ = require('underscore');
 global.environment  = require(__dirname + '/config/environments/' + ENV);
 global.environment.api_hostname = require('os').hostname().split('.')[0];
 
-global.log4js = require('log4js')
+global.log4js = require('log4js');
 log4js_config = {
   appenders: [],
   replaceConsole:true
 };
 
+if (global.environment.uv_threadpool_size) {
+    process.env.UV_THREADPOOL_SIZE = global.environment.uv_threadpool_size;
+}
+
 if ( global.environment.log_filename ) {
   var logdir = path.dirname(global.environment.log_filename);
   // See cwd inlog4js.configure call below
@@ -65,13 +70,25 @@ if ( global.environment.rollbar ) {
 log4js.configure(log4js_config, { cwd: __dirname });
 global.logger = log4js.getLogger();
 
+var redisOpts = _.extend(global.environment.redis, { name: 'windshaft' }),
+    redisPool = new RedisPool(redisOpts);
+
 // Include cartodb_windshaft only _after_ the "global" variable is set
 // See https://github.com/Vizzuality/Windshaft-cartodb/issues/28
-var CartodbWindshaft = require('./lib/cartodb/cartodb_windshaft');
-var serverOptions = require('./lib/cartodb/server_options')();
+var CartodbWindshaft = require('./lib/cartodb/cartodb_windshaft'),
+    serverOptions = require('./lib/cartodb/server_options')(redisPool);
 
 ws = CartodbWindshaft(serverOptions);
 
+if (global.statsClient) {
+    redisPool.on('status', function(status) {
+        var keyPrefix = 'windshaft.redis-pool.' + status.name + '.db' + status.db + '.';
+        global.statsClient.gauge(keyPrefix + 'count', status.count);
+        global.statsClient.gauge(keyPrefix + 'unused', status.unused);
+        global.statsClient.gauge(keyPrefix + 'waiting', status.waiting);
+    });
+}
+
 // Maximum number of connections for one process
 // 128 is a good number if you have up to 1024 filedescriptors
 // 4 is good if you have max 32 filedescriptors

config/environments/development.js.example

@@ -2,6 +2,7 @@ var config = {
      environment: 'development'
     ,port: 8181
     ,host: '127.0.0.1'
+    ,uv_threadpool_size: undefined
     // Regular expression pattern to extract username
     // from hostname. Must have a single grabbing block.
     ,user_from_host: '^(.*)\\.localhost'
@@ -85,7 +86,8 @@ var config = {
       // Milliseconds since last access before renderer cache item expires
       cache_ttl: 60000,
       metatile: 4,
-      bufferSize: 64
+      bufferSize: 64,
+      statsInterval: 5000 // milliseconds between each report to statsd about number of renderers and mapnik pool status
     }
     ,millstone: {
         // Needs to be writable by server user
@@ -102,11 +104,19 @@ var config = {
         // by 2 to know how many possible connections will be
         // kept open by the server. The default is 50.
         max: 50,
+        returnToHead: true, // defines the behaviour of the pool: false => queue, true => stack
         idleTimeoutMillis: 1, // idle time before dropping connection
         reapIntervalMillis: 1, // time between cleanups
+        slowQueries: {
+            log: true,
+            elapsedThreshold: 200
+        },
         slowPool: {
             log: true, // whether a slow acquire must be logged or not
             elapsedThreshold: 25 // the threshold to determine an slow acquire must be reported or not
+        },
+        emitter: {
+            statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd
         }
     }
     ,sqlapi: {

config/environments/production.js.example

@@ -2,6 +2,7 @@ var config = {
     environment: 'production'
     ,port: 8181
     ,host: '127.0.0.1'
+    ,uv_threadpool_size: undefined
     // Regular expression pattern to extract username
     // from hostname. Must have a single grabbing block.
     ,user_from_host: '^(.*)\\.cartodb\\.com$'
@@ -79,7 +80,8 @@ var config = {
       // Milliseconds since last access before renderer cache item expires
       cache_ttl: 60000,
       metatile: 4,
-      bufferSize: 64
+      bufferSize: 64,
+      statsInterval: 5000 // milliseconds between each report to statsd about number of renderers and mapnik pool status
     }
     ,millstone: {
         // Needs to be writable by server user
@@ -96,11 +98,19 @@ var config = {
         // by 2 to know how many possible connections will be
         // kept open by the server. The default is 50.
         max: 50,
+        returnToHead: true, // defines the behaviour of the pool: false => queue, true => stack
         idleTimeoutMillis: 30000, // idle time before dropping connection
         reapIntervalMillis: 1000, // time between cleanups
+        slowQueries: {
+            log: true,
+            elapsedThreshold: 200
+        },
         slowPool: {
             log: true, // whether a slow acquire must be logged or not
             elapsedThreshold: 25 // the threshold to determine an slow acquire must be reported or not
+        },
+        emitter: {
+            statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd
         }
     }
     ,sqlapi: {

config/environments/staging.js.example

@@ -2,6 +2,7 @@ var config = {
     environment: 'production'
     ,port: 8181
     ,host: '127.0.0.1'
+    ,uv_threadpool_size: undefined
     // Regular expression pattern to extract username
     // from hostname. Must have a single grabbing block.
     ,user_from_host: '^(.*)\\.cartodb\\.com$'
@@ -79,7 +80,8 @@ var config = {
       // Milliseconds since last access before renderer cache item expires
       cache_ttl: 60000,
       metatile: 4,
-      bufferSize: 64
+      bufferSize: 64,
+      statsInterval: 5000 // milliseconds between each report to statsd about number of renderers and mapnik pool status
     }
     ,millstone: {
         // Needs to be writable by server user
@@ -96,11 +98,19 @@ var config = {
         // by 2 to know how many possible connections will be
         // kept open by the server. The default is 50.
         max: 50,
+        returnToHead: true, // defines the behaviour of the pool: false => queue, true => stack
         idleTimeoutMillis: 30000, // idle time before dropping connection
         reapIntervalMillis: 1000, // time between cleanups
+        slowQueries: {
+            log: true,
+            elapsedThreshold: 200
+        },
         slowPool: {
             log: true, // whether a slow acquire must be logged or not
             elapsedThreshold: 25 // the threshold to determine an slow acquire must be reported or not
+        },
+        emitter: {
+            statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd
         }
     }
     ,sqlapi: {

config/environments/test.js.example

@@ -2,6 +2,7 @@ var config = {
      environment: 'test'
     ,port: 8888
     ,host: '127.0.0.1'
+    ,uv_threadpool_size: undefined
     // Regular expression pattern to extract username
     // from hostname. Must have a single grabbing block.
     ,user_from_host: '(.*)'
@@ -79,7 +80,8 @@ var config = {
       // Milliseconds since last access before renderer cache item expires
       cache_ttl: 60000,
       metatile: 4,
-      bufferSize: 64
+      bufferSize: 64,
+      statsInterval: 5000 // milliseconds between each report to statsd about number of renderers and mapnik pool status
     }
     ,millstone: {
         // Needs to be writable by server user
@@ -96,11 +98,19 @@ var config = {
         // by 2 to know how many possible connections will be
         // kept open by the server. The default is 50.
         max: 50,
+        returnToHead: true, // defines the behaviour of the pool: false => queue, true => stack
         idleTimeoutMillis: 1, // idle time before dropping connection
         reapIntervalMillis: 1, // time between cleanups
+        slowQueries: {
+            log: true,
+            elapsedThreshold: 200
+        },
         slowPool: {
             log: true, // whether a slow acquire must be logged or not
             elapsedThreshold: 25 // the threshold to determine an slow acquire must be reported or not
+        },
+        emitter: {
+            statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd
         }
     }
     ,sqlapi: {

lib/cartodb/api/query_tables_api.js

@@ -93,6 +93,7 @@ function runQuery(username, options, query, queryHandler, callback) {
     if (shouldQueryPostgresDirectly()) {
         var psql = new PSQL(options);
         psql.query(query, function(err, resultSet) {
+            resultSet = resultSet || {};
             var rows = resultSet.rows || [];
             queryHandler(err, rows, callback);
         });

lib/cartodb/cartodb_windshaft.js

@@ -1,10 +1,6 @@
-
 var _ = require('underscore')
     , Step       = require('step')
     , Windshaft = require('windshaft')
-    , redisPool = require('redis-mpool')(_.extend(global.environment.redis, {name: 'windshaft:cartodb'}))
-    // TODO: instanciate cartoData with redisPool
-    , cartoData  = require('cartodb-redis')(global.environment.redis)
     , SignedMaps = require('./signed_maps.js')
     , TemplateMaps = require('./template_maps.js')
     , Cache = require('./cache_validator')
@@ -26,6 +22,11 @@ var CartodbWindshaft = function(serverOptions) {
       }
     }
 
+    var redisPool = serverOptions.redis.pool
+        || require('redis-mpool')(_.extend(global.environment.redis, {name: 'windshaft:cartodb'}));
+
+    var cartoData = require('cartodb-redis')({pool: redisPool});
+
     if(serverOptions.cache_enabled) {
         console.log("cache invalidation enabled, varnish on ", serverOptions.varnish_host, ' ', serverOptions.varnish_port);
         Cache.init(serverOptions.varnish_host, serverOptions.varnish_port, serverOptions.varnish_secret);
@@ -41,7 +42,7 @@ var CartodbWindshaft = function(serverOptions) {
           err = new Error("map state cannot be changed by unauthenticated request!");
         }
         callback(err, req);
-    }
+    };
 
     // This is for Templated maps
     //
@@ -52,7 +53,7 @@ var CartodbWindshaft = function(serverOptions) {
     serverOptions.signedMaps = new SignedMaps(redisPool);
     var templateMapsOpts = {
       max_user_templates: global.environment.maxUserTemplates
-    }
+    };
     var templateMaps = new TemplateMaps(redisPool, serverOptions.signedMaps, templateMapsOpts);
 
     // boot
@@ -64,7 +65,7 @@ var CartodbWindshaft = function(serverOptions) {
       var version = wsversion();
       version.windshaft_cartodb = require('../../package.json').version;
       return version;
-    }
+    };
 
     var ws_sendResponse = ws.sendResponse;
     // GET routes for which we don't want to request any caching.
@@ -202,7 +203,7 @@ var CartodbWindshaft = function(serverOptions) {
 
     ws.userByReq = function(req) {
         return serverOptions.userByReq(req);
-    }
+    };
 
     // Add a template
     ws.post(template_baseurl, function(req, res) {
@@ -445,7 +446,7 @@ var CartodbWindshaft = function(serverOptions) {
         function prepareResponse(err, tpl_ids){
           if ( err ) throw err;
           // NOTE: might omit "cbduser" if == dbowner ...
-          var ids = _.map(tpl_ids, function(id) { return cdbuser + '@' + id; })
+          var ids = _.map(tpl_ids, function(id) { return cdbuser + '@' + id; });
           return { template_ids: ids };
         },
         function finish(err, response){
@@ -500,7 +501,7 @@ var CartodbWindshaft = function(serverOptions) {
         if ( tpl_id[0] && tpl_id[0] != cdbuser ) {
           var err = new Error('Cannot instanciate map of user "'
                               + tpl_id[0] + '" on database of user "'
-                              + cdbuser + '"')
+                              + cdbuser + '"');
           err.http_status = 403;
           callback(err);
           return;
@@ -665,6 +666,6 @@ var CartodbWindshaft = function(serverOptions) {
     // ---- Template maps interface ends @}
 
     return ws;
-}
+};
 
 module.exports = CartodbWindshaft;

lib/cartodb/server_options.js

@@ -1,10 +1,9 @@
 var   _          = require('underscore')
     , Step       = require('step')
-    , cartoData  = require('cartodb-redis')(global.environment.redis)
     , Cache      = require('./cache_validator')
     , QueryTablesApi   = require('./api/query_tables_api')
     , crypto     = require('crypto')
-    , LZMA       = require('lzma').LZMA;
+    , LZMA       = require('lzma').LZMA
   ;
 
 // This is for backward compatibility with 1.3.3
@@ -16,16 +15,17 @@ if ( _.isUndefined(global.environment.sqlapi.domain) ) {
   }
 }
 
-module.exports = function(){
-
-    var lzmaWorker = new LZMA();
-
-    var queryTablesApi = new QueryTablesApi();
+module.exports = function(redisPool) {
+    var redisOpts = redisPool ? {pool: redisPool} : global.environment.redis;
+    var cartoData = require('cartodb-redis')(redisOpts),
+        lzmaWorker = new LZMA(),
+        queryTablesApi = new QueryTablesApi();
 
     var rendererConfig = _.defaults(global.environment.renderer || {}, {
-      cache_ttl: 60000, // milliseconds
-      metatile: 4,
-      bufferSize: 64
+        cache_ttl: 60000, // milliseconds
+        metatile: 4,
+        bufferSize: 64,
+        statsInterval: 60000
     });
 
     var me = {
@@ -61,7 +61,8 @@ module.exports = function(){
         },
         statsd: global.environment.statsd,
         renderCache: {
-          ttl: rendererConfig.cache_ttl
+            ttl: rendererConfig.cache_ttl,
+            statsInterval: rendererConfig.statsInterval
         },
         redis: global.environment.redis,
         enable_cors: global.environment.enable_cors,
@@ -77,6 +78,9 @@ module.exports = function(){
     // See http://github.com/CartoDB/Windshaft-cartodb/issues/161
     me.redis.unwatchOnRelease = false;
 
+    // Re-use redisPool
+    me.redis.pool = redisPool;
+
 /* This whole block is about generating X-Cache-Channel { */
 
     // TODO: review lifetime of elements of this cache

package.json

@@ -1,7 +1,7 @@
 {
     "private": true,
     "name": "windshaft-cartodb",
-    "version": "1.17.2",
+    "version": "1.20.2",
     "description": "A map tile server for CartoDB",
     "keywords": [
       "cartodb"
@@ -25,12 +25,12 @@
         "node-varnish": "https://github.com/Vizzuality/node-varnish/tarball/0.3.0",
         "underscore"  : "~1.6.0",
         "dot": "~1.0.2",
-        "windshaft": "https://github.com/CartoDB/Windshaft/tarball/0.27.2",
+        "windshaft": "https://github.com/CartoDB/Windshaft/tarball/0.31.0",
         "step": "~0.0.5",
         "request": "~2.9.203",
         "cartodb-redis": "https://github.com/CartoDB/node-cartodb-redis/tarball/0.11.0",
         "cartodb-psql": "https://github.com/CartoDB/node-cartodb-psql/tarball/0.4.0",
-        "redis-mpool": "https://github.com/CartoDB/node-redis-mpool/tarball/0.1.0",
+        "redis-mpool": "https://github.com/CartoDB/node-redis-mpool/tarball/0.3.0",
         "lzma": "~1.3.7",
         "log4js": "~0.6.17",
         "rollbar": "~0.3.13"
@@ -43,5 +43,9 @@
     },
     "scripts": {
         "test": "make check"
+    },
+    "engines": {
+        "node": ">=0.8 <0.11",
+        "npm": ">=1.2.1"
     }
 }