Release 1.8.2

This is again for #157 without closing it

.travis.yml

@@ -11,6 +11,7 @@ env:
 language: node_js
 node_js:
   - "0.8"
+  - "0.10"
 
 notifications:
   irc:

HOWTO_RELEASE

@@ -4,8 +4,10 @@
 4. Run npm install
 5. Test (make check or npm test), fix if broken before proceeding
 6. Run npm shrinkwrap
-7. Commit package.json, npm-shrinwrap.json, NEWS
-8. Tag Major.Minor.Patch
-9. Announce
-10. Stub NEWS/package for next version
+7. Set "from" in npm-shrinkwrap.json for known packages
+   (windshaft, node-varnish, grainstore...)
+8. Commit package.json, npm-shrinwrap.json, NEWS
+9. git tag -a Major.Minor.Patch # use NEWS section as content
+10. Announce
+11. Stub NEWS/package for next version
 

NEWS.md

@@ -1,3 +1,45 @@
+1.8.2 -- 2014-02-25
+-------------------
+
+Enhancements:
+
+ * Allow using ":host" as part of statsd.prefix (#153)
+ * Expand "addCacheChannel" stats
+ * Allow using GET with sql-api for queries shorter than configured len (#155)
+   [ new sqlapi.max_get_sql_length directive, defaults to 2048 ]
+ * Do not log an error for a legit request requiring no X-Cache-Channel
+
+Bug fixes:
+
+ * Fix munin plugin after log format changes (#154)
+
+1.8.1 -- 2014-02-19
+-------------------
+
+Enhancements:
+
+ * Use log4js logger (#138)
+
+Bug fixes:
+
+ * Always generate X-Cache-Channel for token-based tile responses (#152)
+
+1.8.0 -- 2014-02-18
+-------------------
+
+Enhancements:
+
+ * Add script to flush caches (#140)
+ * Add script to list templates
+ * Add statsd support (#139)
+ * Add support for specifying a varnish password
+ * Avoid sending multiple varnish invalidation at once (#135)
+ * Tested with node-0.10 (#141)
+ * Use single redis pooler for torque and grainstore
+ * Reduce cost of garbage collection for localized resources
+ * Allow limiting number of templates for each user (#136)
+ * Allow configuring TTL of mapConfigs via "mapConfigTTL"
+
 1.7.1 -- 2014-02-11
 -------------------
 

app.js

@@ -24,10 +24,19 @@ global.settings     = require(__dirname + '/config/settings');
 global.environment  = require(__dirname + '/config/environments/' + ENV);
 _.extend(global.settings, global.environment);
 
+global.log4js = require('log4js')
+log4js.configure({
+  appenders: [
+    { type: "console", layout: { type:'basic' } }
+  ],
+  replaceConsole:true
+});
+
+
 // Include cartodb_windshaft only _after_ the "global" variable is set
 // See https://github.com/Vizzuality/Windshaft-cartodb/issues/28
 var CartodbWindshaft = require('./lib/cartodb/cartodb_windshaft');
-var serverOptions = require('./lib/cartodb/server_options');
+var serverOptions = require('./lib/cartodb/server_options')();
 
 ws = CartodbWindshaft(serverOptions);
 
@@ -52,3 +61,4 @@ process.on('SIGUSR1', function() {
 process.on('SIGUSR2', function() {
   ws.dumpCacheStats();
 });
+

config/environments/development.js.example

@@ -11,24 +11,31 @@ var config = {
     // See http://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API
     // 
     // Base url for the Templated Maps API
-    // "/maps/named" is the new API,
+    // "/api/v1/map/named" is the new API,
     // "/tiles/template" is for compatibility with versions up to 1.6.x
-    ,base_url_templated: '(?:/maps/named|/tiles/template)'
+    ,base_url_templated: '(?:/api/v1/map/named|/tiles/template)'
     // Base url for the Detached Maps API
     // "maps" is the the new API,
     // "tiles/layergroup" is for compatibility with versions up to 1.6.x
-    ,base_url_detached: '(?:/maps|/tiles/layergroup)'
+    ,base_url_detached: '(?:/api/v1/map|/tiles/layergroup)'
     // Base url for the Inline Maps and Table Maps API
     ,base_url_legacy: '/tiles/:table'
 
     // Maximum number of connections for one process
     // 128 is a good value with a limit of 1024 open file descriptors
     ,maxConnections:128
+    // Maximum number of templates per user. Unlimited by default.
+    ,maxUserTemplates:1024
+    // Seconds since "last creation" before a detached
+    // or template instance map expires. Or: how long do you want
+    // to be able to navigate the map without a reload ?
+    // Defaults to 7200 (2 hours)
+    ,mapConfigTTL: 7200
     // idle socket timeout, in miliseconds 
     ,socket_timeout: 600000
     ,enable_cors: true
     ,cache_enabled: false
-    ,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
+    ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
     // Templated database username for authorized user
     // Supported labels: 'user_id' (read from redis)
     ,postgres_auth_user: 'development_cartodb_user_<%= user_id %>'
@@ -54,6 +61,12 @@ var config = {
         max_size: 500
     }
     ,mapnik_version: undefined
+    ,statsd: {
+        host: 'localhost',
+        port: 8125,
+        prefix: 'dev.'
+        // support all allowed node-statsd options
+    }
     ,renderer: {
       // Milliseconds since last access before renderer cache item expires
       cache_ttl: 60000,
@@ -89,11 +102,16 @@ var config = {
         // the cartodb username and passed to
         // SQL-API requests in the Host HTTP header
         domain: 'localhost.lan',
-        version: 'v1'
+        version: 'v1',
+        // Maximum lenght of SQL query for GET
+        // requests. Longer queries will be sent
+        // using POST. Defaults to 2048
+        max_get_sql_length: 2048
     }
     ,varnish: {
         host: 'localhost',
         port: 6082,
+        secret: 'xxx',
         ttl: 86400
     }
     // If useProfiler is true every response will be served with an

config/environments/production.js.example

@@ -11,24 +11,31 @@ var config = {
     // See http://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API
     // 
     // Base url for the Templated Maps API
-    // "/maps/named" is the new API,
+    // "/api/v1/map/named" is the new API,
     // "/tiles/template" is for compatibility with versions up to 1.6.x
-    ,base_url_templated: '(?:/maps/named|/tiles/template)'
+    ,base_url_templated: '(?:/api/v1/map/named|/tiles/template)'
     // Base url for the Detached Maps API
     // "maps" is the the new API,
     // "tiles/layergroup" is for compatibility with versions up to 1.6.x
-    ,base_url_detached: '(?:/maps|/tiles/layergroup)'
+    ,base_url_detached: '(?:/api/v1/map|/tiles/layergroup)'
     // Base url for the Inline Maps and Table Maps API
     ,base_url_legacy: '/tiles/:table'
 
     // Maximum number of connections for one process
     // 128 is a good value with a limit of 1024 open file descriptors
     ,maxConnections:128
+    // Maximum number of templates per user. Unlimited by default.
+    ,maxUserTemplates:1024
+    // Seconds since "last creation" before a detached
+    // or template instance map expires. Or: how long do you want
+    // to be able to navigate the map without a reload ?
+    // Defaults to 7200 (2 hours)
+    ,mapConfigTTL: 7200
     // idle socket timeout, in miliseconds
     ,socket_timeout: 600000 
     ,enable_cors: true
     ,cache_enabled: true
-    ,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
+    ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
     // Templated database username for authorized user
     // Supported labels: 'user_id' (read from redis)
     ,postgres_auth_user: 'cartodb_user_<%= user_id %>'
@@ -48,6 +55,12 @@ var config = {
         max_size: 500
     }
     ,mapnik_version: undefined
+    ,statsd: {
+        host: 'localhost',
+        port: 8125,
+        prefix: ':host.', // could be hostname, better not containing dots
+        // support all allowed node-statsd options
+    }
     ,renderer: {
       // Milliseconds since last access before renderer cache item expires
       cache_ttl: 60000,
@@ -83,11 +96,16 @@ var config = {
         // the cartodb username and passed to
         // SQL-API requests in the Host HTTP header
         domain: 'cartodb.com',
-        version: 'v2'
+        version: 'v2',
+        // Maximum lenght of SQL query for GET
+        // requests. Longer queries will be sent
+        // using POST. Defaults to 2048
+        max_get_sql_length: 2048
     }
     ,varnish: {
         host: 'localhost',
         port: 6082,
+        secret: 'xxx',
         ttl: 86400
     }
     // If useProfiler is true every response will be served with an

config/environments/staging.js.example

@@ -24,11 +24,18 @@ var config = {
     // Maximum number of connections for one process
     // 128 is a good value with a limit of 1024 open file descriptors
     ,maxConnections:128
+    // Maximum number of templates per user. Unlimited by default.
+    ,maxUserTemplates:1024
+    // Seconds since "last creation" before a detached
+    // or template instance map expires. Or: how long do you want
+    // to be able to navigate the map without a reload ?
+    // Defaults to 7200 (2 hours)
+    ,mapConfigTTL: 7200
     // idle socket timeout, in miliseconds
     ,socket_timeout: 600000 
     ,enable_cors: true
     ,cache_enabled: true
-    ,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms (:res[X-Tiler-Profiler]) -> :res[Content-Type]'
+    ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms (:res[X-Tiler-Profiler]) -> :res[Content-Type]'
     // Templated database username for authorized user
     // Supported labels: 'user_id' (read from redis)
     ,postgres_auth_user: 'cartodb_staging_user_<%= user_id %>'
@@ -48,6 +55,12 @@ var config = {
         max_size: 500
     }
     ,mapnik_version: undefined
+    ,statsd: {
+        host: 'localhost',
+        port: 8125,
+        prefix: 'stage.:host.'
+        // support all allowed node-statsd options
+    }
     ,renderer: {
       // Milliseconds since last access before renderer cache item expires
       cache_ttl: 60000,
@@ -83,11 +96,16 @@ var config = {
         // the cartodb username and passed to
         // SQL-API requests in the Host HTTP header
         domain: 'cartodb.com',
-        version: 'v2'
+        version: 'v2',
+        // Maximum lenght of SQL query for GET
+        // requests. Longer queries will be sent
+        // using POST. Defaults to 2048
+        max_get_sql_length: 2048
     }
     ,varnish: {
         host: 'localhost',
         port: 6082,
+        secret: 'xxx',
         ttl: 86400
     }
     // If useProfiler is true every response will be served with an

config/environments/test.js.example

@@ -11,19 +11,26 @@ var config = {
     // See https://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API
     // 
     // Base url for the Templated Maps API
-    // "/maps/named" is the new API,
+    // "/api/v1/map/named" is the new API,
     // "/tiles/template" is for compatibility with versions up to 1.6.x
-    ,base_url_templated: '(?:/maps/named|/tiles/template)'
+    ,base_url_templated: '(?:/api/v1/map/named|/tiles/template)'
     // Base url for the Detached Maps API
     // "maps" is the the new API,
     // "tiles/layergroup" is for compatibility with versions up to 1.6.x
-    ,base_url_detached: '(?:/maps|/tiles/layergroup)'
+    ,base_url_detached: '(?:/api/v1/map|/tiles/layergroup)'
     // Base url for the Inline Maps and Table Maps API
     ,base_url_legacy: '/tiles/:table'
 
     // Maximum number of connections for one process
     // 128 is a good value with a limit of 1024 open file descriptors
     ,maxConnections:128
+    // Maximum number of templates per user. Unlimited by default.
+    ,maxUserTemplates:1024
+    // Seconds since "last creation" before a detached
+    // or template instance map expires. Or: how long do you want
+    // to be able to navigate the map without a reload ?
+    // Defaults to 7200 (2 hours)
+    ,mapConfigTTL: 7200
     // idle socket timeout, in miliseconds
     ,socket_timeout: 600000 
     ,enable_cors: true
@@ -31,10 +38,10 @@ var config = {
     ,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
     // Templated database username for authorized user
     // Supported labels: 'user_id' (read from redis)
-    ,postgres_auth_user: 'test_cartodb_user_<%= user_id %>'
+    ,postgres_auth_user: 'test_windshaft_cartodb_user_<%= user_id %>'
     // Templated database password for authorized user
     // Supported labels: 'user_id', 'user_password' (both read from redis)
-    ,postgres_auth_pass: 'test_cartodb_user_<%= user_id %>_pass'
+    ,postgres_auth_pass: 'test_windshaft_cartodb_user_<%= user_id %>_pass'
     ,postgres: {
         // Parameters to pass to datasource plugin of mapnik
         // See http://github.com/mapnik/mapnik/wiki/PostGIS
@@ -47,7 +54,13 @@ var config = {
         simplify_geometries: true,
         max_size: 500
     }
-    ,mapnik_version: '2.0.2'
+    ,mapnik_version: ''
+    ,statsd: {
+        host: 'localhost',
+        port: 8125,
+        prefix: 'test.:host.'
+        // support all allowed node-statsd options
+    }
     ,renderer: {
       // Milliseconds since last access before renderer cache item expires
       cache_ttl: 60000,
@@ -85,17 +98,22 @@ var config = {
         domain: 'donot_look_this_up',
         // This port will be used by "make check" for testing purposes
         // It must be available
-        version: 'v1'
+        version: 'v1',
+        // Maximum lenght of SQL query for GET
+        // requests. Longer queries will be sent
+        // using POST. Defaults to 2048
+        max_get_sql_length: 2048
     }
     ,varnish: {
         host: '',
         port: null,
+        secret: 'xxx',
         ttl: 86400
     }
     // If useProfiler is true every response will be served with an
     // X-Tiler-Profile header containing elapsed timing for various 
     // steps taken for producing the response.
-    ,useProfiler:false
+    ,useProfiler:true
 };
 
 module.exports = config;

lib/cartodb/cache_validator.js

@@ -2,16 +2,21 @@ var _             = require('underscore'),
     Varnish       = require('node-varnish'),
     varnish_queue = null;
 
-function init(host, port) {
-    varnish_queue = new Varnish.VarnishQueue(host, port);
+function init(host, port, secret) {
+    varnish_queue = new Varnish.VarnishQueue(host, port, secret);
+    varnish_queue.on('error', function(e) {
+        console.log("[CACHE VALIDATOR ERROR] " + e);
+    });
 }
 
 function invalidate_db(dbname, table) {
+    var cmd = 'purge obj.http.X-Cache-Channel ~ "^' + dbname +
+              ':(.*'+ table +'.*)|(table)$"';
     try{
-        varnish_queue.run_cmd('purge obj.http.X-Cache-Channel ~ "^' + dbname + ':(.*'+ table +'.*)|(table)$"');
-        console.log('[SUCCESS FLUSHING CACHE]');
+        varnish_queue.run_cmd(cmd, false);
     } catch (e) {
-        console.log("[ERROR FLUSHING CACHE] Is enable_cache set to true? Failed for: " + 'purge obj.http.X-Cache-Channel ~ "^' + dbname + ':(.*'+ table +'.*)|(table)$"');
+        console.log("[CACHE VALIDATOR ERROR] could not queue command " +
+                    cmd + " -- " + e);
     }
 }
 

lib/cartodb/cartodb_windshaft.js

@@ -7,14 +7,25 @@ var _ = require('underscore')
     , cartoData  = require('cartodb-redis')(global.environment.redis)
     , SignedMaps = require('./signed_maps.js')
     , TemplateMaps = require('./template_maps.js')
-    , Cache = require('./cache_validator');
+    , Cache = require('./cache_validator')
+    , os = require('os')
+;
 
 var CartodbWindshaft = function(serverOptions) {
    var debug = global.environment.debug;
 
+    // Perform keyword substitution in statsd
+    // See https://github.com/CartoDB/Windshaft-cartodb/issues/153
+    if ( global.environment.statsd ) {
+      if ( global.environment.statsd.prefix ) {
+        var host_token = os.hostname().split('.').reverse().join('.');
+        global.environment.statsd.prefix = global.environment.statsd.prefix.replace(/:host/, host_token);
+      }
+    }
+
     if(serverOptions.cache_enabled) {
         console.log("cache invalidation enabled, varnish on ", serverOptions.varnish_host, ' ', serverOptions.varnish_port);
-        Cache.init(serverOptions.varnish_host, serverOptions.varnish_port);
+        Cache.init(serverOptions.varnish_host, serverOptions.varnish_port, serverOptions.varnish_secret);
         serverOptions.afterStateChange = function(req, data, callback) {
             Cache.invalidate_db(req.params.dbname, req.params.table);
             callback(null, data);
@@ -30,7 +41,10 @@ var CartodbWindshaft = function(serverOptions) {
     }
 
     serverOptions.signedMaps = new SignedMaps(redisPool);
-    var templateMaps = new TemplateMaps(redisPool, serverOptions.signedMaps);
+    var templateMapsOpts = {
+      max_user_templates: global.environment.maxUserTemplates
+    }
+    var templateMaps = new TemplateMaps(redisPool, serverOptions.signedMaps, templateMapsOpts);
 
     // boot
     var ws = new Windshaft.Server(serverOptions);
@@ -43,20 +57,40 @@ var CartodbWindshaft = function(serverOptions) {
       return version;
     }
 
-    // Override sendError to drop added cache headers (if any)
-    // See http://github.com/CartoDB/Windshaft-cartodb/issues/107
-    var ws_sendError = ws.sendError;
-    ws.sendError = function(res) {
-      // NOTE: the "res" object will have no _headers when
-      //       faked by Windshaft, see
-      //       http://github.com/CartoDB/Windshaft-cartodb/issues/109
-      //
-      if ( res._headers ) {
-        delete res._headers['cache-control'];
-        delete res._headers['last-modified'];
-        delete res._headers['x-cache-channel'];
-      } 
-      ws_sendError.apply(this, arguments);
+    var ws_sendResponse = ws.sendResponse;
+    ws.sendResponse = function(res, args) {
+      var that = this;
+      var thatArgs = arguments;
+      var statusCode;
+      if ( args.length > 2 ) statusCode = args[2];
+      else {
+        statusCode = args[1] || 200;
+      }
+      var req = res.req;
+      Step (
+        function addCacheChannel() {
+          if ( ! req ) {
+            // having no associated request can happen when
+            // using fake response objects for testing layergroup
+            // creation
+            return false;
+          }
+          if ( ! req.params ) {
+            // service requests (/version, /) 
+            // have no need for an X-Cache-Channel
+            return false;
+          }
+          if ( statusCode != 200 ) {
+            // We do not want to cache
+            // unsuccessful responses
+            return false;
+          }
+          serverOptions.addCacheChannel(that, req, this);
+        },
+        function sendResponse(err, added) {
+          ws_sendResponse.apply(that, thatArgs);
+        }
+      );
     };
 
     /**
@@ -71,9 +105,9 @@ var CartodbWindshaft = function(serverOptions) {
             function(err, data){
                 if (err){
                     ws.sendError(res, {error: err.message}, 500, 'GET INFOWINDOW', err);
-                    //res.send({error: err.message}, 500);
+                    //ws.sendResponse(res, [{error: err.message}, 500]);
                 } else {
-                    res.send({infowindow: data}, 200);
+                    ws.sendResponse(res, [{infowindow: data}, 200]);
                 }
             }
         );
@@ -92,9 +126,9 @@ var CartodbWindshaft = function(serverOptions) {
             function(err, data){
                 if (err){
                     ws.sendError(res, {error: err.message}, 500, 'GET MAP_METADATA', err);
-                    //res.send(err.message, 500);
+                    //ws.sendResponse(res, [err.message, 500]);
                 } else {
-                    res.send({map_metadata: data}, 200);
+                    ws.sendResponse(res, [{map_metadata: data}, 200]);
                 }
             }
         );
@@ -113,9 +147,9 @@ var CartodbWindshaft = function(serverOptions) {
             function sendResponse(err, data){
                 if (err){
                     ws.sendError(res, {error: err.message}, 500, 'DELETE CACHE', err);
-                    //res.send(500);
+                    //ws.sendResponse(res, [500]);
                 } else {
-                    res.send({status: 'ok'}, 200);
+                    ws.sendResponse(res, [{status: 'ok'}, 200]);
                 }
             }
         );
@@ -162,6 +196,10 @@ var CartodbWindshaft = function(serverOptions) {
           return { template_id: cdbuser + '@' + tpl_id };
         },
         function finish(err, response){
+            if ( req.profiler ) {
+              var report = req.profiler.toString();
+              res.header('X-Tiler-Profiler', report);
+            }
             if (err){
                 response = { error: ''+err };
                 var statusCode = 400;
@@ -170,7 +208,7 @@ var CartodbWindshaft = function(serverOptions) {
                 }
                 ws.sendError(res, response, statusCode, 'POST TEMPLATE', err);
             } else {
-              res.send(response, 200);
+              ws.sendResponse(res, [response, 200]);
             }
         }
       );
@@ -215,6 +253,10 @@ var CartodbWindshaft = function(serverOptions) {
           return { template_id: cdbuser + '@' + tpl_id };
         },
         function finish(err, response){
+            if ( req.profiler ) {
+              var report = req.profiler.toString();
+              res.header('X-Tiler-Profiler', report);
+            }
             if (err){
                 var statusCode = 400;
                 response = { error: ''+err };
@@ -223,7 +265,7 @@ var CartodbWindshaft = function(serverOptions) {
                 }
                 ws.sendError(res, response, statusCode, 'PUT TEMPLATE', err);
             } else {
-              res.send(response, 200);
+              ws.sendResponse(res, [response, 200]);
             }
         }
       );
@@ -281,7 +323,7 @@ var CartodbWindshaft = function(serverOptions) {
                 }
                 ws.sendError(res, response, statusCode, 'GET TEMPLATE', err);
             } else {
-              res.send(response, 200);
+              ws.sendResponse(res, [response, 200]);
             }
         }
       );
@@ -331,7 +373,7 @@ var CartodbWindshaft = function(serverOptions) {
                 }
                 ws.sendError(res, response, statusCode, 'DELETE TEMPLATE', err);
             } else {
-              res.send('', 204);
+              ws.sendResponse(res, ['', 204]);
             }
         }
       );
@@ -371,7 +413,7 @@ var CartodbWindshaft = function(serverOptions) {
                 }
                 ws.sendError(res, response, statusCode, 'GET TEMPLATE LIST', err);
             } else {
-              res.send(response, statusCode);
+              ws.sendResponse(res, [response, statusCode]);
             }
         }
       );
@@ -400,6 +442,7 @@ var CartodbWindshaft = function(serverOptions) {
     // Instantiate a template
     function instanciateTemplate(req, res, template_params, callback) {
       ws.doCORS(res);
+      if ( req.profiler ) req.profiler.done('cors');
       var that = this;
       var response = {};
       var template;
@@ -420,6 +463,7 @@ var CartodbWindshaft = function(serverOptions) {
           templateMaps.getTemplate(cdbuser, tpl_id, this);
         },
         function checkAuthorized(err, data) {
+          if ( req.profiler ) req.profiler.done('getTemplate');
           if ( err ) throw err;
           if ( ! data ) {
             err = new Error("Template '" + tpl_id + "' of user '" + cdbuser + "' not found");
@@ -446,24 +490,32 @@ var CartodbWindshaft = function(serverOptions) {
             throw new Error('template POST data must be of type application/json, it is instead ');
           }*/
           //var template_params = req.body;
+          if ( req.profiler ) req.profiler.done('authorizedByCert');
           return templateMaps.instance(template, template_params);
         },
         function prepareParams(err, instance){
+          if ( req.profiler ) req.profiler.done('TemplateMaps_instance');
           if ( err ) throw err;
           layergroup = instance;
-          fakereq = { query: {}, params: {}, headers: _.clone(req.headers) };
+          fakereq = { query: {}, params: {}, headers: _.clone(req.headers),
+            profiler: req.profiler
+          };
           ws.setDBParams(cdbuser, fakereq.params, this);
         },
         function setApiKey(err){
+          if ( req.profiler ) req.profiler.done('setDBParams');
           if ( err ) throw err;
           cartoData.getUserMapKey(cdbuser, this);
         },
         function createLayergroup(err, val) {
+          if ( req.profiler ) req.profiler.done('getUserMapKey');
           if ( err ) throw err;
           fakereq.params.api_key = val;
           ws.createLayergroup(layergroup, fakereq, this);
         },
         function signLayergroup(err, resp) {
+          // NOTE: createLayergroup uses profiler.start()/end() internally
+          //if ( req.profiler ) req.profiler.done('createLayergroup');
           if ( err ) throw err;
           response = resp;
           var signer = cdbuser;
@@ -481,10 +533,12 @@ var CartodbWindshaft = function(serverOptions) {
           signedMaps.signMap(signer, map_id, crt_id, this);
         },
         function prepareResponse(err) {
+          if ( req.profiler ) req.profiler.done('signMap');
           if ( err ) throw err;
           //console.log("Response from createLayergroup: "); console.dir(response);
           // Add the signature part to the token!
           var tplhash = templateMaps.fingerPrint(template).substring(0,8);
+          if ( req.profiler ) req.profiler.done('fingerPrint');
           response.layergroupid = cdbuser + '@' + tplhash + '@' + response.layergroupid;
           return response;
         },
@@ -492,7 +546,11 @@ var CartodbWindshaft = function(serverOptions) {
       );
     }
 
-    function finish_instanciation(err, response, res) {
+    function finish_instanciation(err, response, res, req) {
+        if ( req.profiler ) {
+          var report = req.profiler.toString();
+          res.header('X-Tiler-Profiler', report);
+        }
         if (err) {
             var statusCode = 400;
             response = { error: ''+err };
@@ -504,11 +562,17 @@ var CartodbWindshaft = function(serverOptions) {
             }
             ws.sendError(res, response, statusCode, 'POST INSTANCE TEMPLATE', err);
         } else {
-          res.send(response, 200);
+          ws.sendResponse(res, [response, 200]);
+        }
+        if ( req.profiler && req.profiler.statsd_client) {
+          req.profiler.sendStats();
         }
     }
 
     ws.post(template_baseurl + '/:template_id', function(req, res) {
+      if ( req.profiler && req.profiler.statsd_client) {
+        req.profiler.start('windshaft-cartodb.instance_template_post');
+      }
       Step(
         function() {
           if ( ! req.headers['content-type'] || req.headers['content-type'].split(';')[0] != 'application/json') {
@@ -516,7 +580,7 @@ var CartodbWindshaft = function(serverOptions) {
           }
           instanciateTemplate(req, res, req.body, this);
         }, function(err, response) {
-          finish_instanciation(err, response, res);
+          finish_instanciation(err, response, res, req);
         }
       );
     });
@@ -526,6 +590,9 @@ var CartodbWindshaft = function(serverOptions) {
      * callback query argument is mandartoy
      */
     ws.get(template_baseurl + '/:template_id/jsonp', function(req, res) {
+      if ( req.profiler && req.profiler.statsd_client) {
+        req.profiler.start('windshaft-cartodb.instance_template_get');
+      }
       Step(
         function() {
           if ( req.query.callback === undefined || req.query.callback.length === 0) {
@@ -541,7 +608,7 @@ var CartodbWindshaft = function(serverOptions) {
           }
           instanciateTemplate(req, res, config, this);
         }, function(err, response) {
-          finish_instanciation(err, response, res);
+          finish_instanciation(err, response, res, req);
         }
       );
     });

lib/cartodb/server_options.js

@@ -47,13 +47,14 @@ module.exports = function(){
           datasource: global.environment.postgres,
           cachedir: global.environment.millstone.cache_basedir,
           mapnik_version: global.environment.mapnik_version || mapnik.versions.mapnik,
-          default_layergroup_ttl: 7200, // seconds (default is 300)
-          gc_prob: 0.01 // default is 0.01 TODO: make configurable via env config
+          default_layergroup_ttl: global.environment.mapConfigTTL || 7200, 
+          gc_prob: 0.01 // @deprecated since Windshaft-1.8.0
         },
         mapnik: {
           metatile: rendererConfig.metatile,
           bufferSize: rendererConfig.bufferSize
         },
+        statsd: global.environment.statsd,
         renderCache: {
           ttl: rendererConfig.cache_ttl
         },
@@ -61,6 +62,7 @@ module.exports = function(){
         enable_cors: global.environment.enable_cors,
         varnish_host: global.environment.varnish.host,
         varnish_port: global.environment.varnish.port,
+        varnish_secret: global.environment.varnish.secret,
         cache_enabled: global.environment.cache_enabled,
         log_format: global.environment.log_format,
         useProfiler: global.environment.useProfiler
@@ -103,16 +105,33 @@ module.exports = function(){
       // call sql api
       //
       // NOTE: using POST to avoid size limits:
-      // Seehttp://github.com/CartoDB/Windshaft-cartodb/issues/111
+      // See http://github.com/CartoDB/Windshaft-cartodb/issues/111
       //
-      // TODO: use "host" header to allow IP based specification
+      // NOTE: uses "host" header to allow IP based specification
       //       of sqlapi address (and avoid a DNS lookup)
       //
-      request.post({
-          url:sqlapi, body:qs, json:true,
-          headers:{host: sqlapihostname}
-        }, function(err, res, body)
-      {
+      // NOTE: allows for keeping up to "maxConnections" concurrent
+      //       sockets opened per SQL-API host.
+      // See http://nodejs.org/api/http.html#http_agent_maxsockets
+      //
+      var maxSockets = global.environment.maxConnections || 128;
+      var maxGetLen = api.max_get_sql_length || 2048;
+      var reqSpec = {
+            url:sqlapi,
+            json:true,
+            headers:{host: sqlapihostname}
+            // http://nodejs.org/api/http.html#http_agent_maxsockets
+            ,pool:{maxSockets:maxSockets}
+            //,timeout:100
+      }
+      if ( sql.length > maxGetLen ) {
+        reqSpec.method = 'POST';
+        reqSpec.body = qs;
+      } else {
+        reqSpec.method = 'GET';
+        reqSpec.qs = qs;
+      }
+      request(reqSpec, function(err, res, body) {
           if (err){
             console.log('ERROR connecting to SQL API on ' + sqlapi + ': ' + err);
             callback(err);
@@ -187,53 +206,101 @@ module.exports = function(){
         return hash.digest('hex');
     }
 
-    me.generateCacheChannel = function(req, callback){
-
-      // use key to call sql api with sql request if present, else
-      // just return dbname and table name base key
-      var dbName     = req.params.dbname;
+    me.generateCacheChannel = function(app, req, callback){
 
+      // Build channelCache key
+      var dbName = req.params.dbname;
       var cacheKey = [ dbName ];
       if ( req.params.token ) cacheKey.push(req.params.token);
       else if ( req.params.sql ) cacheKey.push( me.generateMD5(req.params.sql) );
       cacheKey = cacheKey.join(':');
 
-      if ( me.channelCache.hasOwnProperty(cacheKey) ) {
-        callback(null, me.channelCache[cacheKey]);
-        return;
-      }
-      else if ( req.params.token ) {
-        // cached cache channel for token-based access should be constructed
-        // at cache creation time
-        callback(new Error('missing channel cache for token ' + req.params.token));
-        return;
-      }
+      var that = this;
 
-      if ( ! req.params.sql && ! req.params.token ) {
-        var cacheChannel = me.buildCacheChannel(dbName, [req.params.table]);
-        // not worth caching this
-        callback(null, cacheChannel);
-        return;
-      }
+      Step (
+        function checkCached() {
+          if ( me.channelCache.hasOwnProperty(cacheKey) ) {
+            callback(null, me.channelCache[cacheKey]);
+            return;
+          }
+          return null;
+        },
+        function extractSQL(err) {
+          if ( err ) throw err;
 
-      if ( ! req.params.sql ) {
-        callback(new Error("this request doesn't need an X-Cache-Channel generated"));
-        return;
-      }
+          if ( req.params.token ) {
+            // TODO: cached cache channel for token-based access should
+            //       be constructed at renderer cache creation time
+            // See http://github.com/CartoDB/Windshaft-cartodb/issues/152
+            if ( ! app.mapStore ) {
+              throw new Error('missing channel cache for token ' + req.params.token);
+              return;
+            }
+            var next = this;
+            var mapStore = app.mapStore;
+            Step(
+              function loadFromStore() {
+                mapStore.load(req.params.token, this);
+              },
+              function getSQL(err, mapConfig) {
+                if (req.profiler) req.profiler.done('mapStore_load');
+                if ( err ) throw err;
+                var sql = [];
+                _.each(mapConfig.obj().layers, function(lyr) {
+                  sql.push(lyr.options.sql);
+                });
+                sql = sql.join(';');
+                return sql;
+              },
+              function finish(err, sql) {
+                next(err, sql);
+              }
+            );
+            return;
+          }
 
-      var dbName     = req.params.dbname;
-      var username   = this.userByReq(req);
+          if ( ! req.params.sql ) {
+            return null; // no sql
+          }
 
-      // strip out windshaft/mapnik inserted sql if present
-      var sql = req.params.sql.match(/^\((.*)\)\sas\scdbq$/);
-      sql = (sql != null) ? sql[1] : req.params.sql;
+          // We have sql, and no token...
 
-      me.affectedTables(username, req.params.map_key, sql, function(err, tableNames) {
-          if ( err ) { callback(err); return; }
+          // strip out windshaft/mapnik inserted sql if present
+          var sql = req.params.sql.match(/^\((.*)\)\sas\scdbq$/);
+          sql = (sql != null) ? sql[1] : req.params.sql;
+
+          return sql;
+        },
+        function findAffectedTables(err, sql) {
+          if ( err ) throw err;
+          if ( ! sql ) {
+            if ( ! req.params.table ) {
+              throw new Error("this request doesn't need an X-Cache-Channel generated");
+            }
+            return [req.params.table];
+          }
+          var username   = that.userByReq(req);
+          me.affectedTables(username, req.params.map_key, sql, this);
+        },
+        function buildCacheChannel(err, tableNames) {
+          if ( err ) throw err;
+          if (req.profiler && ! req.params.table ) {
+            req.profiler.done('affectedTables');
+          }
+
+          var dbName     = req.params.dbname;
           var cacheChannel = me.buildCacheChannel(dbName,tableNames);
-          me.channelCache[cacheKey] = cacheChannel; // store for caching
-          callback(null, cacheChannel);
-      });
+          // store for caching from me.generateCacheChannel
+          // (not worth when table was specified in params)
+          if ( ! req.params.table ) {
+            me.channelCache[cacheKey] = cacheChannel;
+          }
+          return cacheChannel;
+        },
+        function finish(err, cacheChannel) {
+          callback(err, cacheChannel);
+        }
+      );
     };
 
     // Set the cache chanel info to invalidate the cache on the frontend server
@@ -245,9 +312,10 @@ module.exports = function(){
     // @param cb function(err, channel) will be called when ready.
     //           the channel parameter will be null if nothing was added
     //
-    me.addCacheChannel = function(req, cb) {
+    me.addCacheChannel = function(app, req, cb) {
         // skip non-GET requests, or requests for which there's no response
         if ( req.method != 'GET' || ! req.res ) { cb(null, null); return; }
+        if (req.profiler) req.profiler.start('addCacheChannel');
         var res = req.res;
         var cache_policy = req.query.cache_policy;
         if ( req.params.token ) cache_policy = 'persist';
@@ -269,7 +337,9 @@ module.exports = function(){
         }
         res.header('Last-Modified', lastUpdated.toUTCString());
 
-        me.generateCacheChannel(req, function(err, channel){
+        me.generateCacheChannel(app, req, function(err, channel){
+            if (req.profiler) req.profiler.done('generateCacheChannel');
+            if (req.profiler) req.profiler.end();
             if ( ! err ) {
               res.header('X-Cache-Channel', channel);
               cb(null, channel);
@@ -325,7 +395,8 @@ module.exports = function(){
 
             if ( err ) { done(err); return; }
             var cacheChannel = me.buildCacheChannel(dbName,tableNames);
-            me.channelCache[cacheKey] = cacheChannel; // store for caching
+            // store for caching from me.afterLayergroupCreate
+            me.channelCache[cacheKey] = cacheChannel;
             // find last updated
             me.findLastUpdated(usr, key, tableNames, function(err, lastUpdated) {
               if (req.profiler) req.profiler.done('findLastUpdated');
@@ -475,9 +546,20 @@ module.exports = function(){
     //
     // @param req express request object
     // @param callback function(err, authorized) 
+    //        NOTE: authorized is expected to be 0 or 1 (integer)
     //                 
     me.authorizedByAPIKey = function(req, callback)
     {
+        var givenKey = req.query.api_key || req.query.map_key;
+        if ( ! givenKey && req.body ) {
+          // check also in request body
+          givenKey = req.body.api_key || req.body.map_key;
+        }
+        if ( ! givenKey ) {
+          callback(null, 0); // no api key, no authorization...
+          return;
+        }
+        //console.log("given ApiKey: " + givenKey);
         var user = me.userByReq(req);
         Step(
           function (){
@@ -485,16 +567,7 @@ module.exports = function(){
           },
           function checkApiKey(err, val){
               if (err) throw err;
-
-              var valid = 0;
-              if ( val ) {
-                if ( val == req.query.map_key ) valid = 1;
-                else if ( val == req.query.api_key ) valid = 1;
-                // check also in request body
-                else if ( req.body && req.body.map_key && val == req.body.map_key ) valid = 1;
-                else if ( req.body && req.body.api_key && val == req.body.api_key ) valid = 1;
-              }
-              return valid;
+              return ( val && givenKey == val ) ? 1 : 0;
           },
           function finish(err, authorized) {
               callback(err, authorized);
@@ -517,6 +590,7 @@ module.exports = function(){
                 that.authorizedByAPIKey(req, this);
             },
             function checkApiKey(err, authorized){
+                if (req.profiler) req.profiler.done('authorizedByAPIKey');
                 if (err) throw err;
 
                 // if not authorized by api_key, continue 
@@ -536,15 +610,27 @@ module.exports = function(){
             },
             function checkSignAuthorized(err, signed_by){
                 if (err) throw err;
+                if (req.profiler) {
+                  if ( req.params._authorizedByApiKey ) {
+                    req.profiler.done('setDBAuth');
+                  } else {
+                    req.profiler.done('authorizedBySigner');
+                  }
+                }
 
                 if ( ! signed_by ) {
-                  // request not authorized by signer, continue
-                  // to check map privacy
-                  return null;
+                  // request not authorized by signer,
+                  // continue to check table privacy,
+                  // if table was given
+                  if ( req.params.table ) return null;
+                  // otherwise return no authorization
+                  callback(err, null);
+                  return;
                 }
 
                 // Authorized by "signed_by" !
                 that.setDBAuth(signed_by, req.params, function(err) {
+                  if (req.profiler) req.profiler.done('setDBAuth');
                   callback(err, true); // authorized (or error)
                 });
             },
@@ -555,9 +641,11 @@ module.exports = function(){
             },
             function getPrivacy(err, dbname){
                 if (err) throw err;
+                if (req.profiler) req.profiler.done('tablePrivacy_getUserDBName');
                 cartoData.getTablePrivacy(dbname, req.params.table, this);
             },
             function(err, privacy){
+                if (req.profiler) req.profiler.done('getTablePrivacy');
                 callback(err, privacy);
             }
         );
@@ -652,6 +740,7 @@ module.exports = function(){
             function getGeometryType(err){
                 if (req.profiler) req.profiler.done('setDBConn');
                 if (err) throw err;
+                if ( ! req.params.table ) return null;
                 cartoData.getTableGeometryType(req.params.dbname, req.params.table, this);
             },
             function finishSetup(err, data){
@@ -670,10 +759,7 @@ module.exports = function(){
                   dbport: global.environment.postgres.port
                 });
 
-                that.addCacheChannel(req, function(err) {
-                  if (req.profiler) req.profiler.done('addCacheChannel');
-                  callback(err, req);
-                });
+                callback(null, req);
             }
         );
     };
@@ -763,4 +849,4 @@ module.exports = function(){
     };
 
     return me;
-}();
+};

lib/cartodb/signed_maps.js

@@ -2,7 +2,7 @@ var crypto    = require('crypto');
 var Step      = require('step');
 var _         = require('underscore');
 
-var debug = global.environment.debug;
+var debug = global.environment ? global.environment.debug : undefined;
 
 // Class handling map signatures and user certificates
 //
@@ -162,6 +162,8 @@ o.authorizedByCert = function(cert, auth) {
 //
 o.isAuthorized = function(signer, map_id, auth, callback) {
   var that = this;
+  var redisClient;
+  var db = that.db_signatures;
   var authorized = false;
   var certificate_id_list;
   var missing_certificates = [];
@@ -169,9 +171,15 @@ o.isAuthorized = function(signer, map_id, auth, callback) {
     console.log("Check auth from signer '" + signer + "' on map '" + map_id + "' with auth '" + auth + "'");
   }
   Step(
-    function getMapSignatures() {
+    function getRedisClient() {
+      that.redis_pool.acquire(db, this);
+    },
+    function getMapSignatures(err, client) {
+      if ( err ) throw err;
+      redisClient = client;
       var map_sig_key = _.template(that.key_map_sig, {signer:signer, map_id:map_id});
-      that._redisCmd('SMEMBERS', [ map_sig_key ], this);
+      redisClient.SMEMBERS(map_sig_key, this);
+      //that._redisCmd('SMEMBERS', [ map_sig_key ], this);
     },
     function getCertificates(err, crt_lst) {
       if ( err ) throw err;
@@ -186,7 +194,8 @@ o.isAuthorized = function(signer, map_id, auth, callback) {
         return crt_lst;
       }
       var map_crt_key = _.template(that.key_map_crt, {signer:signer});
-      that._redisCmd('HMGET', [ map_crt_key ].concat(crt_lst), this);
+      //that._redisCmd('HMGET', [ map_crt_key ].concat(crt_lst), this);
+      redisClient.HMGET(map_crt_key, crt_lst, this);
     },
     function checkCertificates(err, certs) {
       if ( err ) throw err;
@@ -223,6 +232,7 @@ o.isAuthorized = function(signer, map_id, auth, callback) {
           + " missing certificates: "
           + missing_certificates + " (TODO: give cleanup instructions)");
       }
+      if ( redisClient ) that.redis_pool.release(db, redisClient);
       callback(err, authorized);
     }
   );

lib/cartodb/template_maps.js

@@ -16,10 +16,15 @@ var user_template_locks = {};
 //
 // @param signed_maps an instance of a "signed_maps" class,
 //        See signed_maps.js
+//
+// @param opts TemplateMap options. Supported elements:
+//    'max_user_templates' limit on the number of per-user
+//
 // 
-function TemplateMaps(redis_pool, signed_maps) {
+function TemplateMaps(redis_pool, signed_maps, opts) {
   this.redis_pool = redis_pool;
   this.signed_maps = signed_maps;
+  this.opts = opts || {};
 
   // Database containing templates
   // TODO: allow configuring ?
@@ -49,6 +54,10 @@ var o = TemplateMaps.prototype;
 
 //--------------- PRIVATE METHODS --------------------------------
 
+o._userTemplateLimit = function() {
+  return this.opts['max_user_templates'] || 0;
+};
+
 o._acquireRedis = function(callback) {
   this.redis_pool.acquire(this.db_signatures, callback);
 };
@@ -189,6 +198,7 @@ o.addTemplate = function(owner, template, callback) {
 
   // Procedure:
   //
+  // - Check against limit
   // 0. Obtain a lock for user+template_name, fail if impossible
   // 1. Check no other template exists with the same name
   // 2. Install certificate extracted from template, extending
@@ -202,9 +212,18 @@ o.addTemplate = function(owner, template, callback) {
   var usr_tpl_key = _.template(this.key_usr_tpl, {owner:owner});
   var gotLock = false;
   var that = this;
+  var limit = that._userTemplateLimit();
   Step(
+    function checkLimit() {
+      if ( ! limit ) return 0;
+      that._redisCmd('HLEN', [ usr_tpl_key ], this);
+    },
     // try to obtain a lock
-    function obtainLock() {
+    function obtainLock(err, len) {
+      if ( err ) throw err;
+      if ( limit && len >= limit ) {
+        throw new Error("User '" + owner + "' reached limit on number of templates (" + len + "/" + limit + ")");
+      }
       that._obtainTemplateLock(owner, tplname, this);
     },
     function getExistingTemplate(err, locked) {
@@ -238,10 +257,7 @@ o.addTemplate = function(owner, template, callback) {
         // TODO: how to recover this ?!
       }
 
-      if ( ! gotLock ) {
-        if ( err ) throw err;
-        return null;
-      }
+      if ( err && ! gotLock ) throw err;
 
       // release the lock
       var next = this;

npm-shrinkwrap.json

@@ -1,19 +1,20 @@
 {
   "name": "windshaft-cartodb",
-  "version": "1.7.1",
+  "version": "1.8.2",
   "dependencies": {
     "node-varnish": {
-      "version": "0.1.1"
+      "version": "0.2.0",
+      "from": "http://github.com/Vizzuality/node-varnish/tarball/v0.2.0"
     },
     "underscore": {
       "version": "1.3.3"
     },
     "windshaft": {
-      "version": "0.17.2",
-      "from": "http://github.com/CartoDB/Windshaft/tarball/0.17.2",
+      "version": "0.19.0",
+      "from": "http://github.com/CartoDB/Windshaft/tarball/0.19.0-rc1",
       "dependencies": {
         "grainstore": {
-          "version": "0.17.1",
+          "version": "0.18.0",
           "dependencies": {
             "mapnik-reference": {
               "version": "5.0.7"
@@ -144,7 +145,7 @@
                               "version": "0.1.25",
                               "dependencies": {
                                 "graceful-fs": {
-                                  "version": "2.0.1"
+                                  "version": "2.0.2"
                                 }
                               }
                             }
@@ -166,7 +167,7 @@
                               "version": "0.1.25",
                               "dependencies": {
                                 "graceful-fs": {
-                                  "version": "2.0.1"
+                                  "version": "2.0.2"
                                 },
                                 "inherits": {
                                   "version": "2.0.1"
@@ -193,7 +194,7 @@
                               "version": "1.0.25-1",
                               "dependencies": {
                                 "string_decoder": {
-                                  "version": "0.10.25"
+                                  "version": "0.10.25-1"
                                 }
                               }
                             },
@@ -375,6 +376,9 @@
         },
         "torque.js": {
           "version": "2.2.00"
+        },
+        "node-statsd": {
+          "version": "0.0.7"
         }
       }
     },
@@ -409,15 +413,31 @@
     "lzma": {
       "version": "1.2.3"
     },
+    "log4js": {
+      "version": "0.6.10",
+      "dependencies": {
+        "async": {
+          "version": "0.1.15"
+        },
+        "readable-stream": {
+          "version": "1.0.25-1",
+          "dependencies": {
+            "string_decoder": {
+              "version": "0.10.25-1"
+            }
+          }
+        }
+      }
+    },
     "redis": {
       "version": "0.8.6"
     },
-    "semver": {
-      "version": "1.1.4"
-    },
     "strftime": {
       "version": "0.6.2"
     },
+    "semver": {
+      "version": "1.1.4"
+    },
     "mocha": {
       "version": "1.14.0",
       "dependencies": {

package.json

@@ -1,7 +1,7 @@
 {
     "private": true,
     "name": "windshaft-cartodb",
-    "version": "1.7.1",
+    "version": "1.8.2",
     "description": "A map tile server for CartoDB",
     "keywords": [
       "cartodb"
@@ -22,15 +22,16 @@
       "Sandro Santilli <strk@vizzuality.com>"
     ],
     "dependencies": {
-        "node-varnish": "0.1.1",
+        "node-varnish": "http://github.com/Vizzuality/node-varnish/tarball/v0.2.0",
         "underscore"  : "~1.3.3",
-        "windshaft"  : "~0.17.2",
+        "windshaft"  : "http://github.com/CartoDB/Windshaft/tarball/0.19.0",
         "step": "0.0.x",
         "request": "2.9.202",
         "cartodb-redis": "~0.3.0",
         "redis-mpool": "~0.0.2",
         "mapnik": "~0.7.22",
-        "lzma": "~1.2.3"
+        "lzma": "~1.2.3",
+        "log4js": "~0.6.10"
     },
     "devDependencies": {
         "mocha": "1.14.0",

test/acceptance/multilayer.js

@@ -15,7 +15,8 @@ require(__dirname + '/../support/test_helper');
 var windshaft_fixtures = __dirname + '/../../node_modules/windshaft/test/fixtures';
 
 var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft');
-var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
+var ServerOptions = require(__dirname + '/../../lib/cartodb/server_options');
+serverOptions = ServerOptions();
 var server = new CartodbWindshaft(serverOptions);
 server.setMaxListeners(0);
 
@@ -34,6 +35,10 @@ suite('multilayer', function() {
     var expected_last_updated_epoch = 1234567890123; // this is hard-coded into SQLAPIEmu
     var expected_last_updated = new Date(expected_last_updated_epoch).toISOString();
 
+    var test_user = _.template(global.environment.postgres_auth_user, {user_id:1});
+    var test_pubuser = global.environment.postgres.user;
+    var test_database = test_user + '_db';
+
     suiteSetup(function(done){
       sqlapi_server = new SQLAPIEmu(global.environment.sqlapi.port, done);
     });
@@ -107,7 +112,7 @@ suite('multilayer', function() {
               // Check X-Cache-Channel
               cc = res.headers['x-cache-channel'];
               assert.ok(cc); 
-              var dbname = 'test_cartodb_user_1_db'
+              var dbname = test_database;
               assert.equal(cc.substring(0, dbname.length), dbname);
               var jsonquery = cc.substring(dbname.length+1);
               var sentquery = JSON.parse(jsonquery);
@@ -237,7 +242,7 @@ suite('multilayer', function() {
               // Check X-Cache-Channel
               var cc = res.headers['x-cache-channel'];
               assert.ok(cc); 
-              var dbname = 'test_cartodb_user_1_db'
+              var dbname = test_database;
               assert.equal(cc.substring(0, dbname.length), dbname);
               var jsonquery = cc.substring(dbname.length+1);
               var sentquery = JSON.parse(jsonquery);
@@ -270,7 +275,7 @@ suite('multilayer', function() {
               // Check X-Cache-Channel
               var cc = res.headers['x-cache-channel'];
               assert.ok(cc); 
-              var dbname = 'test_cartodb_user_1_db'
+              var dbname = test_database;
               assert.equal(cc.substring(0, dbname.length), dbname);
               var jsonquery = cc.substring(dbname.length+1);
               var sentquery = JSON.parse(jsonquery);
@@ -562,7 +567,7 @@ suite('multilayer', function() {
               // Check X-Cache-Channel
               var cc = res.headers['x-cache-channel'];
               assert.ok(cc); 
-              var dbname = 'test_cartodb_user_1_db'
+              var dbname = test_database;
               assert.equal(cc.substring(0, dbname.length), dbname);
               next(err);
           });
@@ -663,6 +668,123 @@ suite('multilayer', function() {
       );
     });
 
+    // See https://github.com/CartoDB/Windshaft-cartodb/issues/152
+    test("x-cache-channel still works for GETs after tiler restart", function(done) {
+
+      var layergroup =  {
+        version: '1.0.0',
+        layers: [
+           { options: {
+               sql: 'select * from test_table where cartodb_id=1',
+               cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', 
+               cartocss_version: '2.1.0',
+               interactivity: 'cartodb_id'
+             } }
+        ]
+      };
+
+      var expected_token; // = "b4ed64d93a411a59f330ab3d798e4009";
+      Step(
+        function do_post()
+        {
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup?map_key=1234',
+              method: 'POST',
+              headers: {host: 'localhost', 'Content-Type': 'application/json' },
+              data: JSON.stringify(layergroup)
+          }, {}, function(res, err) { next(err, res); });
+        },
+        function check_post(err, res) {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 200, res.body);
+          var parsedBody = JSON.parse(res.body);
+          var expectedBody = { layergroupid: expected_token };
+          // check last modified
+          var qTables = JSON.stringify({
+            'q': 'SELECT CDB_QueryTables($windshaft$'
+                + layergroup.layers[0].options.sql
+                + '$windshaft$)'
+          });
+          assert.equal(parsedBody.last_updated, expected_last_updated);
+          if ( expected_token ) {
+            assert.equal(parsedBody.layergroupid, expected_token + ':' + expected_last_updated_epoch);
+          }
+          else expected_token = parsedBody.layergroupid.split(':')[0];
+          return null;
+        },
+        function do_get0(err)
+        {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup/' + expected_token + ':cb0/0/0/0.png?map_key=1234',
+              method: 'GET',
+              headers: {host: 'localhost' },
+              encoding: 'binary'
+          }, {}, function(res, err) { next(err, res); });
+        },
+        function do_check0(err, res) {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 200, res.body);
+          assert.equal(res.headers['content-type'], "image/png");
+
+          // Check X-Cache-Channel
+          var cc = res.headers['x-cache-channel'];
+          assert.ok(cc, "Missing X-Cache-Channel"); 
+          var dbname = test_database;
+          assert.equal(cc.substring(0, dbname.length), dbname);
+          return null;
+        },
+        function do_restart_server(err, res) {
+          if ( err ) throw err;
+          // hack simulating restart...
+          serverOptions = ServerOptions();
+          server = new CartodbWindshaft(serverOptions);
+          return null;
+        },
+        function do_get1(err)
+        {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup/' + expected_token + ':cb0/0/0/0.png?map_key=1234',
+              method: 'GET',
+              headers: {host: 'localhost' },
+              encoding: 'binary'
+          }, {}, function(res, err) { next(err, res); });
+        },
+        function do_check1(err, res) {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 200, res.body);
+          assert.equal(res.headers['content-type'], "image/png");
+
+          // Check X-Cache-Channel
+          var cc = res.headers['x-cache-channel'];
+          assert.ok(cc, "Missing X-Cache-Channel on restart");
+          var dbname = test_database;
+          assert.equal(cc.substring(0, dbname.length), dbname);
+          return null;
+        },
+        function finish(err) {
+          var errors = [];
+          if ( err ) {
+            errors.push(err.message);
+            console.log("Error: " + err);
+          }
+          redis_client.keys("map_cfg|" + expected_token, function(err, matches) {
+              if ( err ) errors.push(err.message);
+              assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches);
+              redis_client.del(matches, function(err) {
+                if ( err ) errors.push(err.message);
+                if ( errors.length ) done(new Error(errors.join(',')));
+                else done(null);
+              });
+          });
+        }
+      );
+    });
+
     // https://github.com/cartodb/Windshaft-cartodb/issues/81
     test("invalid text-name in CartoCSS", function(done) {
 
@@ -941,10 +1063,12 @@ suite('multilayer', function() {
           var parsedBody = JSON.parse(res.body);
           var token_components = parsedBody.layergroupid.split(':');
           expected_token = token_components[0];
+          var last_request = sqlapi_server.getLastRequest();
+          assert.equal(last_request.method, 'POST');
           return null;
         },
         function cleanup(err) {
-          if ( err ) errors.push(err.message);
+          if ( err ) errors.push('' + err);
           if ( ! expected_token ) return null;
           var next = this;
           redis_client.keys("map_cfg|" + expected_token, function(err, matches) {

test/acceptance/server.js

@@ -12,7 +12,7 @@ var SQLAPIEmu   = require(__dirname + '/../support/SQLAPIEmu.js');
 var helper = require(__dirname + '/../support/test_helper');
 
 var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft');
-var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
+var serverOptions = require(__dirname + '/../../lib/cartodb/server_options')();
 var server = new CartodbWindshaft(serverOptions);
 server.setMaxListeners(0);
 
@@ -22,7 +22,7 @@ suite('server', function() {
     var sqlapi_server;
 
     var mapnik_version = global.environment.mapnik_version || mapnik.versions.mapnik;
-    var test_database = 'test_cartodb_user_1_db';
+    var test_database = _.template(global.environment.postgres_auth_user, {user_id:1}) + '_db';
     var default_style;
     if ( semver.satisfies(mapnik_version, '<2.1.0') ) {
       // 2.0.0 default
@@ -53,12 +53,25 @@ suite('server', function() {
     
     // TODO: I guess this should be a 404 instead...
     test("get call to server returns 200", function(done){
-        assert.response(server, {
-            url: '/',
-            method: 'GET'
-        },{
-            status: 200
-        }, function() { done(); });
+      Step(
+        function doGet() {
+          var next = this;
+          assert.response(server, {
+              url: '/',
+              method: 'GET'
+          },{}, function(res, err) { next(err,res); });
+        },
+        function doCheck(err, res) {
+          if ( err ) throw err;
+          assert.ok(res.statusCode, 200);
+          var cc = res.headers['x-cache-channel'];
+          assert.ok(!cc);
+          return null;
+        },
+        function finish(err) {
+          done(err);
+        }
+      );
     });
 
     /////////////////////////////////////////////////////////////////////////////////
@@ -175,7 +188,8 @@ suite('server', function() {
         },
         function setupRedisBase(err, matches) {
           if ( err ) throw err;
-          assert.equal(matches.length, 0);
+          assert.equal(matches.length, 0,
+            'Unexpected redis keys at test start: ' + matches.join("\n"));
           redis_client.set(base_key, 
             JSON.stringify({ style: style }),
           this);
@@ -1111,8 +1125,8 @@ suite('server', function() {
             var ct = res.headers['content-type'];
             assert.equal(ct, 'image/png');
             var cc = res.headers['x-cache-channel'];
-            assert(cc);
-            var dbname = 'test_cartodb_user_1_db'
+            assert(cc, 'Missing X-Cache-Channel');
+            var dbname = test_database;
             assert.equal(cc.substring(0, dbname.length), dbname);
             var jsonquery = cc.substring(dbname.length+1);
             var sentquery = JSON.parse(jsonquery);
@@ -1148,6 +1162,7 @@ suite('server', function() {
             assert.ok(last_request);
             var host = last_request.headers['host'];
             assert.ok(host);
+            assert.equal(last_request.method, 'GET');
             assert.equal(host, 'localhost.donot_look_this_up');
             return null;
           },

test/acceptance/templates.js

@@ -15,7 +15,7 @@ require(__dirname + '/../support/test_helper');
 var windshaft_fixtures = __dirname + '/../../node_modules/windshaft/test/fixtures';
 
 var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft');
-var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
+var serverOptions = require(__dirname + '/../../lib/cartodb/server_options')();
 var server = new CartodbWindshaft(serverOptions);
 server.setMaxListeners(0);
 

test/unit/cartodb/req2params.test.js

@@ -7,7 +7,12 @@ var assert = require('assert')
 suite('req2params', function() {
 
     // configure redis pool instance to use in tests
-    var opts = require('../../../lib/cartodb/server_options');
+    var opts = require('../../../lib/cartodb/server_options')();
+
+    var test_user = _.template(global.environment.postgres_auth_user, {user_id:1});
+    var test_pubuser = global.environment.postgres.user;
+    var test_database = test_user + '_db';
+
     
     test('can be found in server_options', function(){
       assert.ok(_.isFunction(opts.req2params));
@@ -20,8 +25,8 @@ suite('req2params', function() {
           assert.ok(!req.query.hasOwnProperty('dbuser'), 'dbuser was removed from query');
           assert.ok(req.hasOwnProperty('params'), 'request has params');
           assert.ok(req.params.hasOwnProperty('interactivity'), 'request params have interactivity');
-          assert.equal(req.params.dbname, 'test_cartodb_user_1_db', 'could forge dbname: '+ req.params.dbname);
-          assert.ok(req.params.dbuser === 'testpublicuser', 'could inject dbuser ('+req.params.dbuser+')');
+          assert.equal(req.params.dbname, test_database, 'could forge dbname: '+ req.params.dbname);
+          assert.ok(req.params.dbuser === test_pubuser, 'could inject dbuser ('+req.params.dbuser+')');
           done();
       });
     });
@@ -34,10 +39,8 @@ suite('req2params', function() {
           assert.ok(!req.query.hasOwnProperty('dbuser'), 'dbuser was removed from query');
           assert.ok(req.hasOwnProperty('params'), 'request has params');
           assert.ok(req.params.hasOwnProperty('interactivity'), 'request params have interactivity');
-          // database_name for user "localhost" (see test/support/prepare_db.sh)
-          assert.equal(req.params.dbname, 'test_cartodb_user_1_db');
-          // unauthenticated request gets no dbuser
-          assert.ok(req.params.dbuser === 'testpublicuser', 'could inject dbuser ('+req.params.dbuser+')');
+          assert.equal(req.params.dbname, test_database);
+          assert.ok(req.params.dbuser === test_pubuser, 'could inject dbuser ('+req.params.dbuser+')');
           done();
       });
     });
@@ -50,14 +53,12 @@ suite('req2params', function() {
           assert.ok(!req.query.hasOwnProperty('dbuser'), 'dbuser was removed from query');
           assert.ok(req.hasOwnProperty('params'), 'request has params');
           assert.ok(req.params.hasOwnProperty('interactivity'), 'request params have interactivity');
-          // database_name for user "localhost" (see test/support/prepare_db.sh)
-          assert.equal(req.params.dbname, 'test_cartodb_user_1_db');
-          // id for user "localhost" (see test/support/prepare_db.sh)
-          assert.equal(req.params.dbuser, 'test_cartodb_user_1');
+          assert.equal(req.params.dbname, test_database);
+          assert.equal(req.params.dbuser, test_user);
  
           opts.req2params({headers: { host:'localhost' }, query: {map_key: '1235'} }, function(err, req) {
               // wrong key resets params to no user
-              assert.ok(req.params.dbuser === 'testpublicuser', 'could inject dbuser ('+req.params.dbuser+')');
+              assert.ok(req.params.dbuser === test_pubuser, 'could inject dbuser ('+req.params.dbuser+')');
               done();
           });
       });

test/unit/cartodb/template_maps.test.js

@@ -420,5 +420,87 @@ suite('template_maps', function() {
     catch (e) { err = e; }
     assert.ok(!err);
   });
+
+  // Can set a limit on the number of user templates
+  test('can limit number of user templates', function(done) {
+    var tmap = new TemplateMaps(redis_pool, signed_maps, {
+      max_user_templates: 2
+    });
+    assert.ok(tmap);
+    var tpl = { version:'0.0.1', auth: {}, layergroup: {} };
+    var expectErr = false;
+    var idMe = [];
+    var idYou = [];
+    Step(
+      function oneForMe() {
+        tpl.name = 'oneForMe';
+        tmap.addTemplate('me', tpl, this);
+      },
+      function twoForMe(err, id) {
+        if ( err ) throw err;
+        assert.ok(id);
+        idMe.push(id);
+        tpl.name = 'twoForMe';
+        tmap.addTemplate('me', tpl, this);
+      },
+      function threeForMe(err, id) {
+        if ( err ) throw err;
+        assert.ok(id);
+        idMe.push(id);
+        tpl.name = 'threeForMe';
+        expectErr = true;
+        tmap.addTemplate('me', tpl, this);
+      },
+      function errForMe(err, id) {
+        if ( err && ! expectErr ) throw err;
+        expectErr = false;
+        assert.ok(err);
+        assert.ok(err.message.match(/limit.*template/), err);
+        return null;
+      },
+      function delOneMe(err) {
+        if ( err ) throw err;
+        tmap.delTemplate('me', idMe.shift(), this);
+      },
+      function threeForMeRetry(err) {
+        if ( err ) throw err;
+        tpl.name = 'threeForMe';
+        tmap.addTemplate('me', tpl, this);
+      },
+      function oneForYou(err, id) {
+        if ( err ) throw err;
+        assert.ok(id);
+        idMe.push(id);
+        tpl.name = 'oneForYou';
+        tmap.addTemplate('you', tpl, this);
+      },
+      function twoForYou(err, id) {
+        if ( err ) throw err;
+        assert.ok(id);
+        idYou.push(id);
+        tpl.name = 'twoForYou';
+        tmap.addTemplate('you', tpl, this);
+      },
+      function threeForYou(err, id) {
+        if ( err ) throw err;
+        assert.ok(id);
+        idYou.push(id);
+        tpl.name = 'threeForYou';
+        expectErr = true;
+        tmap.addTemplate('you', tpl, this);
+      },
+      function errForYou(err, id) {
+        if ( err && ! expectErr ) throw err;
+        expectErr = false;
+        assert.ok(err);
+        assert.ok(err.message.match(/limit.*template/), err);
+        return null;
+      },
+      function finish(err) {
+        // TODO: delete all templates
+        done(err);
+      }
+    );
+  });
     
 });

tools/examples/mapconfig_simple.js

@@ -0,0 +1,11 @@
+{"version":"1.0.1",
+	"layers":[{
+    "type":"cartodb",
+	  "options":{
+      "sql":"select 1 as id, ST_SetSRID(ST_MakePoint(0,0),3857) as the_geom_webmercator",
+	    "cartocss":"#style{ marker-width: 12;}",
+	    "cartocss_version":"2.1.1",
+      "Interactivity":"id"
+    }
+  }]
+}

tools/examples/template_simple.js

@@ -0,0 +1,17 @@
+{
+ "version":"0.0.1",
+ "name":"simple",
+ "placeholders":{},
+ "auth":{ "method":"open" },
+ "layergroup":{
+   "version":"1.0.1",
+    "layers":[{
+      "type":"cartodb",
+      "options":{
+        "sql":"select ST_SetSRID(ST_MakePoint(0,0),3857) as the_geom_webmercator",
+        "cartocss":"#s{ marker-width: 12;}",
+        "cartocss_version":"2.1.1"
+      }
+    }]
+  }
+}

tools/flush_cache

@@ -0,0 +1,59 @@
+#!/usr/bin/env node
+
+var path = require('path');
+var request    = require('request');
+
+function usage(me, exitcode) {
+  console.log("Usage: " + me + " [--env <environment>] <username> <tablename>");
+  process.exit(exitcode);
+}
+
+var node_path = process.argv.shift(); 
+var script_path = process.argv.shift(); 
+var basedir = path.dirname(script_path);
+var me = path.basename(script_path);
+
+var ENV = 'development.js';
+var username, table;
+var arg;
+while ( arg = process.argv.shift() ) {
+  if ( arg == '--env' ) {
+    ENV = process.argv.shift();
+  }
+  else if ( ! username ) {
+    username = arg;
+  }
+  else if ( ! table ) {
+    table = arg;
+  }
+  else {
+    console.warn("Unused parameter " + arg);
+  }
+}
+
+if ( ! table ) {
+  usage(me, 1);
+}
+
+global.environment = require('../config/environments/' + ENV);
+
+// _after_ setting global.environment
+var serverOptions = require('../lib/cartodb/server_options');
+
+var host = global.environment.host;
+var port = global.environment.port;
+var re = ''+serverOptions.re_userFromHost;
+var hostname = re.replace(/^\/\^/, '')
+                 .replace(/\/$$/, '')
+                 .replace(/\\/g,'')
+                 .replace(/\([^)]*\)/,username)
+              ;
+//console.log("re: " + re);
+//console.log("hostname: " + hostname);
+
+var url = 'http://' + host + ':' + port + '/tiles/' + table + '/flush_cache';
+request.del({ url: url, headers: { host: hostname } },
+function(err, res, body) {
+  if ( err ) throw err;
+  console.log(res.body);
+});

tools/instanciate_template

@@ -49,5 +49,5 @@ if test x${verbose} = xyes; then
   echo "${res}"
 fi
 
-tok=`echo "$res" | sed 's/.*"template_id":"\([^"]*\)".*/\1/'`
+tok=`echo "$res" | sed 's/.*"layergroupid":"\([^"]*\)".*/\1/'`
 echo $tok

tools/list_templates

@@ -0,0 +1,45 @@
+#!/bin/sh
+
+verbose=no
+tiler_url=http://dev.localhost.lan:8181/tiles/template
+apikey=${CDB_APIKEY}
+
+while test -n "$1"; do
+  if test "$1" = "-v"; then
+    verbose=yes
+  elif test "$1" = "-k"; then
+    shift
+    apikey="$1"
+  elif test "$1" = "-u"; then
+    shift
+    tiler_url="$1"
+  elif test "$1" = "-h" -o "$1" = "-?"; then
+    echo "Usage: $0 [-v] [-k <api_key>] [-u <tiler_url>]" >&2
+    echo "Default <tiler_url> is ${tiler_url}" >&2
+    echo "Default <api_key> is read from CDB_APIKEY env variable" >&2
+    exit 0
+  else
+    echo "Unused parameter $1" >&2
+  fi
+  shift
+done
+
+
+cmd="curl -X GET -sk ${tiler_url}?api_key=${apikey}"
+if test x${verbose} = xyes; then
+  cmd="${cmd} -v"
+fi
+res=`${cmd}`
+if test $? -gt 0; then
+  echo "curl command failed: ${cmd}"
+fi
+
+if test x${verbose} = xyes; then
+  echo "${res}"
+fi
+
+node <<EOF
+var parsed = JSON.parse('$res');
+console.dir(parsed);
+EOF
+

tools/munin/windshaft

@@ -68,7 +68,7 @@ for pid in ${pids}; do
   log=$(grep "${pid}" "${tmpreport}" | grep -w 1w | awk '{print $9}')
   if test -e "${log}"; then
     kill -USR2 "${pid}"
-    cnt=$(tac ${log} | sed -n -e '/ItemKey/p;/^RenderCache/q' | wc -l)
+    cnt=$(tac ${log} | sed -n -e '/ItemKey/p;/ RenderCache /q' | wc -l)
     if test $cnt -gt $maxcache; then maxcache=$cnt; fi
   else
     # report the error...