CDB-3256 Prepares 1.12.1 release

Conflicts:
	NEWS.md
	package.json

.travis.yml

@@ -1,16 +1,32 @@
 before_install:
+  - sudo mv /etc/apt/sources.list.d/pgdg-source.list* /tmp
+  - sudo apt-get -qq purge postgis* postgresql*
+  - sudo apt-add-repository --yes ppa:cartodb/postgresql-9.3
+  - sudo apt-add-repository --yes ppa:cartodb/gis
+  - sudo rm -Rf /var/lib/postgresql /etc/postgresql
   - sudo apt-add-repository --yes ppa:mapnik/v2.1.0
-  - sudo apt-get update -q
-  - sudo apt-get install -q libmapnik-dev
+  - sudo apt-get update
+  - sudo apt-get install -y postgresql-9.3-postgis-2.1
+  - sudo apt-get install -y postgresql-contrib-9.3
+  - sudo apt-get install -y libmapnik-dev
+  - sudo apt-get install -y gdal-bin
+  - echo -e "local\tall\tall\ttrust\nhost\tall\tall\t127.0.0.1/32\ttrust\nhost\tall\tall\t::1/128\ttrust" |sudo tee /etc/postgresql/9.3/main/pg_hba.conf
+  - sudo service postgresql restart
   - createdb template_postgis
   - psql -c "CREATE EXTENSION postgis" template_postgis
 
+before_script:
+  # Tell npm to use known registrars:
+  # see http://blog.npmjs.org/post/78085451721/npms-self-signed-certificate-is-no-more
+  - npm config set ca ""
+
 env:
-  - NPROCS=1 JOBS=1
+  - NPROCS=1 JOBS=1 PGUSER=postgres
 
 language: node_js
 node_js:
   - "0.8"
+  - "0.10"
 
 notifications:
   irc:

HOWTO_RELEASE

@@ -1,11 +1,20 @@
-1. Ensure proper version in package.json 
-2. Ensure NEWS section exists for the new version, review it, add release date
-3. Drop npm-shrinkwrap.json
-4. Run npm install
-5. Test (make check or npm test), fix if broken before proceeding
-6. Run npm shrinkwrap
-7. Commit package.json, npm-shrinwrap.json, NEWS
-8. Tag Major.Minor.Patch
-9. Announce
-10. Stub NEWS/package for next version
+1. Test (make clean all check), fix if broken before proceeding
+2. Ensure proper version in package.json 
+3. Ensure NEWS section exists for the new version, review it, add release date
+4. Drop npm-shrinkwrap.json
+5. Run npm shrinkwrap to recreate npm-shrinkwrap.json
+6. Commit package.json, npm-shrinwrap.json, NEWS
+7. git tag -a Major.Minor.Patch # use NEWS section as content
+8. Announce on cartodb@googlegroups.com
+9. Stub NEWS/package for next version
 
+Versions:
+
+Bugfix releases increment Patch component of version.
+Feature releases increment Minor and set Patch to zero.
+If backward compatibility is broken, increment Major and
+set to zero Minor and Patch.
+
+Branches named 'b<Major>.<Minor>' are kept for any critical
+fix that might need to be shipped before next feature release
+is ready.

LICENCE

@@ -1,27 +0,0 @@
-Copyright (c) 2011, Vizzuality 
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-1. Redistributions of source code must retain the above copyright
-   notice, this list of conditions and the following disclaimer.
-2. Redistributions in binary form must reproduce the above copyright
-   notice, this list of conditions and the following disclaimer in the
-   documentation and/or other materials provided with the distribution.
-3. All advertising materials mentioning features or use of this software
-   must display the following acknowledgement:
-   This product includes software developed by Vizzuality.
-4. Neither the name of Vizzuality nor the
-   names of its contributors may be used to endorse or promote products
-   derived from this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS "AS IS" AND ANY
-EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY
-DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2014, Vizzuality
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without 
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this 
+list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, 
+this list of conditions and the following disclaimer in the documentation 
+and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors 
+may be used to endorse or promote products derived from this software without 
+specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

NEWS.md

@@ -1,3 +1,180 @@
+1.12.1 -- 2014-06-24
+--------------------
+
+Enhancements:
+ - Caches layergroup and sets X-Cache-Channel in GET requests also in named maps
+
+1.12.0 -- 2014-06-24
+--------------------
+
+New features:
+ - Caches layergroup and sets X-Cache-Channel in GET requests
+
+1.11.1 -- 2014-05-07
+--------------------
+
+Enhancements:
+
+ - Upgrade Windshaft to 0.21.0, see
+ http://github.com/CartoDB/Windshaft/blob/0.21.0/NEWS
+
+1.11.0 -- 2014-04-28
+--------------------
+
+New features:
+
+ - Add support for log_filename directive
+ - Reopen log file on SIGHUP, for better logrotate integration
+
+Enhancements:
+
+ - Set default PostgreSQL application name to "cartodb_tiler"
+
+1.10.2 -- 2014-04-08
+--------------------
+
+Bug fixes:
+
+ - Fix show_style tool broken since 1.8.1
+ - Fix X-Cache-Channel of tiles accessed via signed token (#188)
+
+1.10.1 -- 2014-03-21
+--------------------
+
+Bug fixes:
+
+ - Do not cache non-success jsonp responses (#186)
+
+1.10.0 -- 2014-03-20
+--------------------
+
+New features:
+
+ - Add optional support for rollbar (#150)
+
+Enhancements:
+
+ - Do not send connection details to client (#183)
+ - Upgrade node-varnish to 0.3.0
+ - Upgrade Windshaft to 0.20.0, see
+   http://github.com/CartoDB/Windshaft/blob/0.20.0/NEWS
+ - Include tiler version in startup log
+ - Install an uncaught exception handler
+ - Require own fork of node-mapnik, with temptative fix
+   for libxml usage (glibc detected corruptions)
+
+Other changes:
+
+ - Switch to 3-clause BSD license (#184)
+
+1.9.0 -- 2014-03-10
+-------------------
+
+New features:
+
+ - Allow to set server related configuration in serverMetadata (#182)
+
+1.8.5 -- 2014-03-10
+-------------------
+
+Enhancements:
+
+ - Set statsd prefix for all endpoints 
+ - Respond with a permission denied on attempt to access map tiles waiving
+   signature of someone who had not left any (#170)
+ - Do not log an error on GET / (#177)
+ - Do not UNWATCH on every redis client release (#161)
+ - Include API docs (#164)
+ - Add "cacheDns" statsd setting in the example configs
+ - Do not send duplicated stats on template instanciation
+ - Do not die on dns resolution errors (#178, #180)
+
+Bug fixes:
+
+ - Do not cache map creation responses (#176)
+
+1.8.4 -- 2014-03-03
+-------------------
+
+Enhancements:
+
+ - Really skip CDB_TableMetadata lookup for sql affected by no tables (#169)
+ - Upgrade windshaft to 0.19.2, see node_modules/windshaft/NEWS
+ - Clarify obscure "ECONNREFUSED" error message (#171)
+ - Change some http status responses to be more appropriate to the case
+ - Forbid using map signatures of foreign users (#172)
+ - Forbid instanciating templates of foreign users (#173)
+ - Allow passing environment configuration name via NODE_ENV to app.js
+ - Print environment configuration name on app start
+
+Bug fixes:
+
+ - Fix database connection settings on template instanciation (#174)
+
+1.8.3 -- 2014-02-27
+-------------------
+
+Enhancements:
+
+ - Upgrades windshaft to 0.19.1 with many performance improvements,
+   See node_modules/windshaft/NEWS 
+ - Improve speed of instanciating a map (#147, #159, #165)
+ - Give meaningful error on attempts to use map tokens
+   with attribute service (#156)
+ - Reduce sql-api communication timeout, and allow overriding (#167)
+   [ new sqlapi.timeout directive, defaults to 100 ms ]
+ - Do not query CDB_TableMetadata for queries affected by no tables (#168)
+
+1.8.2 -- 2014-02-25
+-------------------
+
+Enhancements:
+
+ * Allow using ":host" as part of statsd.prefix (#153)
+ * Expand "addCacheChannel" stats
+ * Allow using GET with sql-api for queries shorter than configured len (#155)
+   [ new sqlapi.max_get_sql_length directive, defaults to 2048 ]
+ * Do not log an error for a legit request requiring no X-Cache-Channel
+
+Bug fixes:
+
+ * Fix munin plugin after log format changes (#154)
+
+1.8.1 -- 2014-02-19
+-------------------
+
+Enhancements:
+
+ * Use log4js logger (#138)
+
+Bug fixes:
+
+ * Always generate X-Cache-Channel for token-based tile responses (#152)
+
+1.8.0 -- 2014-02-18
+-------------------
+
+Enhancements:
+
+ * Add script to flush caches (#140)
+ * Add script to list templates
+ * Add statsd support (#139)
+ * Add support for specifying a varnish password
+ * Avoid sending multiple varnish invalidation at once (#135)
+ * Tested with node-0.10 (#141)
+ * Use single redis pooler for torque and grainstore
+ * Reduce cost of garbage collection for localized resources
+ * Allow limiting number of templates for each user (#136)
+ * Allow configuring TTL of mapConfigs via "mapConfigTTL"
+
+1.7.1 -- 2014-02-11
+-------------------
+
+Enhancements:
+
+ * Disable debug logging unless "debug" config param evaluates to true (#137)
+ * Require windshaft 0.17.2 for further reducing log noise (#137)
+
 1.7.0 -- 2014-02-11
 -------------------
 

app.js

@@ -7,27 +7,67 @@
  * environments: [development, production]
  */
 
+var path = require('path'),
+    fs = require('fs')
+;
+
+
+if ( process.argv[2] ) ENV = process.argv[2];
+else if ( process.env['NODE_ENV'] ) ENV = process.env['NODE_ENV'];
+else ENV = 'development';
+
+process.env['NODE_ENV'] = ENV;
 
 // sanity check
-var ENV = process.argv[2]
 if (ENV != 'development' && ENV != 'production' && ENV != 'staging' ){
     console.error("\nnode app.js [environment]");
-    console.error("environments: [development, production, staging]\n");
+    console.error("environments: development, production, staging\n");
     process.exit(1);
 }
 
-var _ = require('underscore')
-    , Step       = require('step')
-    ;
+var _ = require('underscore');
+
 // set environment specific variables
-global.settings     = require(__dirname + '/config/settings');
 global.environment  = require(__dirname + '/config/environments/' + ENV);
-_.extend(global.settings, global.environment);
+
+global.log4js = require('log4js')
+log4js_config = {
+  appenders: [],
+  replaceConsole:true
+};
+
+if ( global.environment.log_filename ) {
+  var logdir = path.dirname(global.environment.log_filename);
+  // See cwd inlog4js.configure call below
+  logdir = path.resolve(__dirname, logdir);
+  if ( ! fs.existsSync(logdir) ) {
+    console.error("Log filename directory does not exist: " + logdir);
+    process.exit(1);
+  }
+  console.log("Logs will be written to " + global.environment.log_filename);
+  log4js_config.appenders.push(
+    { type: "file", filename: global.environment.log_filename }
+  );
+} else {
+  log4js_config.appenders.push(
+    { type: "console", layout: { type:'basic' } }
+  );
+}
+
+if ( global.environment.rollbar ) {
+  log4js_config.appenders.push({
+    type: __dirname + "/lib/cartodb/log4js_rollbar.js",
+    options: global.environment.rollbar
+  });
+}
+
+log4js.configure(log4js_config, { cwd: __dirname });
+global.logger = log4js.getLogger();
 
 // Include cartodb_windshaft only _after_ the "global" variable is set
 // See https://github.com/Vizzuality/Windshaft-cartodb/issues/28
 var CartodbWindshaft = require('./lib/cartodb/cartodb_windshaft');
-var serverOptions = require('./lib/cartodb/server_options');
+var serverOptions = require('./lib/cartodb/server_options')();
 
 ws = CartodbWindshaft(serverOptions);
 
@@ -39,8 +79,12 @@ ws.maxConnections = global.environment.maxConnections || 128;
 
 ws.listen(global.environment.port, global.environment.host);
 
+var version = require("./package").version;
+
 ws.on('listening', function() {
-  console.log("Windshaft tileserver started on " + global.environment.host + ':' + global.environment.port);
+  console.log("Windshaft tileserver " + version + " started on "
+              + global.environment.host + ':' + global.environment.port
+              + " (" + ENV + ")");
 });
 
 // DEPRECATED, use SIGUSR2
@@ -52,3 +96,12 @@ process.on('SIGUSR1', function() {
 process.on('SIGUSR2', function() {
   ws.dumpCacheStats();
 });
+
+process.on('SIGHUP', function() {
+  log4js.configure(log4js_config);
+  console.log('Log files reloaded');
+});
+
+process.on('uncaughtException', function(err) {
+  logger.error('Uncaught exception: ' + err.stack); 
+});

config/environments/development.js.example

@@ -11,24 +11,35 @@ var config = {
     // See http://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API
     // 
     // Base url for the Templated Maps API
-    // "/maps/named" is the new API,
+    // "/api/v1/map/named" is the new API,
     // "/tiles/template" is for compatibility with versions up to 1.6.x
-    ,base_url_templated: '(?:/maps/named|/tiles/template)'
+    ,base_url_templated: '(?:/api/v1/map/named|/tiles/template)'
     // Base url for the Detached Maps API
     // "maps" is the the new API,
     // "tiles/layergroup" is for compatibility with versions up to 1.6.x
-    ,base_url_detached: '(?:/maps|/tiles/layergroup)'
+    ,base_url_detached: '(?:/api/v1/map|/tiles/layergroup)'
     // Base url for the Inline Maps and Table Maps API
     ,base_url_legacy: '/tiles/:table'
 
     // Maximum number of connections for one process
     // 128 is a good value with a limit of 1024 open file descriptors
     ,maxConnections:128
-    // idle socket timeout, in miliseconds 
+    // Maximum number of templates per user. Unlimited by default.
+    ,maxUserTemplates:1024
+    // Seconds since "last creation" before a detached
+    // or template instance map expires. Or: how long do you want
+    // to be able to navigate the map without a reload ?
+    // Defaults to 7200 (2 hours)
+    ,mapConfigTTL: 7200
+    // idle socket timeout, in milliseconds 
     ,socket_timeout: 600000
     ,enable_cors: true
     ,cache_enabled: false
-    ,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
+    ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
+    // If log_filename is given logs will be written
+    // there, in append mode. Otherwise stdout is used (default).
+    // Log file will be re-opened on receiving the HUP signal
+    ,log_filename: 'logs/node-windshaft.log'
     // Templated database username for authorized user
     // Supported labels: 'user_id' (read from redis)
     ,postgres_auth_user: 'development_cartodb_user_<%= user_id %>'
@@ -51,9 +62,24 @@ var config = {
         */
         row_limit: 65535,
         simplify_geometries: true,
+        /*
+         * Set persist_connection to false if you want
+         * database connections to be closed on renderer
+         * expiration (1 minute after last use).
+         * Setting to true (the default) would never
+         * close any connection for the server's lifetime
+         */
+        persist_connection: false,
         max_size: 500
     }
     ,mapnik_version: undefined
+    ,statsd: {
+        host: 'localhost',
+        port: 8125,
+        prefix: 'dev.',
+        cacheDns: true
+        // support all allowed node-statsd options
+    }
     ,renderer: {
       // Milliseconds since last access before renderer cache item expires
       cache_ttl: 60000,
@@ -70,9 +96,9 @@ var config = {
         // Max number of connections in each pool.
         // Users will be put on a queue when the limit is hit.
         // Set to maxConnection to have no possible queues.
-        // There are currently 3 pools involved in serving
+        // There are currently 2 pools involved in serving
         // windshaft-cartodb requests so multiply this number
-        // by 3 to know how many possible connections will be
+        // by 2 to know how many possible connections will be
         // kept open by the server. The default is 50.
         max: 50,
         idleTimeoutMillis: 1, // idle time before dropping connection
@@ -89,11 +115,19 @@ var config = {
         // the cartodb username and passed to
         // SQL-API requests in the Host HTTP header
         domain: 'localhost.lan',
-        version: 'v1'
+        version: 'v1',
+        // Maximum lenght of SQL query for GET
+        // requests. Longer queries will be sent
+        // using POST. Defaults to 2048
+        max_get_sql_length: 2048,
+        // Maximum time to wait for a response,
+        // in milliseconds. Defaults to 100.
+        timeout: 100
     }
     ,varnish: {
         host: 'localhost',
         port: 6082,
+        secret: 'xxx',
         ttl: 86400
     }
     // If useProfiler is true every response will be served with an

config/environments/production.js.example

@@ -11,24 +11,35 @@ var config = {
     // See http://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API
     // 
     // Base url for the Templated Maps API
-    // "/maps/named" is the new API,
+    // "/api/v1/map/named" is the new API,
     // "/tiles/template" is for compatibility with versions up to 1.6.x
-    ,base_url_templated: '(?:/maps/named|/tiles/template)'
+    ,base_url_templated: '(?:/api/v1/map/named|/tiles/template)'
     // Base url for the Detached Maps API
     // "maps" is the the new API,
     // "tiles/layergroup" is for compatibility with versions up to 1.6.x
-    ,base_url_detached: '(?:/maps|/tiles/layergroup)'
+    ,base_url_detached: '(?:/api/v1/map|/tiles/layergroup)'
     // Base url for the Inline Maps and Table Maps API
     ,base_url_legacy: '/tiles/:table'
 
     // Maximum number of connections for one process
     // 128 is a good value with a limit of 1024 open file descriptors
     ,maxConnections:128
-    // idle socket timeout, in miliseconds
+    // Maximum number of templates per user. Unlimited by default.
+    ,maxUserTemplates:1024
+    // Seconds since "last creation" before a detached
+    // or template instance map expires. Or: how long do you want
+    // to be able to navigate the map without a reload ?
+    // Defaults to 7200 (2 hours)
+    ,mapConfigTTL: 7200
+    // idle socket timeout, in milliseconds
     ,socket_timeout: 600000 
     ,enable_cors: true
     ,cache_enabled: true
-    ,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
+    ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
+    // If log_filename is given logs will be written
+    // there, in append mode. Otherwise stdout is used (default).
+    // Log file will be re-opened on receiving the HUP signal
+    ,log_filename: 'logs/node-windshaft.log'
     // Templated database username for authorized user
     // Supported labels: 'user_id' (read from redis)
     ,postgres_auth_user: 'cartodb_user_<%= user_id %>'
@@ -44,10 +55,25 @@ var config = {
         port: 6432,
         extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
         row_limit: 65535,
+        /*
+         * Set persist_connection to false if you want
+         * database connections to be closed on renderer
+         * expiration (1 minute after last use).
+         * Setting to true (the default) would never
+         * close any connection for the server's lifetime
+         */
+        persist_connection: false,
         simplify_geometries: true,
         max_size: 500
     }
     ,mapnik_version: undefined
+    ,statsd: {
+        host: 'localhost',
+        port: 8125,
+        prefix: ':host.', // could be hostname, better not containing dots
+        cacheDns: true
+        // support all allowed node-statsd options
+    }
     ,renderer: {
       // Milliseconds since last access before renderer cache item expires
       cache_ttl: 60000,
@@ -64,9 +90,9 @@ var config = {
         // Max number of connections in each pool.
         // Users will be put on a queue when the limit is hit.
         // Set to maxConnection to have no possible queues.
-        // There are currently 3 pools involved in serving
+        // There are currently 2 pools involved in serving
         // windshaft-cartodb requests so multiply this number
-        // by 3 to know how many possible connections will be
+        // by 2 to know how many possible connections will be
         // kept open by the server. The default is 50.
         max: 50,
         idleTimeoutMillis: 30000, // idle time before dropping connection
@@ -83,17 +109,40 @@ var config = {
         // the cartodb username and passed to
         // SQL-API requests in the Host HTTP header
         domain: 'cartodb.com',
-        version: 'v2'
+        version: 'v2',
+        // Maximum lenght of SQL query for GET
+        // requests. Longer queries will be sent
+        // using POST. Defaults to 2048
+        max_get_sql_length: 2048,
+        // Maximum time to wait for a response,
+        // in milliseconds. Defaults to 100.
+        timeout: 100
     }
     ,varnish: {
         host: 'localhost',
         port: 6082,
+        secret: 'xxx',
         ttl: 86400
     }
     // If useProfiler is true every response will be served with an
     // X-Tiler-Profile header containing elapsed timing for various 
     // steps taken for producing the response.
     ,useProfiler:false
+    ,serverMetadata: {
+      cdn_url: {
+        http: 'api.cartocdn.com',
+        https: 'cartocdn.global.ssl.fastly.net'
+      }
+    }
+    // Optional rollbar support
+    ,rollbar: {
+      token: 'secret',
+      // See http://github.com/rollbar/node_rollbar#configuration-reference
+      options: {
+        endpoint: 'https://api.rollbar.com/api/1/',
+        handler: 'inline'
+      }
+    }
 };
 
 module.exports = config;

config/environments/staging.js.example

@@ -24,11 +24,22 @@ var config = {
     // Maximum number of connections for one process
     // 128 is a good value with a limit of 1024 open file descriptors
     ,maxConnections:128
-    // idle socket timeout, in miliseconds
+    // Maximum number of templates per user. Unlimited by default.
+    ,maxUserTemplates:1024
+    // Seconds since "last creation" before a detached
+    // or template instance map expires. Or: how long do you want
+    // to be able to navigate the map without a reload ?
+    // Defaults to 7200 (2 hours)
+    ,mapConfigTTL: 7200
+    // idle socket timeout, in milliseconds
     ,socket_timeout: 600000 
     ,enable_cors: true
     ,cache_enabled: true
-    ,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms (:res[X-Tiler-Profiler]) -> :res[Content-Type]'
+    ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms (:res[X-Tiler-Profiler]) -> :res[Content-Type]'
+    // If log_filename is given logs will be written
+    // there, in append mode. Otherwise stdout is used (default).
+    // Log file will be re-opened on receiving the HUP signal
+    ,log_filename: 'logs/node-windshaft.log'
     // Templated database username for authorized user
     // Supported labels: 'user_id' (read from redis)
     ,postgres_auth_user: 'cartodb_staging_user_<%= user_id %>'
@@ -45,9 +56,24 @@ var config = {
         extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
         row_limit: 65535,
         simplify_geometries: true,
+        /*
+         * Set persist_connection to false if you want
+         * database connections to be closed on renderer
+         * expiration (1 minute after last use).
+         * Setting to true (the default) would never
+         * close any connection for the server's lifetime
+         */
+        persist_connection: false,
         max_size: 500
     }
     ,mapnik_version: undefined
+    ,statsd: {
+        host: 'localhost',
+        port: 8125,
+        prefix: 'stage.:host.',
+        cacheDns: true
+        // support all allowed node-statsd options
+    }
     ,renderer: {
       // Milliseconds since last access before renderer cache item expires
       cache_ttl: 60000,
@@ -64,9 +90,9 @@ var config = {
         // Max number of connections in each pool.
         // Users will be put on a queue when the limit is hit.
         // Set to maxConnection to have no possible queues.
-        // There are currently 3 pools involved in serving
+        // There are currently 2 pools involved in serving
         // windshaft-cartodb requests so multiply this number
-        // by 3 to know how many possible connections will be
+        // by 2 to know how many possible connections will be
         // kept open by the server. The default is 50.
         max: 50,
         idleTimeoutMillis: 30000, // idle time before dropping connection
@@ -83,17 +109,40 @@ var config = {
         // the cartodb username and passed to
         // SQL-API requests in the Host HTTP header
         domain: 'cartodb.com',
-        version: 'v2'
+        version: 'v2',
+        // Maximum lenght of SQL query for GET
+        // requests. Longer queries will be sent
+        // using POST. Defaults to 2048
+        max_get_sql_length: 2048,
+        // Maximum time to wait for a response,
+        // in milliseconds. Defaults to 100.
+        timeout: 100
     }
     ,varnish: {
         host: 'localhost',
         port: 6082,
+        secret: 'xxx',
         ttl: 86400
     }
     // If useProfiler is true every response will be served with an
     // X-Tiler-Profile header containing elapsed timing for various 
     // steps taken for producing the response.
     ,useProfiler:true
+    ,serverMetadata: {
+      cdn_url: {
+        http: 'api.cartocdn.com',
+        https: 'cartocdn.global.ssl.fastly.net'
+      }
+    }
+    // Optional rollbar support
+    ,rollbar: {
+      token: 'secret',
+      // See http://github.com/rollbar/node_rollbar#configuration-reference
+      options: {
+        endpoint: 'https://api.rollbar.com/api/1/',
+        handler: 'inline'
+      }
+    }
 };
 
 module.exports = config;

config/environments/test.js.example

@@ -11,30 +11,41 @@ var config = {
     // See https://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API
     // 
     // Base url for the Templated Maps API
-    // "/maps/named" is the new API,
+    // "/api/v1/map/named" is the new API,
     // "/tiles/template" is for compatibility with versions up to 1.6.x
-    ,base_url_templated: '(?:/maps/named|/tiles/template)'
+    ,base_url_templated: '(?:/api/v1/map/named|/tiles/template)'
     // Base url for the Detached Maps API
     // "maps" is the the new API,
     // "tiles/layergroup" is for compatibility with versions up to 1.6.x
-    ,base_url_detached: '(?:/maps|/tiles/layergroup)'
+    ,base_url_detached: '(?:/api/v1/map|/tiles/layergroup)'
     // Base url for the Inline Maps and Table Maps API
     ,base_url_legacy: '/tiles/:table'
 
     // Maximum number of connections for one process
     // 128 is a good value with a limit of 1024 open file descriptors
     ,maxConnections:128
-    // idle socket timeout, in miliseconds
+    // Maximum number of templates per user. Unlimited by default.
+    ,maxUserTemplates:1024
+    // Seconds since "last creation" before a detached
+    // or template instance map expires. Or: how long do you want
+    // to be able to navigate the map without a reload ?
+    // Defaults to 7200 (2 hours)
+    ,mapConfigTTL: 7200
+    // idle socket timeout, in milliseconds
     ,socket_timeout: 600000 
     ,enable_cors: true
     ,cache_enabled: false
     ,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
+    // If log_filename is given logs will be written
+    // there, in append mode. Otherwise stdout is used (default).
+    // Log file will be re-opened on receiving the HUP signal
+    //,log_filename: 'logs/node-windshaft.log'
     // Templated database username for authorized user
     // Supported labels: 'user_id' (read from redis)
-    ,postgres_auth_user: 'test_cartodb_user_<%= user_id %>'
+    ,postgres_auth_user: 'test_windshaft_cartodb_user_<%= user_id %>'
     // Templated database password for authorized user
     // Supported labels: 'user_id', 'user_password' (both read from redis)
-    ,postgres_auth_pass: 'test_cartodb_user_<%= user_id %>_pass'
+    ,postgres_auth_pass: 'test_windshaft_cartodb_user_<%= user_id %>_pass'
     ,postgres: {
         // Parameters to pass to datasource plugin of mapnik
         // See http://github.com/mapnik/mapnik/wiki/PostGIS
@@ -45,9 +56,24 @@ var config = {
         extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
         row_limit: 65535,
         simplify_geometries: true,
+        /*
+         * Set persist_connection to false if you want
+         * database connections to be closed on renderer
+         * expiration (1 minute after last use).
+         * Setting to true (the default) would never
+         * close any connection for the server's lifetime
+         */
+        persist_connection: false,
         max_size: 500
     }
-    ,mapnik_version: '2.0.2'
+    ,mapnik_version: ''
+    ,statsd: {
+        host: 'localhost',
+        port: 8125,
+        prefix: 'test.:host.',
+        cacheDns: true
+        // support all allowed node-statsd options
+    }
     ,renderer: {
       // Milliseconds since last access before renderer cache item expires
       cache_ttl: 60000,
@@ -64,9 +90,9 @@ var config = {
         // Max number of connections in each pool.
         // Users will be put on a queue when the limit is hit.
         // Set to maxConnection to have no possible queues.
-        // There are currently 3 pools involved in serving
+        // There are currently 2 pools involved in serving
         // windshaft-cartodb requests so multiply this number
-        // by 3 to know how many possible connections will be
+        // by 2 to know how many possible connections will be
         // kept open by the server. The default is 50.
         max: 50,
         idleTimeoutMillis: 1, // idle time before dropping connection
@@ -85,17 +111,25 @@ var config = {
         domain: 'donot_look_this_up',
         // This port will be used by "make check" for testing purposes
         // It must be available
-        version: 'v1'
+        version: 'v1',
+        // Maximum lenght of SQL query for GET
+        // requests. Longer queries will be sent
+        // using POST. Defaults to 2048
+        max_get_sql_length: 2048,
+        // Maximum time to wait for a response,
+        // in milliseconds. Defaults to 100.
+        timeout: 100
     }
     ,varnish: {
         host: '',
         port: null,
+        secret: 'xxx',
         ttl: 86400
     }
     // If useProfiler is true every response will be served with an
     // X-Tiler-Profile header containing elapsed timing for various 
     // steps taken for producing the response.
-    ,useProfiler:false
+    ,useProfiler:true
 };
 
 module.exports = config;

config/settings.js

@@ -1 +0,0 @@
-module.exports.oneDay   = 86400000;

docs/Map-API.md

@@ -0,0 +1,111 @@
+# Kind of maps
+
+Windshaft-CartoDB supports these kind of maps:
+
+ - [Temporary maps](#temporary-maps) (created by anyone)
+   - [Detached maps](#detached-maps)
+   - [Inline maps](#inline-maps) (legacy)
+ - [Persistent maps](#peristent-maps) (created by CartDB user)
+   - [Template maps](#template-maps)
+   - [Table maps](#table-maps) (legacy, deprecated) 
+
+## Temporary maps
+
+Temporary maps have no owners and are anonymous in nature.
+There are two kind of temporary maps:
+
+ - Detached maps (aka MultiLayer-API)
+ - Inline maps 
+
+### Detached maps
+
+Detached maps are maps which are configured with a request
+obtaining a temporary token and then used by referencing
+the obtained token. The token expires automatically when unused.
+
+Anyone can create detached maps, but users will need read access
+to the data source of the map layers.
+
+The configuration format is a [MapConfig]
+(http://github.com/CartoDB/Windshaft/wiki/MapConfig-specification) document.
+
+The HTTP endpoints for creating the map and using it are described [here]
+(http://github.com/CartoDB/Windshaft-cartodb/wiki/MultiLayer-API)
+
+*TODO* cleanup the referenced document
+
+### Inline maps
+
+Inline maps are maps that only exist for a single request,
+being the request for a specific map resource (tile).
+
+Inline maps are always bound to a table, and can only be
+obtained by those having read access to the that table.
+Additionally, users need to have access to any datasource
+specified as part of the configuration.
+
+Inline maps only support PNG and UTF8GRID tiles.
+
+The configuration consist in a set of parameters, to be
+specified in the query string of the tile request:
+
+ * sql - the query to run as datasource, can be an array
+ * style - the CartoCSS style for the datasource, can be an array
+ * style_version - version of the CartoCSS style, can be an array
+ * interactivity - only for fetching UTF8GRID, 
+
+If the style is not provided, style of the associated table is
+used; if the sql is not provided, all records of the associated
+table are used as the datasource; the two possibilities result
+in a mix between _inline_ maps and [Table maps][].
+
+*TODO* specify (or link) api endpoints
+
+## Persistent maps
+
+Persistent maps can only be created by a CartoDB user who has full
+responsibility over editing and deleting them. There are two
+kind of persistent maps:
+
+ - Template maps 
+ - Table maps (legacy, deprecated) 
+
+### Templated maps
+
+Templated maps are templated [MapConfig]
+(http://github.com/CartoDB/Windshaft/wiki/MapConfig-specification) documents
+associated with an authorization certificate.
+
+The authorization certificate determines who can instanciate the
+template and use the resulting map. Authorized users of the instanciated
+maps will have the same database access privilege of the template owner.
+
+The HTTP endpoints for creating and using templated maps are described [here]
+(http://github.com/CartoDB/Windshaft-cartodb/wiki/Template-maps).
+
+*TODO* cleanup the referenced document
+
+### Table maps
+
+Table maps are maps associated with a table.
+Configuration of such maps is limited to the CartoCSS style.
+
+ * style - the CartoCSS style for the datasource, can be an array
+ * style_version - version of the CartoCSS style, can be an array
+
+You can only fetch PNG or UTF8GRID tiles from these maps.
+
+Access method is the same as the one for [Inline maps](#inline-maps)
+
+# Endpoints description 
+
+- **/api/maps/** (same interface than https://github.com/CartoDB/Windshaft/wiki/Multilayer-API)
+- **/api/maps/named** (same interface than https://github.com/CartoDB/Windshaft-cartodb/wiki/Template-maps)
+
+
+NOTE: in case Multilayer-API does not contain this info yet, the
+      endpoint for fetching attributes is this:
+
+- **/api/maps/:map_id/:layer_index/attributes/:feature_id** 
+   - would return { c: 1, d: 2 }
+

docs/MultiLayer-API.md

@@ -0,0 +1,28 @@
+The Windshaft-CartoDB MultiLayer API extends the [Windshaft MultiLayer API](https://github.com/Vizzuality/Windshaft/wiki/Multilayer-API) in a few ways.
+
+## Last modification timestamp embedded in the token
+
+It encodes a timestamp of 'last modification time' into the map token (token:EPOCH) returned to the client.
+It accepts tokens with encoded timestamp from the client considering the token suffix as a cache_buster value.
+
+Clients don't need to be aware of the extension but rather use the API as they would use the base one.
+The only difference will be that the _same_ layergroup configuration may result in different tokens if source data was modified between the mapview requests. 
+
+## Additional attributes in the response object
+
+Windshaft-CartoDB adds the following attributes in the response object
+
+- ``last_update`` field with ISO format (2013-11-30T12:23:10).
+- ``cdn_url`` object containing CDN url client should use (not mandatory) to access the tiles. It's in the form:
+
+   ```json
+   {
+     http: 'http://cdn_url.com/'
+     https: 'https://secure.cdn_url.com/'
+   }
+   ```
+
+
+## Stats tag
+
+Windshaft-CartoDB adds support for a ``stat_tag`` element in the multilayer configuration to help [stats](Redis-stats-format) gathering.

docs/Template-maps.md

@@ -0,0 +1,293 @@
+Template maps are layergroup configurations that rather than being
+fully defined contain variables that can be set to produce a different
+layergroup configurations (instantiation).
+
+Template maps are persistent, can only be created and deleted by the
+CartoDB user showing a valid API_KEY.
+
+Instantiating a signed template map would result in a [signed
+map](https://github.com/CartoDB/Windshaft-cartodb/wiki/Signed-maps)
+instance that would be signed with the same signature as the template.
+
+Deleting a signed template results in deletion of all signatures created
+as a result of instantiation.
+
+
+# Template format
+
+A templated layergroup would allow using placeholders
+in the "cartocss" and "sql" elements in the "option"
+field of any "layer" of a layergroup configuration
+(see https://github.com/CartoDB/Windshaft/wiki/MapConfig-specification).
+
+Valid placeholder names start with a letter and can only
+contain letters, numbers or underscores. They have to be
+written between ``<%= `` and `` %>`` strings in order to be 
+replaced. Example: ``<%= my_color %>``.
+
+The set of supported placeholders for a template will need to be
+explicitly defined specifying type and default value for each.
+
+**placeholder types**
+
+Placeholder type will determine the kind of escaping for the
+associated value. Supported types are:
+
+ * sql_literal (internal single-quotes will be sql-escaped)
+ * sql_ident (internal double-quotes will be sql-escaped)
+ * number (can only contain numerical representation)
+ * css_color (can only contain color names or hex-values)
+ * ... (add more as need arises)
+
+Placeholder default value will be used when not provided at
+instantiation time and could be used to test validity of the
+template by creating a default instance.
+
+Additionally you'll be able to embed an authorization
+certificate that would be used to sign any instance of the template.
+
+```js
+// template.json 
+{
+  version: "0.0.1",
+  // there can be at most 1 template with the same name for any user 
+  // valid names start with a letter and only contains letter, numbers
+  // or underscores
+  name: "template_name", 
+  // embedded authorization certificate
+  auth: {
+   // See https://github.com/CartoDB/Windshaft-cartodb/wiki/Signed-maps
+   method: "token", // or "open" (the default if no "method" is given)
+   // only (required and non empty) for "token" method
+   valid_tokens: ["auth_token1","auth_token2"]
+  },
+  // Variables not listed here are not substituted
+  // Variable not provided at instantiation time trigger an error
+  // A default is required for optional variables
+  // Type specification is used for quoting, to avoid injections
+  placeholders: {
+      color: {
+        type:"css_color",
+        default:"red"
+      },
+      cartodb_id: {
+        type:"number",
+        default: 1
+      }
+  },
+  layergroup: {
+   // see https://github.com/CartoDB/Windshaft/wiki/MapConfig-specification
+   "version": "1.0.1",
+   "layers": [{
+    "type": "cartodb",
+    "options": {
+      "cartocss_version": "2.1.1", 
+      "cartocss": "#layer { polygon-fill: <%= color %>; }",
+      "sql": "select * from european_countries_e WHERE cartodb_id = <%= cartodb_id %>"
+    }
+   }]
+  } 
+}
+```
+
+# Creating a templated map
+
+You can create a signed template map with a single call (for simplicity).
+You'd use a POST sending JSON data:
+
+```sh
+curl -X POST \
+   -H 'Content-Type: application/json' \
+   -d @template.json \
+   'https://docs.cartodb.com/tiles/template?api_key=APIKEY'
+```
+
+The response would be like this:
+```js
+{
+   "template_id":"@template_name"
+}
+```
+
+If a template with the same name exists in the user storage,
+a 400 response is generated.
+
+Errors are in this form:
+```js
+{
+   "error":"Some error string here"
+}
+```
+
+# Updating an existing template
+
+Update of a template map implies removal all signatures from previous
+map instances.
+
+You can update a signed template map with a PUT:
+
+```sh
+curl -X PUT \
+   -H 'Content-Type: application/json' \
+   -d @template.json \
+   'https://docs.cartodb.com/tiles/template/:template_name?api_key=APIKEY'
+```
+A template with the same name will be updated, if any.
+
+The response would be like this:
+```js
+{
+   "template_id":"@template_name"
+}
+```
+
+If a template with the same name does NOT exist,
+a 400 HTTP response is generated with an error in this format:
+
+```js
+{
+   "error":"Some error string here"
+}
+```
+
+
+# Listing available templates
+
+You can get a list of available templates with a GET to ``/template``.
+A valid api_key is required.
+
+```sh
+curl -X GET 'https://docs.cartodb.com/tiles/template?api_key=APIKEY'
+```
+
+The response would be like this:
+```js
+{
+   "template_ids": ["@template_name1","@template_name2"]
+}
+```
+
+Or, on error:
+
+```js
+{
+   "error":"Some error string here"
+}
+```
+
+# Getting a specific template
+
+You can get the definition of a template with a
+GET to ``/template/:template_name``.
+A valid api_key is required.
+
+Example:
+
+```sh
+curl -X GET 'https://docs.cartodb.com/tiles/template/@template_name?auth_token=AUTH_TOKEN'
+```
+
+The response would be like this:
+```js
+{
+   "template": {...}  // see template.json above
+}
+```
+
+Or, on error:
+
+```js
+{
+   "error":"Some error string here"
+}
+```
+
+# Instantiating a template map
+
+You can instantiate a template map passing all required parameters with
+a POST to ``/template/:template_name``.
+
+Valid credentials will be needed, if required by the template.
+
+```js
+// params.js
+{
+ color: '#ff0000',
+ cartodb_id: 3
+}
+```
+
+```sh
+curl -X POST \
+  -H 'Content-Type: application/json' \
+  -d @params.js \
+  'https://docs.cartodb.com/tiles/template/@template_name?auth_token=AUTH_TOKEN'
+
+```
+
+The response would be like this:
+```js
+{
+   "layergroupid":"docs@fd2861af@c01a54877c62831bb51720263f91fb33:123456788",
+   "last_updated":"2013-11-14T11:20:15.000Z"
+}
+```
+
+or, on error:
+
+```js
+{
+   "error":"Some error string here"
+}
+```
+
+You can then use the ``layergroupid`` for fetching tiles and grids as you do
+normally ( see https://github.com/CartoDB/Windshaft/wiki/Multilayer-API).
+But you'll still have to show the ``auth_token``, if required by the template
+(see https://github.com/CartoDB/Windshaft-cartodb/wiki/Signed-maps)
+
+Instances of a signed template map will be signed with the same signature
+certificate associated with the template. Such certificate would contain
+a reference to the template identifier, so that it can be revoked every
+time the template is updated or deleted. 
+
+### using JSONP
+There is also a special endpoint to be able to instanciate using JSONP (for old browsers)
+
+```
+curl 'https://docs.cartodb.com/tiles/template/@template_name/jsonp?auth_token=AUTH_TOKEN&callback=function_name&config=template_params_json'
+```
+
+it takes the ``callback`` function (required), ``auth_token`` in case the template needs auth and ``config`` which is the variabñes for the template (in case it has variables). For example config may be created (using javascript)
+```
+url += "config=" + encodeURIComponent(
+JSON.stringify({ color: 'red' });
+```
+
+the response it's in this format:
+```
+jQuery17205720721024554223_1390996319118(
+{
+layergroupid: "dev@744bd0ed9b047f953fae673d56a47b4d:1390844463021.1401",
+last_updated: "2014-01-27T17:41:03.021Z"
+}
+)
+```
+# Deleting a template map
+
+Deletion of a template map will imply removal all instance signatures
+
+You can delete a templated map with a DELETE to ``/template/:template_name``:
+
+```sh
+curl -X DELETE 'https://docs.cartodb.com/tiles/template/@template_name?auth_token=AUTH_TOKEN'
+```
+
+On success, a 204 (No Content) response would be issued.
+Otherwise a 4xx response with this format:
+
+```js
+{
+   "error":"Some error string here"
+}
+```

docs/metrics.md

@@ -0,0 +1,50 @@
+Windshaft-cartodb metrics
+=========================
+See [Windshaft metrics documentation](https://github.com/CartoDB/Windshaft/blob/master/doc/metrics.md) to understand the full picture.
+
+The next list includes the API endpoints, each endpoint may have several inner timers, some of them are displayed within this list as subitems. Find the description for them in the Inner timers section.
+## Timers
+- **windshaft-cartodb.get_infowindow**: time to retrieve an infowindow popup
+- **windshaft-cartodb.get_map_metadata**: time to retrieve metadata for embedded maps
+- **windshaft-cartodb.flush_cache**: time to flush the tile and sql cache
+- **windshaft-cartodb.get_template**: time to retrieve an specific template
+- **windshaft-cartodb.delete_template**: time to delete an specific template
+- **windshaft-cartodb.get_template_list**: time to retrieve the list of owned templates
+- **windshaft-cartodb.instance_template_post**: time to create a template via HTTP POST
+- **windshaft-cartodb.instance_template_get**: time to create a template via HTTP GET
+    + TemplateMaps_instance
+    + createLayergroup
+
+There are some endpoints that are not being tracked:
+- Adding a template
+- Updating a template
+
+### Inner timers
+Again, each inner timer may have several inner timers.
+
+- **addCacheChannel**: time to add X-Cache-Channel header based on table last modifications
+- **LZMA decompress**: time to decompress request params with LZMA
+- **TemplateMaps_instance**: time to retrieve a map template instance, see *getTemplate* and *authorizedByCert*
+- **affectedTables**: time to check what are the affected tables for adding the cache channel, see *addCacheChannel*
+- **authorize**: time to authorize a request, see *authorizedByAPIKey*, *authorizedByCert*, *authorizedBySigner*
+- **authorizedByAPIKey**: time to authorize using an API KEY
+- **authorizedByCert**: time to authorize a request by a cert, see [signed map](https://github.com/CartoDB/Windshaft-cartodb/wiki/Signed-maps)
+- **authorizedBySigner**: time to authorize a request for a [signed map](https://github.com/CartoDB/Windshaft-cartodb/wiki/Signed-maps)
+- **cartoData.getTableGeometryType**: time to retrieve from redis the geom type for a given table
+- **cors**: time to set the CORS headers
+- **findLastUpdated**: time to retrieve the last update time for a list of tables, see *affectedTables*
+- **fingerPrint**: time to create a fingerprint for a signed map
+- **generateCacheChannel**: time to generate the headers for the cache channel based on the request, see *addCacheChannel*
+- **getSignerMapKey**: time to retrieve from redis the authorized key for a signed map
+- **getTablePrivacy**: time to retrieve from redis the privacy of a table
+- **getTemplate**: time to retrieve from redis the template for a map
+- **getUserMapKey**: time to retrieve from redis the user key for a map
+- **incMapviewCount**: time to incremenent in redis the map views
+- **mapStore_load**: time to retrieve from redis a map configuration
+- **req2params.setup**: time to prepare the params from a request, see *req2params* in Windshaft documentation
+- **setDBAuth**: time to retrieve from redis and set db user and db password from a user
+- **setDBConn**: time to retrieve from redis and set db host and db name from a user
+- **setDBParams**: time to prepare all db params to be able to connect/query a database, see *setDBAuth* and *setDBConn*
+- **signMap**: time to sign in redis layergroup for a map, see signed maps
+- **tablePrivacy_getUserDBName**: time to retrieve from redis the database for a user
+

lib/cartodb/cache_validator.js

@@ -2,16 +2,21 @@ var _             = require('underscore'),
     Varnish       = require('node-varnish'),
     varnish_queue = null;
 
-function init(host, port) {
-    varnish_queue = new Varnish.VarnishQueue(host, port);
+function init(host, port, secret) {
+    varnish_queue = new Varnish.VarnishQueue(host, port, secret);
+    varnish_queue.on('error', function(e) {
+        console.log("[CACHE VALIDATOR ERROR] " + e);
+    });
 }
 
 function invalidate_db(dbname, table) {
+    var cmd = 'purge obj.http.X-Cache-Channel ~ "^' + dbname +
+              ':(.*'+ table +'.*)|(table)$"';
     try{
-        varnish_queue.run_cmd('purge obj.http.X-Cache-Channel ~ "^' + dbname + ':(.*'+ table +'.*)|(table)$"');
-        console.log('[SUCCESS FLUSHING CACHE]');
+        varnish_queue.run_cmd(cmd, false);
     } catch (e) {
-        console.log("[ERROR FLUSHING CACHE] Is enable_cache set to true? Failed for: " + 'purge obj.http.X-Cache-Channel ~ "^' + dbname + ':(.*'+ table +'.*)|(table)$"');
+        console.log("[CACHE VALIDATOR ERROR] could not queue command " +
+                    cmd + " -- " + e);
     }
 }
 

lib/cartodb/cartodb_windshaft.js

@@ -7,14 +7,28 @@ var _ = require('underscore')
     , cartoData  = require('cartodb-redis')(global.environment.redis)
     , SignedMaps = require('./signed_maps.js')
     , TemplateMaps = require('./template_maps.js')
-    , Cache = require('./cache_validator');
+    , Cache = require('./cache_validator')
+    , os = require('os')
+;
+
+if ( ! process.env['PGAPPNAME'] )
+  process.env['PGAPPNAME']='cartodb_tiler';
 
 var CartodbWindshaft = function(serverOptions) {
-   var debug = global.environment.environment === 'development';
+   var debug = global.environment.debug;
+
+    // Perform keyword substitution in statsd
+    // See https://github.com/CartoDB/Windshaft-cartodb/issues/153
+    if ( global.environment.statsd ) {
+      if ( global.environment.statsd.prefix ) {
+        var host_token = os.hostname().split('.').reverse().join('.');
+        global.environment.statsd.prefix = global.environment.statsd.prefix.replace(/:host/, host_token);
+      }
+    }
 
     if(serverOptions.cache_enabled) {
         console.log("cache invalidation enabled, varnish on ", serverOptions.varnish_host, ' ', serverOptions.varnish_port);
-        Cache.init(serverOptions.varnish_host, serverOptions.varnish_port);
+        Cache.init(serverOptions.varnish_host, serverOptions.varnish_port, serverOptions.varnish_secret);
         serverOptions.afterStateChange = function(req, data, callback) {
             Cache.invalidate_db(req.params.dbname, req.params.table);
             callback(null, data);
@@ -29,8 +43,17 @@ var CartodbWindshaft = function(serverOptions) {
         callback(err, req);
     }
 
+    // This is for Templated maps
+    //
+    // "named" is the official, "template" is for backward compatibility up to 1.6.x
+    //
+    var template_baseurl = global.environment.base_url_templated || '(?:/maps/named|/tiles/template)';
+
     serverOptions.signedMaps = new SignedMaps(redisPool);
-    var templateMaps = new TemplateMaps(redisPool, serverOptions.signedMaps);
+    var templateMapsOpts = {
+      max_user_templates: global.environment.maxUserTemplates
+    }
+    var templateMaps = new TemplateMaps(redisPool, serverOptions.signedMaps, templateMapsOpts);
 
     // boot
     var ws = new Windshaft.Server(serverOptions);
@@ -43,19 +66,70 @@ var CartodbWindshaft = function(serverOptions) {
       return version;
     }
 
-    // Override sendError to drop added cache headers (if any)
-    // See http://github.com/CartoDB/Windshaft-cartodb/issues/107
+    var ws_sendResponse = ws.sendResponse;
+    // GET routes for which we don't want to request any caching.
+    // POST/PUT/DELETE requests are never cached anyway.
+    var noCacheGETRoutes = [
+      '/',
+      // See https://github.com/CartoDB/Windshaft-cartodb/issues/176
+      serverOptions.base_url_mapconfig,
+      template_baseurl + '/:template_id/jsonp'
+    ];
+    ws.sendResponse = function(res, args) {
+      var that = this;
+      var thatArgs = arguments;
+      var statusCode;
+      if ( res._windshaftStatusCode ) {
+        // Added by our override of sendError
+        statusCode = res._windshaftStatusCode;
+      } else {
+        if ( args.length > 2 ) statusCode = args[2];
+        else {
+          statusCode = args[1] || 200;
+        }
+      }
+      var req = res.req;
+      Step (
+        function addCacheChannel() {
+          if ( ! req ) {
+            // having no associated request can happen when
+            // using fake response objects for testing layergroup
+            // creation
+            return false;
+          }
+          if ( ! req.params ) {
+            // service requests (/version, /) 
+            // have no need for an X-Cache-Channel
+            return false;
+          }
+          if ( statusCode != 200 ) {
+            // We do not want to cache
+            // unsuccessful responses
+            return false;
+          }
+          if ( _.contains(noCacheGETRoutes, req.route.path) ) {
+//console.log("Skipping cache channel in route:\n" + req.route.path);
+            return false;
+          }
+//console.log("Adding cache channel to route\n" + req.route.path + " not matching any in:\n" + mapCreateRoutes.join("\n"));
+          serverOptions.addCacheChannel(that, req, this);
+        },
+        function sendResponse(err, added) {
+          if ( err ) console.log(err + err.stack);
+          ws_sendResponse.apply(that, thatArgs);
+          return null;
+        },
+        function finish(err) {
+          if ( err ) console.log(err + err.stack);
+        }
+      );
+    };
+
     var ws_sendError = ws.sendError;
-    ws.sendError = function(res) {
-      // NOTE: the "res" object will have no _headers when
-      //       faked by Windshaft, see
-      //       http://github.com/CartoDB/Windshaft-cartodb/issues/109
-      //
-      if ( res._headers ) {
-        delete res._headers['cache-control'];
-        delete res._headers['last-modified'];
-        delete res._headers['x-cache-channel'];
-      } 
+    ws.sendError = function() {
+      var res = arguments[0];
+      var statusCode = arguments[2];
+      res._windshaftStatusCode = statusCode;
       ws_sendError.apply(this, arguments);
     };
 
@@ -63,6 +137,9 @@ var CartodbWindshaft = function(serverOptions) {
      * Helper to allow access to the layer to be used in the maps infowindow popup.
      */
     ws.get(serverOptions.base_url + '/infowindow', function(req, res){
+        if ( req.profiler && req.profiler.statsd_client ) {
+          req.profiler.start('windshaft-cartodb.get_infowindow');
+        }
         ws.doCORS(res);
         Step(
             function(){
@@ -71,9 +148,9 @@ var CartodbWindshaft = function(serverOptions) {
             function(err, data){
                 if (err){
                     ws.sendError(res, {error: err.message}, 500, 'GET INFOWINDOW', err);
-                    //res.send({error: err.message}, 500);
+                    //ws.sendResponse(res, [{error: err.message}, 500]);
                 } else {
-                    res.send({infowindow: data}, 200);
+                    ws.sendResponse(res, [{infowindow: data}, 200]);
                 }
             }
         );
@@ -84,6 +161,9 @@ var CartodbWindshaft = function(serverOptions) {
      * Helper to allow access to metadata to be used in embedded maps.
      */
     ws.get(serverOptions.base_url + '/map_metadata', function(req, res){
+        if ( req.profiler && req.profiler.statsd_client ) {
+          req.profiler.start('windshaft-cartodb.get_map_metadata');
+        }
         ws.doCORS(res);
         Step(
             function(){
@@ -92,9 +172,9 @@ var CartodbWindshaft = function(serverOptions) {
             function(err, data){
                 if (err){
                     ws.sendError(res, {error: err.message}, 500, 'GET MAP_METADATA', err);
-                    //res.send(err.message, 500);
+                    //ws.sendResponse(res, [err.message, 500]);
                 } else {
-                    res.send({map_metadata: data}, 200);
+                    ws.sendResponse(res, [{map_metadata: data}, 200]);
                 }
             }
         );
@@ -105,6 +185,9 @@ var CartodbWindshaft = function(serverOptions) {
      * TODO: Move?
      */
     ws.del(serverOptions.base_url + '/flush_cache', function(req, res){
+        if ( req.profiler && req.profiler.statsd_client ) {
+          req.profiler.start('windshaft-cartodb.flush_cache');
+        }
         ws.doCORS(res);
         Step(
             function flushCache(){
@@ -113,9 +196,9 @@ var CartodbWindshaft = function(serverOptions) {
             function sendResponse(err, data){
                 if (err){
                     ws.sendError(res, {error: err.message}, 500, 'DELETE CACHE', err);
-                    //res.send(500);
+                    //ws.sendResponse(res, [500]);
                 } else {
-                    res.send({status: 'ok'}, 200);
+                    ws.sendResponse(res, [{status: 'ok'}, 200]);
                 }
             }
         );
@@ -127,12 +210,6 @@ var CartodbWindshaft = function(serverOptions) {
         return serverOptions.userByReq(req);
     }
 
-    // This is for Templated maps
-    //
-    // "named" is the official, "template" is for backward compatibility up to 1.6.x
-    //
-    var template_baseurl = global.environment.base_url_templated || '(?:/maps/named|/tiles/template)';
-
     // Add a template
     ws.post(template_baseurl, function(req, res) {
       ws.doCORS(res);
@@ -147,7 +224,7 @@ var CartodbWindshaft = function(serverOptions) {
           if ( err ) throw err;
           if (authenticated !== 1) {
             err = new Error("Only authenticated user can create templated maps");
-            err.http_status = 401;
+            err.http_status = 403;
             throw err;
           }
           var next = this;
@@ -162,6 +239,10 @@ var CartodbWindshaft = function(serverOptions) {
           return { template_id: cdbuser + '@' + tpl_id };
         },
         function finish(err, response){
+            if ( req.profiler ) {
+              var report = req.profiler.toString();
+              res.header('X-Tiler-Profiler', report);
+            }
             if (err){
                 response = { error: ''+err };
                 var statusCode = 400;
@@ -170,7 +251,7 @@ var CartodbWindshaft = function(serverOptions) {
                 }
                 ws.sendError(res, response, statusCode, 'POST TEMPLATE', err);
             } else {
-              res.send(response, 200);
+              ws.sendResponse(res, [response, 200]);
             }
         }
       );
@@ -192,7 +273,7 @@ var CartodbWindshaft = function(serverOptions) {
           if ( err ) throw err;
           if (authenticated !== 1) {
             err = new Error("Only authenticated user can list templated maps");
-            err.http_status = 401;
+            err.http_status = 403;
             throw err;
           }
           if ( ! req.headers['content-type'] || req.headers['content-type'].split(';')[0] != 'application/json' )
@@ -215,6 +296,10 @@ var CartodbWindshaft = function(serverOptions) {
           return { template_id: cdbuser + '@' + tpl_id };
         },
         function finish(err, response){
+            if ( req.profiler ) {
+              var report = req.profiler.toString();
+              res.header('X-Tiler-Profiler', report);
+            }
             if (err){
                 var statusCode = 400;
                 response = { error: ''+err };
@@ -223,7 +308,7 @@ var CartodbWindshaft = function(serverOptions) {
                 }
                 ws.sendError(res, response, statusCode, 'PUT TEMPLATE', err);
             } else {
-              res.send(response, 200);
+              ws.sendResponse(res, [response, 200]);
             }
         }
       );
@@ -231,6 +316,9 @@ var CartodbWindshaft = function(serverOptions) {
 
     // Get a specific template
     ws.get(template_baseurl + '/:template_id', function(req, res) {
+      if ( req.profiler && req.profiler.statsd_client ) {
+        req.profiler.start('windshaft-cartodb.get_template');
+      }
       ws.doCORS(res);
       var that = this;
       var response = {};
@@ -245,7 +333,7 @@ var CartodbWindshaft = function(serverOptions) {
           if ( err ) throw err;
           if (authenticated !== 1) {
             err = new Error("Only authenticated users can get template maps");
-            err.http_status = 401;
+            err.http_status = 403;
             throw err;
           }
           tpl_id = req.params.template_id.split('@');
@@ -281,14 +369,17 @@ var CartodbWindshaft = function(serverOptions) {
                 }
                 ws.sendError(res, response, statusCode, 'GET TEMPLATE', err);
             } else {
-              res.send(response, 200);
+              ws.sendResponse(res, [response, 200]);
             }
         }
       );
     });
 
     // Delete a specific template
-    ws.delete(template_baseurl + '/:template_id', function(req, res) {
+    ws.del(template_baseurl + '/:template_id', function(req, res) {
+      if ( req.profiler && req.profiler.statsd_client ) {
+        req.profiler.start('windshaft-cartodb.delete_template');
+      }
       ws.doCORS(res);
       var that = this;
       var response = {};
@@ -303,7 +394,7 @@ var CartodbWindshaft = function(serverOptions) {
           if ( err ) throw err;
           if (authenticated !== 1) {
             err = new Error("Only authenticated users can delete template maps");
-            err.http_status = 401;
+            err.http_status = 403;
             throw err;
           }
           tpl_id = req.params.template_id.split('@');
@@ -331,7 +422,7 @@ var CartodbWindshaft = function(serverOptions) {
                 }
                 ws.sendError(res, response, statusCode, 'DELETE TEMPLATE', err);
             } else {
-              res.send('', 204);
+              ws.sendResponse(res, ['', 204]);
             }
         }
       );
@@ -339,6 +430,9 @@ var CartodbWindshaft = function(serverOptions) {
 
     // Get a list of owned templates 
     ws.get(template_baseurl, function(req, res) {
+      if ( req.profiler && req.profiler.statsd_client ) {
+        req.profiler.start('windshaft-cartodb.get_template_list');
+      }
       ws.doCORS(res);
       var that = this;
       var response = {};
@@ -351,7 +445,7 @@ var CartodbWindshaft = function(serverOptions) {
           if ( err ) throw err;
           if (authenticated !== 1) {
             err = new Error("Only authenticated user can list templated maps");
-            err.http_status = 401;
+            err.http_status = 403;
             throw err;
           }
           templateMaps.listTemplates(cdbuser, this);
@@ -371,7 +465,7 @@ var CartodbWindshaft = function(serverOptions) {
                 }
                 ws.sendError(res, response, statusCode, 'GET TEMPLATE LIST', err);
             } else {
-              res.send(response, statusCode);
+              ws.sendResponse(res, [response, statusCode]);
             }
         }
       );
@@ -400,6 +494,7 @@ var CartodbWindshaft = function(serverOptions) {
     // Instantiate a template
     function instanciateTemplate(req, res, template_params, callback) {
       ws.doCORS(res);
+      if ( req.profiler ) req.profiler.done('cors');
       var that = this;
       var response = {};
       var template;
@@ -411,7 +506,14 @@ var CartodbWindshaft = function(serverOptions) {
       // Format of template_id: [<template_owner>]@<template_id>
       var tpl_id = req.params.template_id.split('@');
       if ( tpl_id.length > 1 ) {
-        if ( tpl_id[0] ) cdbuser = tpl_id[0];
+        if ( tpl_id[0] && tpl_id[0] != cdbuser ) {
+          var err = new Error('Cannot instanciate map of user "'
+                              + tpl_id[0] + '" on database of user "'
+                              + cdbuser + '"')
+          err.http_status = 403;
+          callback(err);
+          return;
+        }
         tpl_id = tpl_id[1];
       }
       var auth_token = req.query.auth_token;
@@ -420,6 +522,7 @@ var CartodbWindshaft = function(serverOptions) {
           templateMaps.getTemplate(cdbuser, tpl_id, this);
         },
         function checkAuthorized(err, data) {
+          if ( req.profiler ) req.profiler.done('getTemplate');
           if ( err ) throw err;
           if ( ! data ) {
             err = new Error("Template '" + tpl_id + "' of user '" + cdbuser + "' not found");
@@ -434,36 +537,46 @@ var CartodbWindshaft = function(serverOptions) {
             authorized = signedMaps.authorizedByCert(cert, auth_token);
           } catch (err) {
             // we catch to add http_status
-            err.http_status = 401;
+            err.http_status = 403;
             throw err;
           }
           if ( ! authorized ) {
             err = new Error('Unauthorized template instanciation');
-            err.http_status = 401;
+            err.http_status = 403;
             throw err;
           }
           /*if ( (! req.headers['content-type'] || req.headers['content-type'].split(';')[0] != 'application/json') && req.query.callback === undefined) {
             throw new Error('template POST data must be of type application/json, it is instead ');
           }*/
           //var template_params = req.body;
+          if ( req.profiler ) req.profiler.done('authorizedByCert');
           return templateMaps.instance(template, template_params);
         },
         function prepareParams(err, instance){
+          if ( req.profiler ) req.profiler.done('TemplateMaps_instance');
           if ( err ) throw err;
           layergroup = instance;
-          fakereq = { query: {}, params: {}, headers: _.clone(req.headers) };
+          fakereq = { query: {}, params: {}, headers: _.clone(req.headers),
+            method: req.method,
+            res: res,
+            profiler: req.profiler
+          };
           ws.setDBParams(cdbuser, fakereq.params, this);
         },
         function setApiKey(err){
+          if ( req.profiler ) req.profiler.done('setDBParams');
           if ( err ) throw err;
           cartoData.getUserMapKey(cdbuser, this);
         },
         function createLayergroup(err, val) {
+          if ( req.profiler ) req.profiler.done('getUserMapKey');
           if ( err ) throw err;
           fakereq.params.api_key = val;
           ws.createLayergroup(layergroup, fakereq, this);
         },
         function signLayergroup(err, resp) {
+          // NOTE: createLayergroup uses profiler.start()/end() internally
+          //if ( req.profiler ) req.profiler.done('createLayergroup');
           if ( err ) throw err;
           response = resp;
           var signer = cdbuser;
@@ -481,10 +594,12 @@ var CartodbWindshaft = function(serverOptions) {
           signedMaps.signMap(signer, map_id, crt_id, this);
         },
         function prepareResponse(err) {
+          if ( req.profiler ) req.profiler.done('signMap');
           if ( err ) throw err;
           //console.log("Response from createLayergroup: "); console.dir(response);
           // Add the signature part to the token!
           var tplhash = templateMaps.fingerPrint(template).substring(0,8);
+          if ( req.profiler ) req.profiler.done('fingerPrint');
           response.layergroupid = cdbuser + '@' + tplhash + '@' + response.layergroupid;
           return response;
         },
@@ -492,7 +607,11 @@ var CartodbWindshaft = function(serverOptions) {
       );
     }
 
-    function finish_instanciation(err, response, res) {
+    function finish_instanciation(err, response, res, req) {
+        if ( req.profiler ) {
+          var report = req.profiler.toString();
+          res.header('X-Tiler-Profiler', report);
+        }
         if (err) {
             var statusCode = 400;
             response = { error: ''+err };
@@ -504,11 +623,14 @@ var CartodbWindshaft = function(serverOptions) {
             }
             ws.sendError(res, response, statusCode, 'POST INSTANCE TEMPLATE', err);
         } else {
-          res.send(response, 200);
+          ws.sendResponse(res, [response, 200]);
         }
     }
 
     ws.post(template_baseurl + '/:template_id', function(req, res) {
+      if ( req.profiler && req.profiler.statsd_client) {
+        req.profiler.start('windshaft-cartodb.instance_template_post');
+      }
       Step(
         function() {
           if ( ! req.headers['content-type'] || req.headers['content-type'].split(';')[0] != 'application/json') {
@@ -516,7 +638,7 @@ var CartodbWindshaft = function(serverOptions) {
           }
           instanciateTemplate(req, res, req.body, this);
         }, function(err, response) {
-          finish_instanciation(err, response, res);
+          finish_instanciation(err, response, res, req);
         }
       );
     });
@@ -526,6 +648,9 @@ var CartodbWindshaft = function(serverOptions) {
      * callback query argument is mandartoy
      */
     ws.get(template_baseurl + '/:template_id/jsonp', function(req, res) {
+      if ( req.profiler && req.profiler.statsd_client) {
+        req.profiler.start('windshaft-cartodb.instance_template_get');
+      }
       Step(
         function() {
           if ( req.query.callback === undefined || req.query.callback.length === 0) {
@@ -541,7 +666,7 @@ var CartodbWindshaft = function(serverOptions) {
           }
           instanciateTemplate(req, res, config, this);
         }, function(err, response) {
-          finish_instanciation(err, response, res);
+          finish_instanciation(err, response, res, req);
         }
       );
     });

lib/cartodb/log4js_rollbar.js

@@ -0,0 +1,49 @@
+var rollbar = require("rollbar");
+
+/**
+ * Rollbar Appender. Sends logging events to Rollbar using node-rollbar 
+ *
+ * @param config object with rollbar configuration data
+ * {
+ *   token: 'your-secret-token',
+ *   options: node-rollbar options
+ * }
+ */
+function rollbarAppender(config) {
+
+  var opt = config.options;
+	rollbar.init(opt.token, opt.options);
+	
+	return function(loggingEvent) {
+/*
+For logger.trace('one','two','three'):
+{ startTime: Wed Mar 12 2014 16:27:40 GMT+0100 (CET),
+  categoryName: '[default]',
+  data: [ 'one', 'two', 'three' ],
+  level: { level: 5000, levelStr: 'TRACE' },
+  logger: { category: '[default]', _events: { log: [Object] } } }
+*/
+
+    // Levels:
+    // TRACE  5000
+    // DEBUG 10000
+    // INFO  20000
+    // WARN  30000
+    // ERROR 40000
+    // FATAL 50000
+    //
+    // We only log error and higher errors
+    //
+    if ( loggingEvent.level.level < 40000 ) return;
+
+    rollbar.reportMessage(loggingEvent.data);
+	};
+}
+
+function configure(config) {
+	return rollbarAppender(config);
+}
+
+exports.name      = "rollbar";
+exports.appender  = rollbarAppender;
+exports.configure = configure;

lib/cartodb/server_options.js

@@ -47,13 +47,14 @@ module.exports = function(){
           datasource: global.environment.postgres,
           cachedir: global.environment.millstone.cache_basedir,
           mapnik_version: global.environment.mapnik_version || mapnik.versions.mapnik,
-          default_layergroup_ttl: 7200, // seconds (default is 300)
-          gc_prob: 0.01 // default is 0.01 TODO: make configurable via env config
+          default_layergroup_ttl: global.environment.mapConfigTTL || 7200, 
+          gc_prob: 0.01 // @deprecated since Windshaft-1.8.0
         },
         mapnik: {
           metatile: rendererConfig.metatile,
           bufferSize: rendererConfig.bufferSize
         },
+        statsd: global.environment.statsd,
         renderCache: {
           ttl: rendererConfig.cache_ttl
         },
@@ -61,10 +62,15 @@ module.exports = function(){
         enable_cors: global.environment.enable_cors,
         varnish_host: global.environment.varnish.host,
         varnish_port: global.environment.varnish.port,
+        varnish_secret: global.environment.varnish.secret,
         cache_enabled: global.environment.cache_enabled,
         log_format: global.environment.log_format,
         useProfiler: global.environment.useProfiler
     };
+    
+    // Do not send unwatch on release
+    // See http://github.com/CartoDB/Windshaft-cartodb/issues/161
+    me.redis.unwatchOnRelease = false;
 
     // Be nice and warn if configured mapnik version
     // is != instaled mapnik version
@@ -103,16 +109,35 @@ module.exports = function(){
       // call sql api
       //
       // NOTE: using POST to avoid size limits:
-      // Seehttp://github.com/CartoDB/Windshaft-cartodb/issues/111
+      // See http://github.com/CartoDB/Windshaft-cartodb/issues/111
       //
-      // TODO: use "host" header to allow IP based specification
+      // NOTE: uses "host" header to allow IP based specification
       //       of sqlapi address (and avoid a DNS lookup)
       //
-      request.post({
-          url:sqlapi, body:qs, json:true,
-          headers:{host: sqlapihostname}
-        }, function(err, res, body)
-      {
+      // NOTE: allows for keeping up to "maxConnections" concurrent
+      //       sockets opened per SQL-API host.
+      // See http://nodejs.org/api/http.html#http_agent_maxsockets
+      //
+      var maxSockets = global.environment.maxConnections || 128;
+      var maxGetLen = api.max_get_sql_length || 2048;
+      var maxSQLTime = api.timeout || 100; // 1/10 of a second by default
+      var reqSpec = {
+            url:sqlapi,
+            json:true,
+            headers:{host: sqlapihostname}
+            // http://nodejs.org/api/http.html#http_agent_maxsockets
+            ,pool:{maxSockets:maxSockets}
+            // timeout in milliseconds
+            ,timeout:maxSQLTime
+      }
+      if ( sql.length > maxGetLen ) {
+        reqSpec.method = 'POST';
+        reqSpec.body = qs;
+      } else {
+        reqSpec.method = 'GET';
+        reqSpec.qs = qs;
+      }
+      request(reqSpec, function(err, res, body) {
           if (err){
             console.log('ERROR connecting to SQL API on ' + sqlapi + ': ' + err);
             callback(err);
@@ -172,7 +197,7 @@ module.exports = function(){
           }
           var qtables = rows[0].cdb_querytables;
           var tableNames = qtables.split(/^\{(.*)\}$/)[1];
-          tableNames = tableNames.split(',');
+          tableNames = tableNames ? tableNames.split(',') : [];
           callback(null, tableNames);
       });
     };
@@ -187,53 +212,124 @@ module.exports = function(){
         return hash.digest('hex');
     }
 
-    me.generateCacheChannel = function(req, callback){
-
-      // use key to call sql api with sql request if present, else
-      // just return dbname and table name base key
-      var dbName     = req.params.dbname;
+    me.generateCacheChannel = function(app, req, callback){
 
+      // Build channelCache key
+      var dbName = req.params.dbname;
       var cacheKey = [ dbName ];
       if ( req.params.token ) cacheKey.push(req.params.token);
       else if ( req.params.sql ) cacheKey.push( me.generateMD5(req.params.sql) );
       cacheKey = cacheKey.join(':');
 
-      if ( me.channelCache.hasOwnProperty(cacheKey) ) {
-        callback(null, me.channelCache[cacheKey]);
-        return;
-      }
-      else if ( req.params.token ) {
-        // cached cache channel for token-based access should be constructed
-        // at cache creation time
-        callback(new Error('missing channel cache for token ' + req.params.token));
-        return;
-      }
+      var that = this;
 
-      if ( ! req.params.sql && ! req.params.token ) {
-        var cacheChannel = me.buildCacheChannel(dbName, [req.params.table]);
-        // not worth caching this
-        callback(null, cacheChannel);
-        return;
-      }
+      Step (
+        function checkCached() {
+          if ( me.channelCache.hasOwnProperty(cacheKey) ) {
+            callback(null, me.channelCache[cacheKey]);
+            return;
+          }
+          return null;
+        },
+        function extractSQL(err) {
+          if ( err ) throw err;
 
-      if ( ! req.params.sql ) {
-        callback(new Error("this request doesn't need an X-Cache-Channel generated"));
-        return;
-      }
+          if ( req.params.token ) {
+            // TODO: cached cache channel for token-based access should
+            //       be constructed at renderer cache creation time
+            // See http://github.com/CartoDB/Windshaft-cartodb/issues/152
+            if ( ! app.mapStore ) {
+              throw new Error('missing channel cache for token ' + req.params.token);
+              return;
+            }
+            var next = this;
+            var mapStore = app.mapStore;
+            Step(
+              function loadFromStore() {
+                mapStore.load(req.params.token, this);
+              },
+              function getSQL(err, mapConfig) {
+                if (req.profiler) req.profiler.done('mapStore_load');
+                if ( err ) throw err;
+                var sql = [];
+                _.each(mapConfig.obj().layers, function(lyr) {
+                  sql.push(lyr.options.sql);
+                });
+                sql = sql.join(';');
+                return sql;
+              },
+              function finish(err, sql) {
+                next(err, sql);
+              }
+            );
+            return;
+          }
 
-      var dbName     = req.params.dbname;
-      var username   = this.userByReq(req);
+          if ( ! req.params.sql ) {
+            return null; // no sql
+          }
 
-      // strip out windshaft/mapnik inserted sql if present
-      var sql = req.params.sql.match(/^\((.*)\)\sas\scdbq$/);
-      sql = (sql != null) ? sql[1] : req.params.sql;
+          // We have sql, and no token...
 
-      me.affectedTables(username, req.params.map_key, sql, function(err, tableNames) {
-          if ( err ) { callback(err); return; }
+          // strip out windshaft/mapnik inserted sql if present
+          var sql = req.params.sql.match(/^\((.*)\)\sas\scdbq$/);
+          sql = (sql != null) ? sql[1] : req.params.sql;
+
+          return sql;
+        },
+        function findAffectedTables(err, sql) {
+          if ( err ) throw err;
+          if ( ! sql ) {
+            if ( ! req.params.table ) {
+              throw new Error("this request doesn't need an X-Cache-Channel generated");
+            }
+            return [req.params.table];
+          }
+          var user, key;
+          var next = this;
+          Step (
+            function findUserKey() {
+              if ( req.params.hasOwnProperty('_authorizedBySigner') ) {
+                user = req.params._authorizedBySigner;
+                cartoData.getUserMapKey(user, this);
+              } else {
+                user = that.userByReq(req);
+                key = req.params.map_key || req.params.api_key;
+                return null;
+              }
+            },
+            function getAffected(err, data) {
+              if ( err ) throw err;
+              if ( data ) {
+                if ( req.profiler ) req.profiler.done('getSignerMapKey');
+                key = data;
+              }
+              me.affectedTables(user, key, sql, this); // in addCacheChannel
+            },
+            function finish(err, data) {
+              next(err,data);
+            }
+          );
+        },
+        function buildCacheChannel(err, tableNames) {
+          if ( err ) throw err;
+          if (req.profiler && ! req.params.table ) {
+            req.profiler.done('affectedTables');
+          }
+
+          var dbName     = req.params.dbname;
           var cacheChannel = me.buildCacheChannel(dbName,tableNames);
-          me.channelCache[cacheKey] = cacheChannel; // store for caching
-          callback(null, cacheChannel);
-      });
+          // store for caching from me.generateCacheChannel
+          // (not worth when table was specified in params)
+          if ( ! req.params.table ) {
+            me.channelCache[cacheKey] = cacheChannel;
+          }
+          return cacheChannel;
+        },
+        function finish(err, cacheChannel) {
+          callback(err, cacheChannel);
+        }
+      );
     };
 
     // Set the cache chanel info to invalidate the cache on the frontend server
@@ -245,9 +341,10 @@ module.exports = function(){
     // @param cb function(err, channel) will be called when ready.
     //           the channel parameter will be null if nothing was added
     //
-    me.addCacheChannel = function(req, cb) {
+    me.addCacheChannel = function(app, req, cb) {
         // skip non-GET requests, or requests for which there's no response
         if ( req.method != 'GET' || ! req.res ) { cb(null, null); return; }
+        if (req.profiler) req.profiler.start('addCacheChannel');
         var res = req.res;
         var cache_policy = req.query.cache_policy;
         if ( req.params.token ) cache_policy = 'persist';
@@ -269,7 +366,9 @@ module.exports = function(){
         }
         res.header('Last-Modified', lastUpdated.toUTCString());
 
-        me.generateCacheChannel(req, function(err, channel){
+        me.generateCacheChannel(app, req, function(err, channel){
+            if (req.profiler) req.profiler.done('generateCacheChannel');
+            if (req.profiler) req.profiler.end();
             if ( ! err ) {
               res.header('X-Cache-Channel', channel);
               cb(null, channel);
@@ -299,6 +398,14 @@ module.exports = function(){
           }
         }
 
+        // include in layergroup response the variables in serverMedata
+        // those variables are useful to send to the client information
+        // about how to reach this server or information about it
+        var serverMetadata = global.environment.serverMetadata;
+        if (serverMetadata) {
+          _.extend(response, serverMetadata);
+        }
+
         // Don't wait for the mapview count increment to
         // take place before proceeding. Error will be logged
         // asyncronously
@@ -319,22 +426,45 @@ module.exports = function(){
         var key    = req.params.map_key || req.params.api_key;
 
         var cacheKey = dbName + ':' + token;
+        var tabNames;
 
-        me.affectedTables(usr, key, sql, function(err, tableNames) {
+        Step(
+          function getTables() {
+            me.affectedTables(usr, key, sql, this); // in afterLayergroupCreate
+          },
+          function getLastupdated(err, tableNames) {
             if (req.profiler) req.profiler.done('affectedTables');
-
-            if ( err ) { done(err); return; }
+            if ( err ) throw err;
             var cacheChannel = me.buildCacheChannel(dbName,tableNames);
-            me.channelCache[cacheKey] = cacheChannel; // store for caching
+            // store for caching from me.afterLayergroupCreate
+            me.channelCache[cacheKey] = cacheChannel;
+            if (req.res && req.method == 'GET') {
+                var res = req.res;
+                if ( req.query && req.query.cache_policy == 'persist' ) {
+                    res.header('Cache-Control', 'public,max-age=31536000'); // 1 year
+                } else {
+                    var ttl = global.environment.varnish.ttl || 86400;
+                    res.header('Cache-Control', 'public,max-age='+ttl+',must-revalidate');
+                }
+                res.header('Last-Modified', (new Date()).toUTCString());
+                res.header('X-Cache-Channel', cacheChannel);
+            }
             // find last updated
-            me.findLastUpdated(usr, key, tableNames, function(err, lastUpdated) {
-              if (req.profiler) req.profiler.done('findLastUpdated');
-              if ( err ) { done(err); return; }
-              response.layergroupid = response.layergroupid + ':' + lastUpdated; // use epoch
-              response.last_updated = new Date(lastUpdated).toISOString(); // TODO: use ISO format
-              done(null);
-            });
-        });
+            if ( ! tableNames.length ) return 0; // skip for no affected tables
+            tabNames = tableNames;
+            me.findLastUpdated(usr, key, tableNames, this);
+          },
+          function(err, lastUpdated) {
+            if ( err ) throw err;
+            if (req.profiler && tabNames) req.profiler.done('findLastUpdated');
+            response.layergroupid = response.layergroupid + ':' + lastUpdated; // use epoch
+            response.last_updated = new Date(lastUpdated).toISOString(); 
+            return null;
+          },
+          function finish(err) {
+            done(err);
+          }
+        );
     };
 
 /* X-Cache-Channel generation } */
@@ -419,6 +549,14 @@ module.exports = function(){
     // @param callback function(err) 
     //
     me.setDBConn = function(dbowner, params, callback) {
+      // Add default database connection parameters
+      // if none given
+      _.defaults(params, {
+        dbuser: global.environment.postgres.user,
+        dbpassword: global.environment.postgres.password,
+        dbhost: global.environment.postgres.host,
+        dbport: global.environment.postgres.port
+      });
       Step(
         function getDatabaseHost(){
             cartoData.getUserDBHost(dbowner, this);
@@ -463,7 +601,7 @@ module.exports = function(){
       var layergroup_id = req.params.token;
       var auth_token = req.params.auth_token;
 
-console.log("Checking authorization from signer " + signer + " for resource " + layergroup_id + " with auth_token " + auth_token);
+      //console.log("Checking authorization from signer " + signer + " for resource " + layergroup_id + " with auth_token " + auth_token);
 
       me.signedMaps.isAuthorized(signer, layergroup_id, auth_token,
         function(err, authorized) {
@@ -475,9 +613,20 @@ console.log("Checking authorization from signer " + signer + " for resource " +
     //
     // @param req express request object
     // @param callback function(err, authorized) 
+    //        NOTE: authorized is expected to be 0 or 1 (integer)
     //                 
     me.authorizedByAPIKey = function(req, callback)
     {
+        var givenKey = req.query.api_key || req.query.map_key;
+        if ( ! givenKey && req.body ) {
+          // check also in request body
+          givenKey = req.body.api_key || req.body.map_key;
+        }
+        if ( ! givenKey ) {
+          callback(null, 0); // no api key, no authorization...
+          return;
+        }
+        //console.log("given ApiKey: " + givenKey);
         var user = me.userByReq(req);
         Step(
           function (){
@@ -485,16 +634,7 @@ console.log("Checking authorization from signer " + signer + " for resource " +
           },
           function checkApiKey(err, val){
               if (err) throw err;
-
-              var valid = 0;
-              if ( val ) {
-                if ( val == req.query.map_key ) valid = 1;
-                else if ( val == req.query.api_key ) valid = 1;
-                // check also in request body
-                else if ( req.body && req.body.map_key && val == req.body.map_key ) valid = 1;
-                else if ( req.body && req.body.api_key && val == req.body.api_key ) valid = 1;
-              }
-              return valid;
+              return ( val && givenKey == val ) ? 1 : 0;
           },
           function finish(err, authorized) {
               callback(err, authorized);
@@ -517,6 +657,7 @@ console.log("Checking authorization from signer " + signer + " for resource " +
                 that.authorizedByAPIKey(req, this);
             },
             function checkApiKey(err, authorized){
+                if (req.profiler) req.profiler.done('authorizedByAPIKey');
                 if (err) throw err;
 
                 // if not authorized by api_key, continue 
@@ -536,15 +677,37 @@ console.log("Checking authorization from signer " + signer + " for resource " +
             },
             function checkSignAuthorized(err, signed_by){
                 if (err) throw err;
+                if (req.profiler) {
+                  if ( req.params._authorizedByApiKey ) {
+                    req.profiler.done('setDBAuth');
+                  } else {
+                    req.profiler.done('authorizedBySigner');
+                  }
+                }
 
                 if ( ! signed_by ) {
-                  // request not authorized by signer, continue
-                  // to check map privacy
-                  return null;
+                  // request not authorized by signer.
+  
+                  // if table was given, continue to check table privacy
+                  if ( req.params.table ) return null;
+
+                  // if no signer name was given, let dbparams and
+                  // PostgreSQL do the rest.
+                  // 
+                  if ( ! req.params.signer ) {
+                    callback(null, true); // authorized so far
+                    return;
+                  }
+
+                  // if signer name was given, return no authorization
+                  callback(null, false); 
+                  return;
                 }
 
                 // Authorized by "signed_by" !
+                _.extend(req.params, { _authorizedBySigner: signed_by });
                 that.setDBAuth(signed_by, req.params, function(err) {
+                  if (req.profiler) req.profiler.done('setDBAuth');
                   callback(err, true); // authorized (or error)
                 });
             },
@@ -555,10 +718,12 @@ console.log("Checking authorization from signer " + signer + " for resource " +
             },
             function getPrivacy(err, dbname){
                 if (err) throw err;
+                if (req.profiler) req.profiler.done('tablePrivacy_getUserDBName');
                 cartoData.getTablePrivacy(dbname, req.params.table, this);
             },
             function(err, privacy){
-                callback(err, privacy);
+                if (req.profiler) req.profiler.done('getTablePrivacy');
+                callback(err, privacy !== "0");
             }
         );
     };
@@ -606,6 +771,8 @@ console.log("Checking authorization from signer " + signer + " for resource " +
         _.each(bad_query, function(key){ delete req.query[key]; });
         req.params =  _.extend({}, req.params); // shuffle things as request is a strange array/object
 
+        var user = me.userByReq(req);
+
         if ( req.params.token ) {
           //console.log("Request parameters include token " + req.params.token);
           var tksplit = req.params.token.split(':');
@@ -614,7 +781,13 @@ console.log("Checking authorization from signer " + signer + " for resource " +
           tksplit = req.params.token.split('@');
           if ( tksplit.length > 1 ) {
             req.params.signer = tksplit.shift();
-            if ( ! req.params.signer ) req.params.signer = this.userByReq(req);
+            if ( ! req.params.signer ) req.params.signer = user;
+            else if ( req.params.signer != user ) {
+              var err = new Error('Cannot use map signature of user "' + req.params.signer + '" on database of user "' + user + '"')
+              err.http_status = 403;
+              callback(err);
+              return;
+            }
             if ( tksplit.length > 1 ) {
               var template_hash = tksplit.shift(); // unused
             }
@@ -633,25 +806,28 @@ console.log("Checking authorization from signer " + signer + " for resource " +
 
         if (req.profiler) req.profiler.done('req2params.setup');
 
-        var user = me.userByReq(req);
-
         Step(
             function getPrivacy(){
                 me.authorize(req, this);
             },
-            function gatekeep(err, data){
+            function gatekeep(err, authorized){
                 if (req.profiler) req.profiler.done('authorize');
                 if(err) throw err;
-                if(data === "0") throw new Error("Sorry, you are unauthorized (permission denied)");
-                return data;
+                if(!authorized) {
+                  err = new Error("Sorry, you are unauthorized (permission denied)");
+                  err.http_status = 403;
+                  throw err;
+                }
+                return null;
             },
-            function getDatabase(err, data){
+            function getDatabase(err){
                 if(err) throw err;
                 that.setDBConn(user, req.params, this);
             },
             function getGeometryType(err){
                 if (req.profiler) req.profiler.done('setDBConn');
                 if (err) throw err;
+                if ( ! req.params.table ) return null;
                 cartoData.getTableGeometryType(req.params.dbname, req.params.table, this);
             },
             function finishSetup(err, data){
@@ -670,10 +846,7 @@ console.log("Checking authorization from signer " + signer + " for resource " +
                   dbport: global.environment.postgres.port
                 });
 
-                that.addCacheChannel(req, function(err) {
-                  if (req.profiler) req.profiler.done('addCacheChannel');
-                  callback(err, req);
-                });
+                callback(null, req);
             }
         );
     };
@@ -763,4 +936,4 @@ console.log("Checking authorization from signer " + signer + " for resource " +
     };
 
     return me;
-}();
+};

lib/cartodb/signed_maps.js

@@ -2,6 +2,7 @@ var crypto    = require('crypto');
 var Step      = require('step');
 var _         = require('underscore');
 
+var debug = global.environment ? global.environment.debug : undefined;
 
 // Class handling map signatures and user certificates
 //
@@ -161,18 +162,30 @@ o.authorizedByCert = function(cert, auth) {
 //
 o.isAuthorized = function(signer, map_id, auth, callback) {
   var that = this;
+  var redisClient;
+  var db = that.db_signatures;
   var authorized = false;
   var certificate_id_list;
   var missing_certificates = [];
-console.log("Check auth from signer '" + signer + "' on map '" + map_id + "' with auth '" + auth + "'");
+  if ( debug ) {
+    console.log("Check auth from signer '" + signer + "' on map '" + map_id + "' with auth '" + auth + "'");
+  }
   Step(
-    function getMapSignatures() {
+    function getRedisClient() {
+      that.redis_pool.acquire(db, this);
+    },
+    function getMapSignatures(err, client) {
+      if ( err ) throw err;
+      redisClient = client;
       var map_sig_key = _.template(that.key_map_sig, {signer:signer, map_id:map_id});
-      that._redisCmd('SMEMBERS', [ map_sig_key ], this);
+      redisClient.SMEMBERS(map_sig_key, this);
+      //that._redisCmd('SMEMBERS', [ map_sig_key ], this);
     },
     function getCertificates(err, crt_lst) {
       if ( err ) throw err;
-      console.log("Map '" + map_id + "' is signed by " + crt_lst.length + " certificates of user '" + signer + "': " + crt_lst);
+      if ( debug ) {
+        console.log("Map '" + map_id + "' is signed by " + crt_lst.length + " certificates of user '" + signer);
+      }
       certificate_id_list = crt_lst;
       if ( ! crt_lst.length ) {
         // No certs, avoid calling redis with short args list.
@@ -181,7 +194,8 @@ console.log("Check auth from signer '" + signer + "' on map '" + map_id + "' wit
         return crt_lst;
       }
       var map_crt_key = _.template(that.key_map_crt, {signer:signer});
-      that._redisCmd('HMGET', [ map_crt_key ].concat(crt_lst), this);
+      //that._redisCmd('HMGET', [ map_crt_key ].concat(crt_lst), this);
+      redisClient.HMGET(map_crt_key, crt_lst, this);
     },
     function checkCertificates(err, certs) {
       if ( err ) throw err;
@@ -201,8 +215,10 @@ console.log("Check auth from signer '" + signer + "' on map '" + map_id + "' wit
           continue;
         }
         if ( authorized ) {
-          console.log("Access to map '" + map_id + "' authorized by cert '"
-            + certificate_id_list[i] + "' of user '" + signer + "'");
+          if ( debug ) {
+            console.log("Access to map '" + map_id + "' authorized by cert '"
+                        + certificate_id_list[i] + "' of user '" + signer + "'");
+          }
           //console.dir(cert);
           break; // no need to further check certs
         }
@@ -216,6 +232,7 @@ console.log("Check auth from signer '" + signer + "' on map '" + map_id + "' wit
           + " missing certificates: "
           + missing_certificates + " (TODO: give cleanup instructions)");
       }
+      if ( redisClient ) that.redis_pool.release(db, redisClient);
       callback(err, authorized);
     }
   );
@@ -279,8 +296,10 @@ o.delCertificate = function(signer, crt_id, callback) {
     function delMapSignaturesReference(err, map_id_list) {
       if ( err ) throw err;
       signed_map_list = map_id_list;
-      console.log("Certificate '" + crt_id + "' from user '" + signer
-        + "' was used to sign " + signed_map_list.length + " maps");
+      if ( debug ) {
+        console.log("Certificate '" + crt_id + "' from user '" + signer
+          + "' was used to sign " + signed_map_list.length + " maps");
+      }
       redis_client.DEL(crt_sig_key, this);
     },
     function delMapSignatures(err) {
@@ -297,16 +316,18 @@ o.delCertificate = function(signer, crt_id, callback) {
     },
     function reportTransaction(err, rets) {
       if ( err ) throw err;
-      for (var i=0; i<signed_map_list.length; ++i) {
-        var ret = rets[i];
-        if ( ! ret ) {
-          console.log("No signature with certificate '" + crt_id
-            + "' of user '" + signer + "' found in map '"
-            + signed_map_list[i] + "'");
-        } else {
-          console.log("Signature with certificate '" + crt_id
-            + "' of user '" + signer + "' removed from map '"
-            + signed_map_list[i] + "'");
+      if ( debug ) {
+        for (var i=0; i<signed_map_list.length; ++i) {
+          var ret = rets[i];
+          if ( ! ret ) {
+            console.log("No signature with certificate '" + crt_id
+              + "' of user '" + signer + "' found in map '"
+              + signed_map_list[i] + "'");
+          } else {
+            console.log("Signature with certificate '" + crt_id
+              + "' of user '" + signer + "' removed from map '"
+              + signed_map_list[i] + "'");
+          }
         }
       }
       return null;
@@ -333,7 +354,9 @@ o.signMap = function(signer, map_id, crt_id, callback) {
   Step(
     function addMapSignature() {
       var map_sig_key = _.template(that.key_map_sig, {signer:signer, map_id:map_id});
-console.log("Adding " + crt_id + " to " + map_sig_key);
+      if ( debug ) {
+        console.log("Adding " + crt_id + " to " + map_sig_key);
+      }
       that._redisCmd('SADD', [ map_sig_key, crt_id ], this);
     },
     function addCertificateUsage(err) {

lib/cartodb/template_maps.js

@@ -16,10 +16,15 @@ var user_template_locks = {};
 //
 // @param signed_maps an instance of a "signed_maps" class,
 //        See signed_maps.js
+//
+// @param opts TemplateMap options. Supported elements:
+//    'max_user_templates' limit on the number of per-user
+//
 // 
-function TemplateMaps(redis_pool, signed_maps) {
+function TemplateMaps(redis_pool, signed_maps, opts) {
   this.redis_pool = redis_pool;
   this.signed_maps = signed_maps;
+  this.opts = opts || {};
 
   // Database containing templates
   // TODO: allow configuring ?
@@ -49,6 +54,10 @@ var o = TemplateMaps.prototype;
 
 //--------------- PRIVATE METHODS --------------------------------
 
+o._userTemplateLimit = function() {
+  return this.opts['max_user_templates'] || 0;
+};
+
 o._acquireRedis = function(callback) {
   this.redis_pool.acquire(this.db_signatures, callback);
 };
@@ -189,6 +198,7 @@ o.addTemplate = function(owner, template, callback) {
 
   // Procedure:
   //
+  // - Check against limit
   // 0. Obtain a lock for user+template_name, fail if impossible
   // 1. Check no other template exists with the same name
   // 2. Install certificate extracted from template, extending
@@ -202,9 +212,18 @@ o.addTemplate = function(owner, template, callback) {
   var usr_tpl_key = _.template(this.key_usr_tpl, {owner:owner});
   var gotLock = false;
   var that = this;
+  var limit = that._userTemplateLimit();
   Step(
+    function checkLimit() {
+      if ( ! limit ) return 0;
+      that._redisCmd('HLEN', [ usr_tpl_key ], this);
+    },
     // try to obtain a lock
-    function obtainLock() {
+    function obtainLock(err, len) {
+      if ( err ) throw err;
+      if ( limit && len >= limit ) {
+        throw new Error("User '" + owner + "' reached limit on number of templates (" + len + "/" + limit + ")");
+      }
       that._obtainTemplateLock(owner, tplname, this);
     },
     function getExistingTemplate(err, locked) {
@@ -238,10 +257,7 @@ o.addTemplate = function(owner, template, callback) {
         // TODO: how to recover this ?!
       }
 
-      if ( ! gotLock ) {
-        if ( err ) throw err;
-        return null;
-      }
+      if ( err && ! gotLock ) throw err;
 
       // release the lock
       var next = this;

npm-shrinkwrap.json

@@ -1,30 +1,60 @@
 {
   "name": "windshaft-cartodb",
-  "version": "1.7.0",
+  "version": "1.12.1",
   "dependencies": {
     "node-varnish": {
-      "version": "0.1.1"
+      "version": "0.3.0",
+      "from": "http://github.com/Vizzuality/node-varnish/tarball/0.3.0"
     },
     "underscore": {
       "version": "1.3.3"
     },
     "windshaft": {
-      "version": "0.17.1",
+      "version": "0.21.0",
+      "from": "http://github.com/CartoDB/Windshaft/tarball/0.21.0",
       "dependencies": {
         "grainstore": {
-          "version": "0.17.1",
+          "version": "0.18.1",
           "dependencies": {
-            "mapnik-reference": {
-              "version": "5.0.7"
-            },
-            "millstone": {
-              "version": "0.6.11",
+            "carto": {
+              "version": "0.9.5-cdb2",
+              "from": "http://github.com/CartoDB/carto/tarball/0.9.5-cdb2",
               "dependencies": {
                 "underscore": {
-                  "version": "1.5.2"
+                  "version": "1.4.4"
+                },
+                "xml2js": {
+                  "version": "0.2.8",
+                  "dependencies": {
+                    "sax": {
+                      "version": "0.5.8"
+                    }
+                  }
+                },
+                "optimist": {
+                  "version": "0.6.1",
+                  "dependencies": {
+                    "wordwrap": {
+                      "version": "0.0.2"
+                    },
+                    "minimist": {
+                      "version": "0.0.10"
+                    }
+                  }
+                }
+              }
+            },
+            "mapnik-reference": {
+              "version": "5.0.9"
+            },
+            "millstone": {
+              "version": "0.6.14",
+              "dependencies": {
+                "underscore": {
+                  "version": "1.6.0"
                 },
                 "request": {
-                  "version": "2.26.0",
+                  "version": "2.34.0",
                   "dependencies": {
                     "qs": {
                       "version": "0.6.6"
@@ -35,6 +65,33 @@
                     "forever-agent": {
                       "version": "0.5.2"
                     },
+                    "node-uuid": {
+                      "version": "1.4.1"
+                    },
+                    "tough-cookie": {
+                      "version": "0.12.1",
+                      "dependencies": {
+                        "punycode": {
+                          "version": "1.2.4"
+                        }
+                      }
+                    },
+                    "form-data": {
+                      "version": "0.1.4",
+                      "dependencies": {
+                        "combined-stream": {
+                          "version": "0.0.5",
+                          "dependencies": {
+                            "delayed-stream": {
+                              "version": "0.0.5"
+                            }
+                          }
+                        },
+                        "async": {
+                          "version": "0.9.0"
+                        }
+                      }
+                    },
                     "tunnel-agent": {
                       "version": "0.3.0"
                     },
@@ -52,6 +109,9 @@
                         }
                       }
                     },
+                    "oauth-sign": {
+                      "version": "0.3.0"
+                    },
                     "hawk": {
                       "version": "1.0.0",
                       "dependencies": {
@@ -69,50 +129,252 @@
                         }
                       }
                     },
-                    "aws-sign": {
-                      "version": "0.3.0"
+                    "aws-sign2": {
+                      "version": "0.5.0"
+                    }
+                  }
+                },
+                "srs": {
+                  "version": "0.4.3",
+                  "dependencies": {
+                    "nan": {
+                      "version": "1.2.0"
                     },
-                    "oauth-sign": {
-                      "version": "0.3.0"
-                    },
-                    "cookie-jar": {
-                      "version": "0.3.0"
-                    },
-                    "node-uuid": {
-                      "version": "1.4.1"
-                    },
-                    "form-data": {
-                      "version": "0.1.2",
+                    "node-pre-gyp": {
+                      "version": "0.5.17",
                       "dependencies": {
-                        "combined-stream": {
-                          "version": "0.0.4",
+                        "nopt": {
+                          "version": "2.2.1",
                           "dependencies": {
-                            "delayed-stream": {
-                              "version": "0.0.5"
+                            "abbrev": {
+                              "version": "1.0.5"
                             }
                           }
                         },
-                        "async": {
-                          "version": "0.2.10"
+                        "npmlog": {
+                          "version": "0.0.6",
+                          "dependencies": {
+                            "ansi": {
+                              "version": "0.2.1"
+                            }
+                          }
+                        },
+                        "request": {
+                          "version": "2.36.0",
+                          "dependencies": {
+                            "qs": {
+                              "version": "0.6.6"
+                            },
+                            "json-stringify-safe": {
+                              "version": "5.0.0"
+                            },
+                            "mime": {
+                              "version": "1.2.11"
+                            },
+                            "forever-agent": {
+                              "version": "0.5.2"
+                            },
+                            "node-uuid": {
+                              "version": "1.4.1"
+                            },
+                            "tough-cookie": {
+                              "version": "0.12.1",
+                              "dependencies": {
+                                "punycode": {
+                                  "version": "1.2.4"
+                                }
+                              }
+                            },
+                            "form-data": {
+                              "version": "0.1.2",
+                              "dependencies": {
+                                "combined-stream": {
+                                  "version": "0.0.4",
+                                  "dependencies": {
+                                    "delayed-stream": {
+                                      "version": "0.0.5"
+                                    }
+                                  }
+                                },
+                                "async": {
+                                  "version": "0.2.10"
+                                }
+                              }
+                            },
+                            "tunnel-agent": {
+                              "version": "0.4.0"
+                            },
+                            "http-signature": {
+                              "version": "0.10.0",
+                              "dependencies": {
+                                "assert-plus": {
+                                  "version": "0.1.2"
+                                },
+                                "asn1": {
+                                  "version": "0.1.11"
+                                },
+                                "ctype": {
+                                  "version": "0.5.2"
+                                }
+                              }
+                            },
+                            "oauth-sign": {
+                              "version": "0.3.0"
+                            },
+                            "hawk": {
+                              "version": "1.0.0",
+                              "dependencies": {
+                                "hoek": {
+                                  "version": "0.9.1"
+                                },
+                                "boom": {
+                                  "version": "0.4.2"
+                                },
+                                "cryptiles": {
+                                  "version": "0.2.2"
+                                },
+                                "sntp": {
+                                  "version": "0.2.4"
+                                }
+                              }
+                            },
+                            "aws-sign2": {
+                              "version": "0.5.0"
+                            }
+                          }
+                        },
+                        "semver": {
+                          "version": "2.3.0"
+                        },
+                        "tar": {
+                          "version": "0.1.19",
+                          "dependencies": {
+                            "inherits": {
+                              "version": "2.0.1"
+                            },
+                            "block-stream": {
+                              "version": "0.0.7"
+                            },
+                            "fstream": {
+                              "version": "0.1.25",
+                              "dependencies": {
+                                "mkdirp": {
+                                  "version": "0.3.5"
+                                },
+                                "graceful-fs": {
+                                  "version": "2.0.3"
+                                }
+                              }
+                            }
+                          }
+                        },
+                        "tar-pack": {
+                          "version": "2.0.0",
+                          "dependencies": {
+                            "uid-number": {
+                              "version": "0.0.3"
+                            },
+                            "once": {
+                              "version": "1.1.1"
+                            },
+                            "debug": {
+                              "version": "0.7.4"
+                            },
+                            "fstream": {
+                              "version": "0.1.25",
+                              "dependencies": {
+                                "mkdirp": {
+                                  "version": "0.3.5"
+                                },
+                                "graceful-fs": {
+                                  "version": "2.0.3"
+                                },
+                                "inherits": {
+                                  "version": "2.0.1"
+                                }
+                              }
+                            },
+                            "fstream-ignore": {
+                              "version": "0.0.7",
+                              "dependencies": {
+                                "minimatch": {
+                                  "version": "0.2.14",
+                                  "dependencies": {
+                                    "lru-cache": {
+                                      "version": "2.5.0"
+                                    },
+                                    "sigmund": {
+                                      "version": "1.0.0"
+                                    }
+                                  }
+                                },
+                                "inherits": {
+                                  "version": "2.0.1"
+                                }
+                              }
+                            },
+                            "readable-stream": {
+                              "version": "1.0.27-1",
+                              "dependencies": {
+                                "core-util-is": {
+                                  "version": "1.0.1"
+                                },
+                                "isarray": {
+                                  "version": "0.0.1"
+                                },
+                                "string_decoder": {
+                                  "version": "0.10.25-1"
+                                },
+                                "inherits": {
+                                  "version": "2.0.1"
+                                }
+                              }
+                            },
+                            "graceful-fs": {
+                              "version": "1.2.3"
+                            }
+                          }
+                        },
+                        "mkdirp": {
+                          "version": "0.5.0",
+                          "dependencies": {
+                            "minimist": {
+                              "version": "0.0.8"
+                            }
+                          }
+                        },
+                        "rc": {
+                          "version": "0.4.0",
+                          "dependencies": {
+                            "minimist": {
+                              "version": "0.0.10"
+                            },
+                            "deep-extend": {
+                              "version": "0.2.10"
+                            },
+                            "strip-json-comments": {
+                              "version": "0.1.3"
+                            },
+                            "ini": {
+                              "version": "1.1.0"
+                            }
+                          }
+                        },
+                        "rimraf": {
+                          "version": "2.2.8"
                         }
                       }
                     }
                   }
                 },
-                "srs": {
-                  "version": "0.3.10"
-                },
                 "zipfile": {
-                  "version": "0.4.3"
-                },
-                "sqlite3": {
-                  "version": "2.2.0",
+                  "version": "0.5.2",
                   "dependencies": {
                     "node-pre-gyp": {
-                      "version": "0.2.6",
+                      "version": "0.5.8",
                       "dependencies": {
                         "nopt": {
-                          "version": "2.1.2",
+                          "version": "2.2.0",
                           "dependencies": {
                             "abbrev": {
                               "version": "1.0.4"
@@ -127,8 +389,92 @@
                             }
                           }
                         },
+                        "request": {
+                          "version": "2.34.0",
+                          "dependencies": {
+                            "qs": {
+                              "version": "0.6.6"
+                            },
+                            "json-stringify-safe": {
+                              "version": "5.0.0"
+                            },
+                            "forever-agent": {
+                              "version": "0.5.2"
+                            },
+                            "node-uuid": {
+                              "version": "1.4.1"
+                            },
+                            "mime": {
+                              "version": "1.2.11"
+                            },
+                            "tough-cookie": {
+                              "version": "0.12.1",
+                              "dependencies": {
+                                "punycode": {
+                                  "version": "1.2.4"
+                                }
+                              }
+                            },
+                            "form-data": {
+                              "version": "0.1.2",
+                              "dependencies": {
+                                "combined-stream": {
+                                  "version": "0.0.4",
+                                  "dependencies": {
+                                    "delayed-stream": {
+                                      "version": "0.0.5"
+                                    }
+                                  }
+                                },
+                                "async": {
+                                  "version": "0.2.10"
+                                }
+                              }
+                            },
+                            "tunnel-agent": {
+                              "version": "0.3.0"
+                            },
+                            "http-signature": {
+                              "version": "0.10.0",
+                              "dependencies": {
+                                "assert-plus": {
+                                  "version": "0.1.2"
+                                },
+                                "asn1": {
+                                  "version": "0.1.11"
+                                },
+                                "ctype": {
+                                  "version": "0.5.2"
+                                }
+                              }
+                            },
+                            "oauth-sign": {
+                              "version": "0.3.0"
+                            },
+                            "hawk": {
+                              "version": "1.0.0",
+                              "dependencies": {
+                                "hoek": {
+                                  "version": "0.9.1"
+                                },
+                                "boom": {
+                                  "version": "0.4.2"
+                                },
+                                "cryptiles": {
+                                  "version": "0.2.2"
+                                },
+                                "sntp": {
+                                  "version": "0.2.4"
+                                }
+                              }
+                            },
+                            "aws-sign2": {
+                              "version": "0.5.0"
+                            }
+                          }
+                        },
                         "semver": {
-                          "version": "2.1.0"
+                          "version": "2.2.1"
                         },
                         "tar": {
                           "version": "0.1.19",
@@ -143,7 +489,7 @@
                               "version": "0.1.25",
                               "dependencies": {
                                 "graceful-fs": {
-                                  "version": "2.0.1"
+                                  "version": "2.0.3"
                                 }
                               }
                             }
@@ -165,7 +511,7 @@
                               "version": "0.1.25",
                               "dependencies": {
                                 "graceful-fs": {
-                                  "version": "2.0.1"
+                                  "version": "2.0.3"
                                 },
                                 "inherits": {
                                   "version": "2.0.1"
@@ -178,6 +524,9 @@
                                 "minimatch": {
                                   "version": "0.2.14",
                                   "dependencies": {
+                                    "lru-cache": {
+                                      "version": "2.5.0"
+                                    },
                                     "sigmund": {
                                       "version": "1.0.0"
                                     }
@@ -189,10 +538,19 @@
                               }
                             },
                             "readable-stream": {
-                              "version": "1.0.25-1",
+                              "version": "1.0.26-4",
                               "dependencies": {
+                                "core-util-is": {
+                                  "version": "1.0.1"
+                                },
+                                "isarray": {
+                                  "version": "0.0.1"
+                                },
                                 "string_decoder": {
-                                  "version": "0.10.25"
+                                  "version": "0.10.25-1"
+                                },
+                                "inherits": {
+                                  "version": "2.0.1"
                                 }
                               }
                             },
@@ -201,33 +559,231 @@
                             }
                           }
                         },
-                        "aws-sdk": {
-                          "version": "2.0.0-rc9",
+                        "mkdirp": {
+                          "version": "0.3.5"
+                        },
+                        "rc": {
+                          "version": "0.3.4",
                           "dependencies": {
-                            "xml2js": {
-                              "version": "0.2.4",
-                              "dependencies": {
-                                "sax": {
-                                  "version": "0.6.0"
-                                }
-                              }
+                            "minimist": {
+                              "version": "0.0.8"
                             },
-                            "xmlbuilder": {
-                              "version": "0.4.2"
+                            "deep-extend": {
+                              "version": "0.2.8"
+                            },
+                            "ini": {
+                              "version": "1.1.0"
                             }
                           }
                         },
-                        "rc": {
-                          "version": "0.3.3",
+                        "rimraf": {
+                          "version": "2.2.6"
+                        }
+                      }
+                    }
+                  }
+                },
+                "sqlite3": {
+                  "version": "2.2.3",
+                  "dependencies": {
+                    "node-pre-gyp": {
+                      "version": "0.5.9",
+                      "dependencies": {
+                        "nopt": {
+                          "version": "2.2.0",
                           "dependencies": {
-                            "optimist": {
-                              "version": "0.3.7",
+                            "abbrev": {
+                              "version": "1.0.4"
+                            }
+                          }
+                        },
+                        "npmlog": {
+                          "version": "0.0.6",
+                          "dependencies": {
+                            "ansi": {
+                              "version": "0.2.1"
+                            }
+                          }
+                        },
+                        "request": {
+                          "version": "2.34.0",
+                          "dependencies": {
+                            "qs": {
+                              "version": "0.6.6"
+                            },
+                            "json-stringify-safe": {
+                              "version": "5.0.0"
+                            },
+                            "forever-agent": {
+                              "version": "0.5.2"
+                            },
+                            "node-uuid": {
+                              "version": "1.4.1"
+                            },
+                            "mime": {
+                              "version": "1.2.11"
+                            },
+                            "tough-cookie": {
+                              "version": "0.12.1",
                               "dependencies": {
-                                "wordwrap": {
-                                  "version": "0.0.2"
+                                "punycode": {
+                                  "version": "1.2.4"
                                 }
                               }
                             },
+                            "form-data": {
+                              "version": "0.1.2",
+                              "dependencies": {
+                                "combined-stream": {
+                                  "version": "0.0.4",
+                                  "dependencies": {
+                                    "delayed-stream": {
+                                      "version": "0.0.5"
+                                    }
+                                  }
+                                },
+                                "async": {
+                                  "version": "0.2.10"
+                                }
+                              }
+                            },
+                            "tunnel-agent": {
+                              "version": "0.3.0"
+                            },
+                            "http-signature": {
+                              "version": "0.10.0",
+                              "dependencies": {
+                                "assert-plus": {
+                                  "version": "0.1.2"
+                                },
+                                "asn1": {
+                                  "version": "0.1.11"
+                                },
+                                "ctype": {
+                                  "version": "0.5.2"
+                                }
+                              }
+                            },
+                            "oauth-sign": {
+                              "version": "0.3.0"
+                            },
+                            "hawk": {
+                              "version": "1.0.0",
+                              "dependencies": {
+                                "hoek": {
+                                  "version": "0.9.1"
+                                },
+                                "boom": {
+                                  "version": "0.4.2"
+                                },
+                                "cryptiles": {
+                                  "version": "0.2.2"
+                                },
+                                "sntp": {
+                                  "version": "0.2.4"
+                                }
+                              }
+                            },
+                            "aws-sign2": {
+                              "version": "0.5.0"
+                            }
+                          }
+                        },
+                        "semver": {
+                          "version": "2.2.1"
+                        },
+                        "tar": {
+                          "version": "0.1.19",
+                          "dependencies": {
+                            "inherits": {
+                              "version": "2.0.1"
+                            },
+                            "block-stream": {
+                              "version": "0.0.7"
+                            },
+                            "fstream": {
+                              "version": "0.1.25",
+                              "dependencies": {
+                                "graceful-fs": {
+                                  "version": "2.0.3"
+                                }
+                              }
+                            }
+                          }
+                        },
+                        "tar-pack": {
+                          "version": "2.0.0",
+                          "dependencies": {
+                            "uid-number": {
+                              "version": "0.0.3"
+                            },
+                            "once": {
+                              "version": "1.1.1"
+                            },
+                            "debug": {
+                              "version": "0.7.4"
+                            },
+                            "fstream": {
+                              "version": "0.1.25",
+                              "dependencies": {
+                                "graceful-fs": {
+                                  "version": "2.0.3"
+                                },
+                                "inherits": {
+                                  "version": "2.0.1"
+                                }
+                              }
+                            },
+                            "fstream-ignore": {
+                              "version": "0.0.7",
+                              "dependencies": {
+                                "minimatch": {
+                                  "version": "0.2.14",
+                                  "dependencies": {
+                                    "lru-cache": {
+                                      "version": "2.5.0"
+                                    },
+                                    "sigmund": {
+                                      "version": "1.0.0"
+                                    }
+                                  }
+                                },
+                                "inherits": {
+                                  "version": "2.0.1"
+                                }
+                              }
+                            },
+                            "readable-stream": {
+                              "version": "1.0.26-4",
+                              "dependencies": {
+                                "core-util-is": {
+                                  "version": "1.0.1"
+                                },
+                                "isarray": {
+                                  "version": "0.0.1"
+                                },
+                                "string_decoder": {
+                                  "version": "0.10.25-1"
+                                },
+                                "inherits": {
+                                  "version": "2.0.1"
+                                }
+                              }
+                            },
+                            "graceful-fs": {
+                              "version": "1.2.3"
+                            }
+                          }
+                        },
+                        "mkdirp": {
+                          "version": "0.3.5"
+                        },
+                        "rc": {
+                          "version": "0.3.4",
+                          "dependencies": {
+                            "minimist": {
+                              "version": "0.0.8"
+                            },
                             "deep-extend": {
                               "version": "0.2.8"
                             },
@@ -256,7 +812,7 @@
                       "version": "0.0.2"
                     },
                     "minimist": {
-                      "version": "0.0.7"
+                      "version": "0.0.10"
                     }
                   }
                 }
@@ -274,7 +830,7 @@
               "version": "1.9.2",
               "dependencies": {
                 "formidable": {
-                  "version": "1.0.14"
+                  "version": "1.0.15"
                 }
               }
             },
@@ -306,11 +862,8 @@
           }
         },
         "tilelive-mapnik": {
-          "version": "0.6.5",
+          "version": "0.6.9",
           "dependencies": {
-            "eio": {
-              "version": "0.2.2"
-            },
             "mime": {
               "version": "1.2.11"
             },
@@ -319,18 +872,15 @@
             }
           }
         },
-        "lru-cache": {
-          "version": "2.3.1"
-        },
         "carto": {
-          "version": "0.9.5-cdb2",
-          "from": "http://github.com/CartoDB/carto/tarball/0.9.5-cdb2",
+          "version": "0.9.5-cdb3",
+          "from": "http://github.com/CartoDB/carto/tarball/0.9.5-cdb3",
           "dependencies": {
             "underscore": {
               "version": "1.4.4"
             },
             "mapnik-reference": {
-              "version": "5.0.7"
+              "version": "5.0.9"
             },
             "xml2js": {
               "version": "0.2.8",
@@ -347,19 +897,15 @@
                   "version": "0.0.2"
                 },
                 "minimist": {
-                  "version": "0.0.7"
+                  "version": "0.0.10"
                 }
               }
             }
           }
         },
-        "underscore.string": {
-          "version": "1.1.6",
-          "dependencies": {
-            "underscore": {
-              "version": "1.1.7"
-            }
-          }
+        "step-profiler": {
+          "version": "0.0.1",
+          "from": "git://github.com/CartoDB/node-step-profiler.git#0.0.1"
         },
         "pg": {
           "version": "2.6.2",
@@ -374,6 +920,9 @@
         },
         "torque.js": {
           "version": "2.2.00"
+        },
+        "node-statsd": {
+          "version": "0.0.7"
         }
       }
     },
@@ -387,36 +936,80 @@
       "version": "0.3.0"
     },
     "redis-mpool": {
-      "version": "0.0.3",
+      "version": "0.0.4",
+      "from": "http://github.com/CartoDB/node-redis-mpool/tarball/0.0.4",
       "dependencies": {
         "generic-pool": {
           "version": "2.0.4"
         },
         "hiredis": {
-          "version": "0.1.16",
+          "version": "0.1.17",
           "dependencies": {
             "bindings": {
-              "version": "1.1.1"
+              "version": "1.2.0"
+            },
+            "nan": {
+              "version": "1.1.2"
             }
           }
         }
       }
     },
     "mapnik": {
-      "version": "0.7.25"
+      "version": "0.7.26-cdb1",
+      "from": "http://github.com/Vizzuality/node-mapnik/tarball/0.7.26-cdb1"
     },
     "lzma": {
       "version": "1.2.3"
     },
+    "log4js": {
+      "version": "0.6.14",
+      "dependencies": {
+        "async": {
+          "version": "0.1.15"
+        },
+        "readable-stream": {
+          "version": "1.0.27-1",
+          "dependencies": {
+            "core-util-is": {
+              "version": "1.0.1"
+            },
+            "isarray": {
+              "version": "0.0.1"
+            },
+            "string_decoder": {
+              "version": "0.10.25-1"
+            },
+            "inherits": {
+              "version": "2.0.1"
+            }
+          }
+        }
+      }
+    },
+    "rollbar": {
+      "version": "0.3.8",
+      "dependencies": {
+        "node-uuid": {
+          "version": "1.4.1"
+        },
+        "lru-cache": {
+          "version": "2.2.4"
+        },
+        "json-stringify-safe": {
+          "version": "5.0.0"
+        }
+      }
+    },
+    "redis": {
+      "version": "0.8.6"
+    },
     "semver": {
       "version": "1.1.4"
     },
     "strftime": {
       "version": "0.6.2"
     },
-    "redis": {
-      "version": "0.8.6"
-    },
     "mocha": {
       "version": "1.14.0",
       "dependencies": {
@@ -441,7 +1034,12 @@
           "version": "1.0.7"
         },
         "debug": {
-          "version": "0.7.4"
+          "version": "1.0.2",
+          "dependencies": {
+            "ms": {
+              "version": "0.6.2"
+            }
+          }
         },
         "mkdirp": {
           "version": "0.3.5"
@@ -461,7 +1059,7 @@
               }
             },
             "graceful-fs": {
-              "version": "2.0.1"
+              "version": "2.0.3"
             },
             "inherits": {
               "version": "2.0.1"

package.json

@@ -1,7 +1,7 @@
 {
     "private": true,
     "name": "windshaft-cartodb",
-    "version": "1.7.0",
+    "version": "1.12.1",
     "description": "A map tile server for CartoDB",
     "keywords": [
       "cartodb"
@@ -22,15 +22,17 @@
       "Sandro Santilli <strk@vizzuality.com>"
     ],
     "dependencies": {
-        "node-varnish": "0.1.1",
+        "node-varnish": "http://github.com/Vizzuality/node-varnish/tarball/0.3.0",
         "underscore"  : "~1.3.3",
-        "windshaft"  : "~0.17.1",
+        "windshaft"  : "http://github.com/CartoDB/Windshaft/tarball/0.21.0",
         "step": "0.0.x",
         "request": "2.9.202",
         "cartodb-redis": "~0.3.0",
-        "redis-mpool": "~0.0.2",
-        "mapnik": "~0.7.22",
-        "lzma": "~1.2.3"
+        "redis-mpool": "http://github.com/CartoDB/node-redis-mpool/tarball/0.0.4",
+        "mapnik": "http://github.com/Vizzuality/node-mapnik/tarball/0.7.26-cdb1",
+        "lzma": "~1.2.3",
+        "log4js": "~0.6.10",
+        "rollbar": "~0.3.1"
     },
     "devDependencies": {
         "mocha": "1.14.0",

run_tests.sh

@@ -5,6 +5,8 @@ OPT_CREATE_PGSQL=yes # create the PostgreSQL test environment
 OPT_DROP_REDIS=yes   # drop the redis test environment
 OPT_DROP_PGSQL=yes   # drop the PostgreSQL test environment
 
+export PGAPPNAME=cartodb_tiler_tester
+
 cd $(dirname $0)
 BASEDIR=$(pwd)
 cd -

test/acceptance/multilayer.js

@@ -10,23 +10,16 @@ var strftime    = require('strftime');
 var SQLAPIEmu   = require(__dirname + '/../support/SQLAPIEmu.js');
 var redis_stats_db = 5;
 
-require(__dirname + '/../support/test_helper');
+var helper = require(__dirname + '/../support/test_helper');
 
 var windshaft_fixtures = __dirname + '/../../node_modules/windshaft/test/fixtures';
 
 var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft');
-var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
+var ServerOptions = require(__dirname + '/../../lib/cartodb/server_options');
+serverOptions = ServerOptions();
 var server = new CartodbWindshaft(serverOptions);
 server.setMaxListeners(0);
 
-// Check that the response headers do not request caching
-// Throws on failure
-function checkNoCache(res) {
-  assert.ok(!res.headers.hasOwnProperty('x-cache-channel'));
-  assert.ok(!res.headers.hasOwnProperty('cache-control')); // is this correct ?
-  assert.ok(!res.headers.hasOwnProperty('last-modified')); // is this correct ?
-}
-
 suite('multilayer', function() {
 
     var redis_client = redis.createClient(global.environment.redis.port);
@@ -34,6 +27,10 @@ suite('multilayer', function() {
     var expected_last_updated_epoch = 1234567890123; // this is hard-coded into SQLAPIEmu
     var expected_last_updated = new Date(expected_last_updated_epoch).toISOString();
 
+    var test_user = _.template(global.environment.postgres_auth_user, {user_id:1});
+    var test_pubuser = global.environment.postgres.user;
+    var test_database = test_user + '_db';
+
     suiteSetup(function(done){
       sqlapi_server = new SQLAPIEmu(global.environment.sqlapi.port, done);
     });
@@ -107,7 +104,7 @@ suite('multilayer', function() {
               // Check X-Cache-Channel
               cc = res.headers['x-cache-channel'];
               assert.ok(cc); 
-              var dbname = 'test_cartodb_user_1_db'
+              var dbname = test_database;
               assert.equal(cc.substring(0, dbname.length), dbname);
               var jsonquery = cc.substring(dbname.length+1);
               var sentquery = JSON.parse(jsonquery);
@@ -122,6 +119,24 @@ suite('multilayer', function() {
               });
           });
         },
+        // See https://github.com/CartoDB/Windshaft-cartodb/issues/170
+        function do_get_tile_nosignature(err)
+        {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup/localhost@' + expected_token + ':cb0/0/0/0.png',
+              method: 'GET',
+              headers: {host: 'localhost' },
+              encoding: 'binary'
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 403, res.statusCode + ':' + res.body);
+              var parsed = JSON.parse(res.body);
+              var msg = parsed.error; // TODO: should it be "errors" ?
+              assert.ok(msg.match(/permission denied/i), msg);
+              next(err);
+          });
+        },
         function do_get_grid_layer0(err)
         {
           if ( err ) throw err;
@@ -178,6 +193,134 @@ suite('multilayer', function() {
     });
 
 
+    test("should include serverMedata in the response", function(done) {
+      global.environment.serverMetadata = { cdn_url : { http:'test', https: 'tests' } }
+      var layergroup =  {
+        version: '1.0.0',
+        layers: [
+           { options: {
+               sql: 'select cartodb_id, ST_Translate(the_geom_webmercator, 5e6, 0) as the_geom_webmercator from test_table limit 2',
+               cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', 
+               cartocss_version: '2.0.1'
+             } }
+        ]
+      };
+
+      var expected_token; 
+      Step(
+        function do_create_get()
+        {
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup?config=' + encodeURIComponent(JSON.stringify(layergroup)),
+              method: 'GET',
+              headers: {host: 'localhost'} 
+          }, {}, function(res, err) { next(err, res); });
+        },
+        function do_check_create(err, res) {
+          var parsed = JSON.parse(res.body);
+          assert.ok(_.isEqual(parsed.cdn_url, global.environment.serverMetadata.cdn_url));
+          done();
+        }
+      )
+    });
+
+
+    test("get creation requests has cache", function(done) {
+
+      var layergroup =  {
+        version: '1.0.0',
+        layers: [
+           { options: {
+               sql: 'select cartodb_id, ST_Translate(the_geom_webmercator, 5e6, 0) as the_geom_webmercator from test_table limit 2',
+               cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', 
+               cartocss_version: '2.0.1'
+             } }
+        ]
+      };
+
+      var expected_token; 
+      Step(
+        function do_create_get()
+        {
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup?config=' + encodeURIComponent(JSON.stringify(layergroup)),
+              method: 'GET',
+              headers: {host: 'localhost'} 
+          }, {}, function(res, err) { next(err, res); });
+        },
+        function do_check_create(err, res) {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 200, res.body);
+          var parsedBody = JSON.parse(res.body);
+          expected_token = parsedBody.layergroupid.split(':')[0];
+          helper.checkCache(res);
+          return null;
+        },
+        function finish(err) {
+          var errors = [];
+          if ( err ) {
+            errors.push(err.message);
+            console.log("Error: " + err);
+          }
+          redis_client.keys("map_cfg|" + expected_token, function(err, matches) {
+              if ( err ) errors.push(err.message);
+              assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches);
+              redis_client.del(matches, function(err) {
+                if ( err ) errors.push(err.message);
+                if ( errors.length ) done(new Error(errors));
+                else done(null);
+              });
+          });
+        }
+      );
+    });
+
+    test("get creation has no cache if sql is bogus", function(done) {
+        var layergroup =  {
+            version: '1.0.0',
+            layers: [
+                { options: {
+                    sql: 'select bogus(0,0) as the_geom_webmercator',
+                    cartocss: '#layer { polygon-fill: red; }',
+                    cartocss_version: '2.0.1'
+                } }
+            ]
+        };
+        assert.response(server, {
+            url: '/tiles/layergroup?config=' + encodeURIComponent(JSON.stringify(layergroup)),
+            method: 'GET',
+            headers: {host: 'localhost'}
+        }, {}, function(res) {
+            assert.notEqual(res.statusCode, 200);
+            helper.checkNoCache(res);
+            done();
+        });
+    });
+
+    test("get creation has no cache if cartocss is not valid", function(done) {
+        var layergroup =  {
+            version: '1.0.0',
+            layers: [
+                { options: {
+                    sql: 'select cartodb_id, ST_Translate(the_geom_webmercator, 5e6, 0) as the_geom_webmercator from test_table limit 2',
+                    cartocss: '#layer { invalid-rule:red; }',
+                    cartocss_version: '2.0.1'
+                } }
+            ]
+        };
+        assert.response(server, {
+            url: '/tiles/layergroup?config=' + encodeURIComponent(JSON.stringify(layergroup)),
+            method: 'GET',
+            headers: {host: 'localhost'}
+        }, {}, function(res) {
+            assert.notEqual(res.statusCode, 200);
+            helper.checkNoCache(res);
+            done();
+        });
+    });
+
     test("layergroup can hold substitution tokens", function(done) {
 
       var layergroup =  {
@@ -237,7 +380,7 @@ suite('multilayer', function() {
               // Check X-Cache-Channel
               var cc = res.headers['x-cache-channel'];
               assert.ok(cc); 
-              var dbname = 'test_cartodb_user_1_db'
+              var dbname = test_database;
               assert.equal(cc.substring(0, dbname.length), dbname);
               var jsonquery = cc.substring(dbname.length+1);
               var sentquery = JSON.parse(jsonquery);
@@ -270,7 +413,7 @@ suite('multilayer', function() {
               // Check X-Cache-Channel
               var cc = res.headers['x-cache-channel'];
               assert.ok(cc); 
-              var dbname = 'test_cartodb_user_1_db'
+              var dbname = test_database;
               assert.equal(cc.substring(0, dbname.length), dbname);
               var jsonquery = cc.substring(dbname.length+1);
               var sentquery = JSON.parse(jsonquery);
@@ -488,11 +631,11 @@ suite('multilayer', function() {
           headers: {host: 'localhost', 'Content-Type': 'application/json' },
           data: JSON.stringify(layergroup)
       }, {}, function(res) {
-          assert.equal(res.statusCode, 400, res.body);
+          assert.equal(res.statusCode, 404, res.statusCode + ": " + res.body);
           var parsed = JSON.parse(res.body);
           var msg = parsed.errors[0];
           assert.ok(msg.match(/bogus.*exist/), msg);
-          checkNoCache(res);
+          helper.checkNoCache(res);
           done();
       });
     });
@@ -562,7 +705,7 @@ suite('multilayer', function() {
               // Check X-Cache-Channel
               var cc = res.headers['x-cache-channel'];
               assert.ok(cc); 
-              var dbname = 'test_cartodb_user_1_db'
+              var dbname = test_database;
               assert.equal(cc.substring(0, dbname.length), dbname);
               next(err);
           });
@@ -606,7 +749,7 @@ suite('multilayer', function() {
               headers: {host: 'localhost' },
               encoding: 'binary'
           }, {}, function(res) {
-              assert.equal(res.statusCode, 401);
+              assert.equal(res.statusCode, 403);
               var re = RegExp('permission denied');
               assert.ok(res.body.match(re), 'No "permission denied" error: ' + res.body);
               next(err);
@@ -622,7 +765,7 @@ suite('multilayer', function() {
               headers: {host: 'localhost' },
               method: 'GET'
           }, {}, function(res) {
-              assert.equal(res.statusCode, 401);
+              assert.equal(res.statusCode, 403);
               var re = RegExp('permission denied');
               assert.ok(res.body.match(re), 'No "permission denied" error: ' + res.body);
               next(err);
@@ -638,7 +781,7 @@ suite('multilayer', function() {
               headers: {host: 'localhost' },
               method: 'GET'
           }, {}, function(res) {
-              assert.equal(res.statusCode, 401);
+              assert.equal(res.statusCode, 403);
               var re = RegExp('permission denied');
               assert.ok(res.body.match(re), 'No "permission denied" error: ' + res.body);
               next(err);
@@ -663,6 +806,123 @@ suite('multilayer', function() {
       );
     });
 
+    // See https://github.com/CartoDB/Windshaft-cartodb/issues/152
+    test("x-cache-channel still works for GETs after tiler restart", function(done) {
+
+      var layergroup =  {
+        version: '1.0.0',
+        layers: [
+           { options: {
+               sql: 'select * from test_table where cartodb_id=1',
+               cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', 
+               cartocss_version: '2.1.0',
+               interactivity: 'cartodb_id'
+             } }
+        ]
+      };
+
+      var expected_token; // = "b4ed64d93a411a59f330ab3d798e4009";
+      Step(
+        function do_post()
+        {
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup?map_key=1234',
+              method: 'POST',
+              headers: {host: 'localhost', 'Content-Type': 'application/json' },
+              data: JSON.stringify(layergroup)
+          }, {}, function(res, err) { next(err, res); });
+        },
+        function check_post(err, res) {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 200, res.body);
+          var parsedBody = JSON.parse(res.body);
+          var expectedBody = { layergroupid: expected_token };
+          // check last modified
+          var qTables = JSON.stringify({
+            'q': 'SELECT CDB_QueryTables($windshaft$'
+                + layergroup.layers[0].options.sql
+                + '$windshaft$)'
+          });
+          assert.equal(parsedBody.last_updated, expected_last_updated);
+          if ( expected_token ) {
+            assert.equal(parsedBody.layergroupid, expected_token + ':' + expected_last_updated_epoch);
+          }
+          else expected_token = parsedBody.layergroupid.split(':')[0];
+          return null;
+        },
+        function do_get0(err)
+        {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup/' + expected_token + ':cb0/0/0/0.png?map_key=1234',
+              method: 'GET',
+              headers: {host: 'localhost' },
+              encoding: 'binary'
+          }, {}, function(res, err) { next(err, res); });
+        },
+        function do_check0(err, res) {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 200, res.body);
+          assert.equal(res.headers['content-type'], "image/png");
+
+          // Check X-Cache-Channel
+          var cc = res.headers['x-cache-channel'];
+          assert.ok(cc, "Missing X-Cache-Channel"); 
+          var dbname = test_database;
+          assert.equal(cc.substring(0, dbname.length), dbname);
+          return null;
+        },
+        function do_restart_server(err, res) {
+          if ( err ) throw err;
+          // hack simulating restart...
+          serverOptions = ServerOptions();
+          server = new CartodbWindshaft(serverOptions);
+          return null;
+        },
+        function do_get1(err)
+        {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup/' + expected_token + ':cb0/0/0/0.png?map_key=1234',
+              method: 'GET',
+              headers: {host: 'localhost' },
+              encoding: 'binary'
+          }, {}, function(res, err) { next(err, res); });
+        },
+        function do_check1(err, res) {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 200, res.body);
+          assert.equal(res.headers['content-type'], "image/png");
+
+          // Check X-Cache-Channel
+          var cc = res.headers['x-cache-channel'];
+          assert.ok(cc, "Missing X-Cache-Channel on restart");
+          var dbname = test_database;
+          assert.equal(cc.substring(0, dbname.length), dbname);
+          return null;
+        },
+        function finish(err) {
+          var errors = [];
+          if ( err ) {
+            errors.push(err.message);
+            console.log("Error: " + err);
+          }
+          redis_client.keys("map_cfg|" + expected_token, function(err, matches) {
+              if ( err ) errors.push(err.message);
+              assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches);
+              redis_client.del(matches, function(err) {
+                if ( err ) errors.push(err.message);
+                if ( errors.length ) done(new Error(errors.join(',')));
+                else done(null);
+              });
+          });
+        }
+      );
+    });
+
     // https://github.com/cartodb/Windshaft-cartodb/issues/81
     test("invalid text-name in CartoCSS", function(done) {
 
@@ -941,10 +1201,12 @@ suite('multilayer', function() {
           var parsedBody = JSON.parse(res.body);
           var token_components = parsedBody.layergroupid.split(':');
           expected_token = token_components[0];
+          var last_request = sqlapi_server.getLastRequest();
+          assert.equal(last_request.method, 'POST');
           return null;
         },
         function cleanup(err) {
-          if ( err ) errors.push(err.message);
+          if ( err ) errors.push('' + err);
           if ( ! expected_token ) return null;
           var next = this;
           redis_client.keys("map_cfg|" + expected_token, function(err, matches) {
@@ -1004,6 +1266,47 @@ suite('multilayer', function() {
       );
     });
 
+    // See https://github.com/CartoDB/Windshaft-cartodb/issues/167
+    test("lack of response from sql-api will result in a timeout", function(done) {
+
+      var layergroup =  {
+        version: '1.0.0',
+        layers: [
+           { options: {
+               sql: "select *, 'SQLAPINOANSWER' from test_table",
+               cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', 
+               cartocss_version: '2.1.0'
+             } }
+        ]
+      };
+
+      Step(
+        function do_post()
+        {
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup',
+              method: 'POST',
+              headers: {host: 'localhost', 'Content-Type': 'application/json' },
+              data: JSON.stringify(layergroup)
+          }, {}, function(res, err) { next(err, res); });
+        },
+        function check_post(err, res) {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body);
+          var parsed = JSON.parse(res.body);
+          assert.ok(parsed.errors, 'Missing "errors" in response: ' + JSON.stringify(parsed));
+          assert.equal(parsed.errors.length, 1);
+          var msg = parsed.errors[0];
+          assert.ok(msg, /could not fetch source tables/, msg);
+          return null;
+        },
+        function finish(err) {
+          done(err);
+        }
+      );
+    });
+
 
     suiteTeardown(function(done) {
 

test/acceptance/server.js

@@ -12,7 +12,7 @@ var SQLAPIEmu   = require(__dirname + '/../support/SQLAPIEmu.js');
 var helper = require(__dirname + '/../support/test_helper');
 
 var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft');
-var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
+var serverOptions = require(__dirname + '/../../lib/cartodb/server_options')();
 var server = new CartodbWindshaft(serverOptions);
 server.setMaxListeners(0);
 
@@ -22,7 +22,7 @@ suite('server', function() {
     var sqlapi_server;
 
     var mapnik_version = global.environment.mapnik_version || mapnik.versions.mapnik;
-    var test_database = 'test_cartodb_user_1_db';
+    var test_database = _.template(global.environment.postgres_auth_user, {user_id:1}) + '_db';
     var default_style;
     if ( semver.satisfies(mapnik_version, '<2.1.0') ) {
       // 2.0.0 default
@@ -53,12 +53,25 @@ suite('server', function() {
     
     // TODO: I guess this should be a 404 instead...
     test("get call to server returns 200", function(done){
-        assert.response(server, {
-            url: '/',
-            method: 'GET'
-        },{
-            status: 200
-        }, function() { done(); });
+      Step(
+        function doGet() {
+          var next = this;
+          assert.response(server, {
+              url: '/',
+              method: 'GET'
+          },{}, function(res, err) { next(err,res); });
+        },
+        function doCheck(err, res) {
+          if ( err ) throw err;
+          assert.ok(res.statusCode, 200);
+          var cc = res.headers['x-cache-channel'];
+          assert.ok(!cc);
+          return null;
+        },
+        function finish(err) {
+          done(err);
+        }
+      );
     });
 
     /////////////////////////////////////////////////////////////////////////////////
@@ -121,8 +134,7 @@ suite('server', function() {
             method: 'GET'
         },{
         }, function(res) {
-          // FIXME: should be 401 Unauthorized
-          assert.equal(res.statusCode, 400, res.body);
+          assert.equal(res.statusCode, 403, res.statusCode + ':' + res.body);
           assert.deepEqual(JSON.parse(res.body),
             {error: 'Sorry, you are unauthorized (permission denied)'});
           assert.ok(!res.headers.hasOwnProperty('cache-control'));
@@ -139,7 +151,7 @@ suite('server', function() {
             method: 'GET'
         },{
         }, function(res) {
-          // FIXME: should be 401 Unauthorized
+          // FIXME: should be 403 Forbidden or 404 User Not Found
           assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body);
           assert.deepEqual(JSON.parse(res.body),
             {error:"missing unknown_user's database_name in redis (try CARTODB/script/restore_redis)"});
@@ -175,7 +187,8 @@ suite('server', function() {
         },
         function setupRedisBase(err, matches) {
           if ( err ) throw err;
-          assert.equal(matches.length, 0);
+          assert.equal(matches.length, 0,
+            'Unexpected redis keys at test start: ' + matches.join("\n"));
           redis_client.set(base_key, 
             JSON.stringify({ style: style }),
           this);
@@ -306,7 +319,7 @@ suite('server', function() {
               headers: {host: 'localhost', 'Content-Type': 'application/x-www-form-urlencoded' },
               data: querystring.stringify({style: 'Map { background-color:#aaa; }'})
           },{}, function(res) {
-            // FIXME: should be 401 Unauthorized
+            // FIXME: should be 403 Forbidden
             assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body);
             assert.ok(res.body.indexOf('map state cannot be changed by unauthenticated request') != -1, res.body);
 
@@ -404,8 +417,8 @@ suite('server', function() {
             method: 'DELETE',
             headers: {host: 'localhost'},
         },{}, function(res) { 
-          // FIXME: should be 401 Unauthorized
-          assert.equal(res.statusCode, 500, res.body);
+          // FIXME: should be 403 Forbidden
+          assert.equal(res.statusCode, 400, res.body);
           assert.ok(res.body.indexOf('map state cannot be changed by unauthenticated request') != -1, res.body);
           // check that the style wasn't really deleted !
           assert.response(server, {
@@ -518,7 +531,7 @@ suite('server', function() {
             url: '/tiles/test_table_private_1/infowindow',
             method: 'GET'
         },{}, function(res) {
-          // FIXME: should be 401 Unauthorized
+          // FIXME: should be 403 Forbidden
           assert.equal(res.statusCode, 500, res.statusCode + ': ' + res.body);
           done();
         });
@@ -533,7 +546,7 @@ suite('server', function() {
             method: 'GET'
         },{
         }, function(res) {
-          // FIXME: should be 401 Unauthorized
+          // FIXME: should be 403 Forbidden
           assert.equal(res.statusCode, 500, res.statusCode + ': ' + res.body);
           assert.deepEqual(JSON.parse(res.body),
             {error:"missing unknown_user's database_name in redis (try CARTODB/script/restore_redis)"});
@@ -601,8 +614,26 @@ suite('server', function() {
             url: '/tiles/test_table_private_1/6/31/24.grid.json',
             method: 'GET'
         },{}, function(res) {
-          // 401 Unauthorized
-          assert.equal(res.statusCode, 401, res.statusCode + ': ' + res.body);
+          // 403 Forbidden
+          assert.equal(res.statusCode, 403, res.statusCode + ': ' + res.body);
+          done();
+        });
+    });
+
+    // See http://github.com/CartoDB/Windshaft-cartodb/issues/186
+    test("get'ing the grid of a private table should fail when unauthenticated (jsonp)",
+    function(done) {
+        assert.response(server, {
+            headers: {host: 'localhost'},
+            url: '/tiles/test_table_private_1/6/31/24.grid.json?callback=x',
+            method: 'GET'
+        },{}, function(res) {
+          // It's forbidden, but jsonp calls for status = 200
+          assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+          // Still, we do NOT want to add caching headers here
+          // See https://github.com/CartoDB/Windshaft-cartodb/issues/186
+          assert.ok(!res.headers.hasOwnProperty('cache-control'),
+            "Unexpected Cache-Control: " + res.headers['cache-control']);
           done();
         });
     });
@@ -616,7 +647,7 @@ suite('server', function() {
             method: 'GET'
         },{
         }, function(res) {
-          // FIXME: should be 401 Unauthorized
+          // FIXME: should be 403 Forbidden
           assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body);
           assert.deepEqual(JSON.parse(res.body),
             {error:"missing unknown_user's database_name in redis (try CARTODB/script/restore_redis)"});
@@ -750,8 +781,8 @@ suite('server', function() {
             method: 'GET'
         },{
         }, function(res) {
-          // 401 Unauthorized
-          assert.equal(res.statusCode, 401, res.statusCode + ': ' + res.body);
+          // 403 Forbidden
+          assert.equal(res.statusCode, 403, res.statusCode + ': ' + res.body);
           done();
         });
     });
@@ -767,7 +798,7 @@ suite('server', function() {
             method: 'GET'
         },{
         }, function(res) {
-          // FIXME: should be 401 Unauthorized
+          // FIXME: should be 403 Forbidden
           assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body);
           assert.deepEqual(JSON.parse(res.body),
             {error:"missing unknown_user's database_name in redis (try CARTODB/script/restore_redis)"});
@@ -791,8 +822,8 @@ suite('server', function() {
             method: 'GET'
         },{
         }, function(res) {
-          // 401 Unauthorized
-          assert.equal(res.statusCode, 401, res.statusCode + ': ' + res.body);
+          // 403 Forbidden
+          assert.equal(res.statusCode, 403, res.statusCode + ': ' + res.body);
           // Failed in 1.6.0 of https://github.com/CartoDB/Windshaft-cartodb/issues/107
           assert.ok(!res.headers.hasOwnProperty('cache-control'),
             "Unexpected Cache-Control: " + res.headers['cache-control']);
@@ -858,8 +889,8 @@ suite('server', function() {
     // See https://github.com/CartoDB/Windshaft-cartodb/issues/89
     test("get'ing a tile with a user-specific database password",  function(done){
         var style = querystring.stringify({style: test_style_black_200, style_version: '2.0.0'});
-        var backupDBPass = global.settings.postgres_auth_pass;
-        global.settings.postgres_auth_pass = '<%= user_password %>';
+        var backupDBPass = global.environment.postgres_auth_pass;
+        global.environment.postgres_auth_pass = '<%= user_password %>';
         Step (
           function() {
             var next = this;
@@ -886,7 +917,7 @@ suite('server', function() {
               return null
           },
           function finish(err) {
-            global.settings.postgres_auth_pass = backupDBPass;
+            global.environment.postgres_auth_pass = backupDBPass;
             done(err);
           }
         );
@@ -1111,8 +1142,8 @@ suite('server', function() {
             var ct = res.headers['content-type'];
             assert.equal(ct, 'image/png');
             var cc = res.headers['x-cache-channel'];
-            assert(cc);
-            var dbname = 'test_cartodb_user_1_db'
+            assert(cc, 'Missing X-Cache-Channel');
+            var dbname = test_database;
             assert.equal(cc.substring(0, dbname.length), dbname);
             var jsonquery = cc.substring(dbname.length+1);
             var sentquery = JSON.parse(jsonquery);
@@ -1148,6 +1179,7 @@ suite('server', function() {
             assert.ok(last_request);
             var host = last_request.headers['host'];
             assert.ok(host);
+            assert.equal(last_request.method, 'GET');
             assert.equal(host, 'localhost.donot_look_this_up');
             return null;
           },
@@ -1264,7 +1296,7 @@ suite('server', function() {
             url: '/tiles/test_table_private_1/map_metadata',
             method: 'GET'
         },{}, function(res) {
-          // FIXME: should be 401 instead
+          // FIXME: should be 403 instead
           assert.equal(res.statusCode, 500, res.statusCode + ': ' + res.body);
           assert.ok(!res.headers.hasOwnProperty('cache-control'));
           done();

test/acceptance/templates.js

@@ -1,5 +1,4 @@
 var assert      = require('../support/assert');
-var tests       = module.exports = {};
 var _           = require('underscore');
 var redis       = require('redis');
 var querystring = require('querystring');
@@ -10,12 +9,19 @@ var strftime    = require('strftime');
 var SQLAPIEmu   = require(__dirname + '/../support/SQLAPIEmu.js');
 var redis_stats_db = 5;
 
-require(__dirname + '/../support/test_helper');
+// Pollute the PG environment to make sure
+// configuration settings are always enforced
+// See https://github.com/CartoDB/Windshaft-cartodb/issues/174
+process.env['PGPORT'] = '666';
+process.env['PGHOST'] = 'fake';
+
+var helper = require(__dirname + '/../support/test_helper');
 
 var windshaft_fixtures = __dirname + '/../../node_modules/windshaft/test/fixtures';
 
 var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft');
-var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
+var ServerOptions = require(__dirname + '/../../lib/cartodb/server_options');
+var serverOptions = ServerOptions();
 var server = new CartodbWindshaft(serverOptions);
 server.setMaxListeners(0);
 
@@ -69,7 +75,7 @@ suite('template_api', function() {
         function postTemplate(err, res)
         {
           if ( err ) throw err;
-          assert.equal(res.statusCode, 401);
+          assert.equal(res.statusCode, 403);
           var parsed = JSON.parse(res.body);
           assert.ok(parsed.hasOwnProperty('error'), res.body);
           err = parsed.error;
@@ -303,6 +309,52 @@ suite('template_api', function() {
       });
     });
 
+    test("instance endpoint should return server metadata", function(done){
+      global.environment.serverMetadata = { cdn_url : { http:'test', https: 'tests' } }
+      var tmpl = _.clone(template_acceptance1)
+      tmpl.name = "rambotemplate2"
+
+      Step(function postTemplate1(err, res) {
+          var next = this;
+          var post_request = {
+              url: '/tiles/template?api_key=1234',
+              method: 'POST',
+              headers: {host: 'localhost', 'Content-Type': 'application/json' },
+              data: JSON.stringify(tmpl)
+          };
+          assert.response(server, post_request, {}, function(res) { 
+            next(null, res); 
+          });
+        },
+        function testCORS() {
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/template/' + tmpl.name,
+              method: 'POST',
+              headers: {host: 'localhost', 'Content-Type': 'application/json' },
+          },{
+              status: 200
+          }, function(res) { 
+            var parsed = JSON.parse(res.body);
+            assert.ok(_.isEqual(parsed.cdn_url, global.environment.serverMetadata.cdn_url));
+            next(null); 
+          });
+        },
+        function deleteTemplate(err) {
+          if ( err ) throw err;
+          var del_request = {
+              url: '/tiles/template/' + tmpl.name + '?api_key=1234', 
+              method: 'DELETE',
+              headers: {host: 'localhost', 'Content-Type': 'application/json' }
+          }
+          var next = this;
+          assert.response(server, del_request, {},
+            function(res) { done(); });
+        }
+      );
+    });
+
+
 
     test("can list templates", function(done) {
 
@@ -364,7 +416,7 @@ suite('template_api', function() {
         function litsTemplates(err, res)
         {
           if ( err ) throw err;
-          assert.equal(res.statusCode, 401, res.statusCode + ': ' + res.body);
+          assert.equal(res.statusCode, 403, res.statusCode + ': ' + res.body);
           var parsed = JSON.parse(res.body);
           assert.ok(parsed.hasOwnProperty('error'),
             'Missing error from response: ' + res.body);
@@ -577,7 +629,7 @@ suite('template_api', function() {
         function getTemplate(err, res)
         {
           if ( err ) throw err;
-          assert.equal(res.statusCode, 401, res.statusCode + ": " + res.body);
+          assert.equal(res.statusCode, 403, res.statusCode + ": " + res.body);
           var parsedBody = JSON.parse(res.body);
           assert.ok(parsedBody.hasOwnProperty('error'), res.body);
           assert.ok(parsedBody.error.match(/only.*authenticated.*user/i),
@@ -686,7 +738,7 @@ suite('template_api', function() {
         function deleteTemplate(err, res)
         {
           if ( err ) throw err;
-          assert.equal(res.statusCode, 401, res.statusCode + ": " + res.body);
+          assert.equal(res.statusCode, 403, res.statusCode + ": " + res.body);
           var parsed = JSON.parse(res.body);
           assert.ok(parsed.hasOwnProperty('error'),
             "Missing 'error' from response body: " + res.body);
@@ -811,10 +863,10 @@ suite('template_api', function() {
           assert.response(server, post_request, {},
             function(res) { next(null, res); });
         },
-        function instanciateAuth(err, res)
-        {
+        // See https://github.com/CartoDB/Windshaft-cartodb/issues/173
+        function instanciateForeignDB(err, res) {
           if ( err ) throw err;
-          assert.equal(res.statusCode, 401,
+          assert.equal(res.statusCode, 403,
             'Unexpected success instanciating template with no auth: '
             + res.statusCode + ': ' + res.body);
           var parsed = JSON.parse(res.body);
@@ -822,6 +874,25 @@ suite('template_api', function() {
             "Missing 'error' from response body: " + res.body);
           assert.ok(parsed.error.match(/unauthorized/i),
             'Unexpected error for unauthorized instance : ' + parsed.error);
+          var post_request = {
+              url: '/tiles/template/' + tpl_id + '?auth_token=valid2',
+              method: 'POST',
+              headers: {host: 'foreign', 'Content-Type': 'application/json' },
+              data: JSON.stringify(template_params)
+          }
+          var next = this;
+          assert.response(server, post_request, {},
+            function(res) { next(null, res); });
+        },
+        function instanciateAuth(err, res)
+        {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 403, res.statusCode + ': ' + res.body);
+          var parsed = JSON.parse(res.body);
+          assert.ok(parsed.hasOwnProperty('error'),
+            "Missing 'error' from response body: " + res.body);
+          assert.ok(parsed.error.match(/cannot instanciate/i),
+            'Unexpected error for forbidden instance : ' + parsed.error);
           var post_request = {
               url: '/tiles/template/' + tpl_id + '?auth_token=valid2',
               method: 'POST',
@@ -858,14 +929,14 @@ suite('template_api', function() {
         },
         function fetchTileAuth(err, res) {
           if ( err ) throw err;
-          assert.equal(res.statusCode, 401,
+          assert.equal(res.statusCode, 403,
             'Fetching tile with no auth: ' + res.statusCode + ': ' + res.body);
           var parsed = JSON.parse(res.body);
           assert.ok(parsed.hasOwnProperty('error'),
             "Missing 'error' from response body: " + res.body);
           assert.ok(parsed.error.match(/permission denied/i),
             'Unexpected error for unauthorized instance '
-            + '(expected /permission denied): ' + parsed.error);
+            + '(expected /permission denied/): ' + parsed.error);
           var get_request = {
               url: '/tiles/layergroup/' + layergroupid + '/0/0/0.png?auth_token=valid1',
               method: 'GET',
@@ -884,6 +955,33 @@ suite('template_api', function() {
           assert.equal(res.headers['content-type'], "image/png");
           return null;
         },
+        // See https://github.com/CartoDB/Windshaft-cartodb/issues/172
+        function fetchTileForeignSignature(err, res) {
+          if ( err ) throw err;
+          var foreignsigned = layergroupid.replace(/[^@]*@/, 'foreign@');
+          var get_request = {
+              url: '/tiles/layergroup/' + foreignsigned + '/0/0/0.png?auth_token=valid1',
+              method: 'GET',
+              headers: {host: 'localhost' },
+              encoding: 'binary'
+          }
+          var next = this;
+          assert.response(server, get_request, {},
+            function(res) { next(null, res); });
+        },
+        function checkForeignSignerError(err, res) {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 403, 
+            'Unexpected error for authorized instance: '
+            + res.statusCode + ' -- ' + res.body);
+          var parsed = JSON.parse(res.body);
+          assert.ok(parsed.hasOwnProperty('error'),
+            "Missing 'error' from response body: " + res.body);
+          assert.ok(parsed.error.match(/cannot use/i),
+            'Unexpected error for unauthorized instance '
+            + '(expected /cannot use/): ' + parsed.error);
+          return null;
+        },
         function deleteTemplate(err)
         {
           if ( err ) throw err;
@@ -912,7 +1010,7 @@ suite('template_api', function() {
         },
         function checkTileDeleted(err, res) {
           if ( err ) throw err;
-          assert.equal(res.statusCode, 401, 
+          assert.equal(res.statusCode, 403,
             'Unexpected statusCode fetch tile after signature revokal: '
             + res.statusCode + ':' + res.body); 
           var parsed = JSON.parse(res.body);
@@ -1008,7 +1106,7 @@ suite('template_api', function() {
         function instanciateAuth(err, res)
         {
           if ( err ) throw err;
-          assert.equal(res.statusCode, 401,
+          assert.equal(res.statusCode, 403,
             'Unexpected success instanciating template with no auth: '
             + res.statusCode + ': ' + res.body);
           var parsed = JSON.parse(res.body);
@@ -1052,7 +1150,7 @@ suite('template_api', function() {
         },
         function fetchTileAuth(err, res) {
           if ( err ) throw err;
-          assert.equal(res.statusCode, 401,
+          assert.equal(res.statusCode, 403,
             'Fetching tile with no auth: ' + res.statusCode + ': ' + res.body);
           var parsed = JSON.parse(res.body);
           assert.ok(parsed.hasOwnProperty('error'),
@@ -1070,12 +1168,37 @@ suite('template_api', function() {
           assert.response(server, get_request, {},
             function(res) { next(null, res); });
         },
-        function checkTile(err, res) {
+        function checkTile_fetchOnRestart(err, res) {
           if ( err ) throw err;
           assert.equal(res.statusCode, 200, 
             'Unexpected error for authorized instance: '
             + res.statusCode + ' -- ' + res.body);
           assert.equal(res.headers['content-type'], "application/json; charset=utf-8");
+          var cc = res.headers['x-cache-channel'];
+          assert.ok(cc);
+          assert.ok(cc.match, /ciao/, cc);
+          // hack simulating restart...
+          serverOptions = ServerOptions(); // need to clean channel cache
+          server = new CartodbWindshaft(serverOptions);
+          var get_request = {
+              url: '/tiles/layergroup/' + layergroupid + ':cb1/0/0/0/1.json.torque?auth_token=valid1',
+              method: 'GET',
+              headers: {host: 'localhost' },
+              encoding: 'binary'
+          }
+          var next = this;
+          assert.response(server, get_request, {},
+            function(res) { next(null, res); });
+        },
+        function checkCacheChannel(err, res) {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 200, 
+            'Unexpected error for authorized instance: '
+            + res.statusCode + ' -- ' + res.body);
+          assert.equal(res.headers['content-type'], "application/json; charset=utf-8");
+          var cc = res.headers['x-cache-channel'];
+          assert.ok(cc, "Missing X-Cache-Channel on fetch-after-restart");
+          assert.ok(cc.match, /ciao/, cc);
           return null;
         },
         function deleteTemplate(err)
@@ -1106,7 +1229,7 @@ suite('template_api', function() {
         },
         function checkTileDeleted(err, res) {
           if ( err ) throw err;
-          assert.equal(res.statusCode, 401, 
+          assert.equal(res.statusCode, 403,
             'Unexpected statusCode fetch tile after signature revokal: '
             + res.statusCode + ':' + res.body); 
           var parsed = JSON.parse(res.body);
@@ -1204,7 +1327,7 @@ suite('template_api', function() {
         function instanciateAuth(err, res)
         {
           if ( err ) throw err;
-          assert.equal(res.statusCode, 401,
+          assert.equal(res.statusCode, 403,
             'Unexpected success instanciating template with no auth: '
             + res.statusCode + ': ' + res.body);
           var parsed = JSON.parse(res.body);
@@ -1248,7 +1371,7 @@ suite('template_api', function() {
         },
         function fetchAttributeAuth(err, res) {
           if ( err ) throw err;
-          assert.equal(res.statusCode, 401,
+          assert.equal(res.statusCode, 403,
             'Fetching tile with no auth: ' + res.statusCode + ': ' + res.body);
           var parsed = JSON.parse(res.body);
           assert.ok(parsed.hasOwnProperty('error'),
@@ -1302,7 +1425,7 @@ suite('template_api', function() {
         },
         function checkTileDeleted(err, res) {
           if ( err ) throw err;
-          assert.equal(res.statusCode, 401, 
+          assert.equal(res.statusCode, 403, 
             'Unexpected statusCode fetch tile after signature revokal: '
             + res.statusCode + ':' + res.body); 
           var parsed = JSON.parse(res.body);
@@ -1393,6 +1516,7 @@ suite('template_api', function() {
               headers: {host: 'localhost', 'Content-Type': 'application/json' },
               data: JSON.stringify(template_params)
           }
+          helper.checkNoCache(res);
           var next = this;
           assert.response(server, post_request, {},
             function(res) { next(null, res); });
@@ -1464,13 +1588,16 @@ suite('template_api', function() {
           assert.response(server, post_request, {},
             function(res) { next(null, res); });
         },
-        function instanciateAuth(err, res)
+        function checkInstanciation(err, res)
         {
           if ( err ) throw err;
-          assert.equal(res.statusCode, 200,
-            'Unexpected success instanciating template with no auth: '
-            + res.statusCode + ': ' + res.body);
-          done();
+          assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+          // See https://github.com/CartoDB/Windshaft-cartodb/issues/176
+          helper.checkCache(res);
+          return null;
+        },
+        function finish(err) {
+          done(err);
         }
       );
     });
@@ -1535,12 +1662,12 @@ suite('template_api', function() {
           assert.response(server, post_request, {},
             function(res) { next(null, res); });
         },
-        function instanciateAuth(err, res)
+        function checkInstanciation(err, res)
         {
           if ( err ) throw err;
-          assert.equal(res.statusCode, 200,
-            'Unexpected success instanciating template with no auth: '
-            + res.statusCode + ': ' + res.body);
+          assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+          // See https://github.com/CartoDB/Windshaft-cartodb/issues/176
+          helper.checkNoCache(res);
           return null;
         },
         function finish(err) {

test/support/SQLAPIEmu.js

@@ -42,6 +42,9 @@ o.prototype.handleQuery = function(query, res) {
     if ( query.q.match('SQLAPIERROR') ) {
       res.statusCode = 400;
       res.write(JSON.stringify({'error':'Some error occurred'}));
+    } else if ( query.q.match('SQLAPINOANSWER') ) {
+      console.log("SQLAPIEmulator will never respond, on request");
+      return;
     } else if ( query.q.match('EPOCH.* as max') ) {
       // This is the structure of the known query sent by tiler
       var row = {

test/support/prepare_db.sh

@@ -76,7 +76,7 @@ if test x"$PREPARE_PGSQL" = xyes; then
     sed "s/:PUBLICPASS/${PUBLICPASS}/" | 
     sed "s/:TESTUSER/${TESTUSER}/" | 
     sed "s/:TESTPASS/${TESTPASS}/" | 
-    psql ${TEST_DB}
+    psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
 
 fi
 

test/support/test_helper.js

@@ -6,12 +6,12 @@
  */
 
 var _ = require('underscore');
+var assert = require('assert');
 var LZMA  = require('lzma/lzma_worker.js').LZMA;
 
 // set environment specific variables
-global.settings     = require(__dirname + '/../../config/settings');
 global.environment  = require(__dirname + '/../../config/environments/test');
-_.extend(global.settings, global.environment);
+process.env.NODE_ENV = 'test';
 
 
 // Utility function to compress & encode LZMA
@@ -28,7 +28,30 @@ function lzma_compress_to_base64(payload, mode, callback) {
   );
 }
 
-module.exports = {
-  lzma_compress_to_base64: lzma_compress_to_base64
+// Check that the response headers do not request caching
+// Throws on failure
+function checkNoCache(res) {
+  assert.ok(!res.headers.hasOwnProperty('x-cache-channel'));
+  assert.ok(!res.headers.hasOwnProperty('cache-control')); // is this correct ?
+  assert.ok(!res.headers.hasOwnProperty('last-modified')); // is this correct ?
 }
 
+
+/**
+ * Check that the response headers do not request caching
+ * @see checkNoCache
+ * @param res
+ */
+function checkCache(res) {
+    assert.ok(res.headers.hasOwnProperty('x-cache-channel'));
+    assert.ok(res.headers.hasOwnProperty('cache-control'));
+    assert.ok(res.headers.hasOwnProperty('last-modified'));
+}
+
+
+module.exports = {
+  lzma_compress_to_base64: lzma_compress_to_base64,
+  checkNoCache: checkNoCache,
+  checkCache: checkCache
+};
+

test/unit/cartodb/req2params.test.js

@@ -7,7 +7,12 @@ var assert = require('assert')
 suite('req2params', function() {
 
     // configure redis pool instance to use in tests
-    var opts = require('../../../lib/cartodb/server_options');
+    var opts = require('../../../lib/cartodb/server_options')();
+
+    var test_user = _.template(global.environment.postgres_auth_user, {user_id:1});
+    var test_pubuser = global.environment.postgres.user;
+    var test_database = test_user + '_db';
+
     
     test('can be found in server_options', function(){
       assert.ok(_.isFunction(opts.req2params));
@@ -20,8 +25,8 @@ suite('req2params', function() {
           assert.ok(!req.query.hasOwnProperty('dbuser'), 'dbuser was removed from query');
           assert.ok(req.hasOwnProperty('params'), 'request has params');
           assert.ok(req.params.hasOwnProperty('interactivity'), 'request params have interactivity');
-          assert.equal(req.params.dbname, 'test_cartodb_user_1_db', 'could forge dbname: '+ req.params.dbname);
-          assert.ok(req.params.dbuser === 'testpublicuser', 'could inject dbuser ('+req.params.dbuser+')');
+          assert.equal(req.params.dbname, test_database, 'could forge dbname: '+ req.params.dbname);
+          assert.ok(req.params.dbuser === test_pubuser, 'could inject dbuser ('+req.params.dbuser+')');
           done();
       });
     });
@@ -34,10 +39,8 @@ suite('req2params', function() {
           assert.ok(!req.query.hasOwnProperty('dbuser'), 'dbuser was removed from query');
           assert.ok(req.hasOwnProperty('params'), 'request has params');
           assert.ok(req.params.hasOwnProperty('interactivity'), 'request params have interactivity');
-          // database_name for user "localhost" (see test/support/prepare_db.sh)
-          assert.equal(req.params.dbname, 'test_cartodb_user_1_db');
-          // unauthenticated request gets no dbuser
-          assert.ok(req.params.dbuser === 'testpublicuser', 'could inject dbuser ('+req.params.dbuser+')');
+          assert.equal(req.params.dbname, test_database);
+          assert.ok(req.params.dbuser === test_pubuser, 'could inject dbuser ('+req.params.dbuser+')');
           done();
       });
     });
@@ -50,14 +53,12 @@ suite('req2params', function() {
           assert.ok(!req.query.hasOwnProperty('dbuser'), 'dbuser was removed from query');
           assert.ok(req.hasOwnProperty('params'), 'request has params');
           assert.ok(req.params.hasOwnProperty('interactivity'), 'request params have interactivity');
-          // database_name for user "localhost" (see test/support/prepare_db.sh)
-          assert.equal(req.params.dbname, 'test_cartodb_user_1_db');
-          // id for user "localhost" (see test/support/prepare_db.sh)
-          assert.equal(req.params.dbuser, 'test_cartodb_user_1');
+          assert.equal(req.params.dbname, test_database);
+          assert.equal(req.params.dbuser, test_user);
  
           opts.req2params({headers: { host:'localhost' }, query: {map_key: '1235'} }, function(err, req) {
               // wrong key resets params to no user
-              assert.ok(req.params.dbuser === 'testpublicuser', 'could inject dbuser ('+req.params.dbuser+')');
+              assert.ok(req.params.dbuser === test_pubuser, 'could inject dbuser ('+req.params.dbuser+')');
               done();
           });
       });

test/unit/cartodb/template_maps.test.js

@@ -420,5 +420,87 @@ suite('template_maps', function() {
     catch (e) { err = e; }
     assert.ok(!err);
   });
+
+  // Can set a limit on the number of user templates
+  test('can limit number of user templates', function(done) {
+    var tmap = new TemplateMaps(redis_pool, signed_maps, {
+      max_user_templates: 2
+    });
+    assert.ok(tmap);
+    var tpl = { version:'0.0.1', auth: {}, layergroup: {} };
+    var expectErr = false;
+    var idMe = [];
+    var idYou = [];
+    Step(
+      function oneForMe() {
+        tpl.name = 'oneForMe';
+        tmap.addTemplate('me', tpl, this);
+      },
+      function twoForMe(err, id) {
+        if ( err ) throw err;
+        assert.ok(id);
+        idMe.push(id);
+        tpl.name = 'twoForMe';
+        tmap.addTemplate('me', tpl, this);
+      },
+      function threeForMe(err, id) {
+        if ( err ) throw err;
+        assert.ok(id);
+        idMe.push(id);
+        tpl.name = 'threeForMe';
+        expectErr = true;
+        tmap.addTemplate('me', tpl, this);
+      },
+      function errForMe(err, id) {
+        if ( err && ! expectErr ) throw err;
+        expectErr = false;
+        assert.ok(err);
+        assert.ok(err.message.match(/limit.*template/), err);
+        return null;
+      },
+      function delOneMe(err) {
+        if ( err ) throw err;
+        tmap.delTemplate('me', idMe.shift(), this);
+      },
+      function threeForMeRetry(err) {
+        if ( err ) throw err;
+        tpl.name = 'threeForMe';
+        tmap.addTemplate('me', tpl, this);
+      },
+      function oneForYou(err, id) {
+        if ( err ) throw err;
+        assert.ok(id);
+        idMe.push(id);
+        tpl.name = 'oneForYou';
+        tmap.addTemplate('you', tpl, this);
+      },
+      function twoForYou(err, id) {
+        if ( err ) throw err;
+        assert.ok(id);
+        idYou.push(id);
+        tpl.name = 'twoForYou';
+        tmap.addTemplate('you', tpl, this);
+      },
+      function threeForYou(err, id) {
+        if ( err ) throw err;
+        assert.ok(id);
+        idYou.push(id);
+        tpl.name = 'threeForYou';
+        expectErr = true;
+        tmap.addTemplate('you', tpl, this);
+      },
+      function errForYou(err, id) {
+        if ( err && ! expectErr ) throw err;
+        expectErr = false;
+        assert.ok(err);
+        assert.ok(err.message.match(/limit.*template/), err);
+        return null;
+      },
+      function finish(err) {
+        // TODO: delete all templates
+        done(err);
+      }
+    );
+  });
     
 });

tools/examples/mapconfig_simple.js

@@ -0,0 +1,11 @@
+{"version":"1.0.1",
+	"layers":[{
+    "type":"cartodb",
+	  "options":{
+      "sql":"select 1 as id, ST_SetSRID(ST_MakePoint(0,0),3857) as the_geom_webmercator",
+	    "cartocss":"#style{ marker-width: 12;}",
+	    "cartocss_version":"2.1.1",
+      "Interactivity":"id"
+    }
+  }]
+}

tools/examples/mapconfig_slow.js

@@ -0,0 +1,11 @@
+{"version":"1.0.1",
+	"layers":[{
+    "type":"cartodb",
+	  "options":{
+      "sql":"select 1 as id, ST_Transform(ST_SetSRID(ST_MakePoint(x/1000,x/2000),4326),3857) as the_geom_webmercator FROM generate_series(-170000,170000) x",
+	    "cartocss":"#style{ marker-width: 12;}",
+	    "cartocss_version":"2.1.1",
+      "Interactivity":"id"
+    }
+  }]
+}

tools/examples/mapconfig_torque.js

@@ -0,0 +1,10 @@
+{"version":"1.0.1",
+	"layers":[{
+    "type":"torque",
+	  "options":{
+      "sql":"select 1 as id, ST_SetSRID(ST_MakePoint(0,0),3857) as the_geom_webmercator",
+	    "cartocss":"Map{ -torque-time-attribute:'id'; -torque-aggregation-function:'count(id)'; -torque-frame-count:2; -torque-resolution:2}",
+      "cartocss_version": "2.1.1"
+    }
+  }]
+}

tools/examples/template_simple.js

@@ -0,0 +1,17 @@
+{
+ "version":"0.0.1",
+ "name":"simple",
+ "placeholders":{},
+ "auth":{ "method":"open" },
+ "layergroup":{
+   "version":"1.0.1",
+    "layers":[{
+      "type":"cartodb",
+      "options":{
+        "sql":"select ST_SetSRID(ST_MakePoint(0,0),3857) as the_geom_webmercator",
+        "cartocss":"#s{ marker-width: 12;}",
+        "cartocss_version":"2.1.1"
+      }
+    }]
+  }
+}

tools/flush_cache

@@ -0,0 +1,59 @@
+#!/usr/bin/env node
+
+var path = require('path');
+var request    = require('request');
+
+function usage(me, exitcode) {
+  console.log("Usage: " + me + " [--env <environment>] <username> <tablename>");
+  process.exit(exitcode);
+}
+
+var node_path = process.argv.shift(); 
+var script_path = process.argv.shift(); 
+var basedir = path.dirname(script_path);
+var me = path.basename(script_path);
+
+var ENV = 'development.js';
+var username, table;
+var arg;
+while ( arg = process.argv.shift() ) {
+  if ( arg == '--env' ) {
+    ENV = process.argv.shift();
+  }
+  else if ( ! username ) {
+    username = arg;
+  }
+  else if ( ! table ) {
+    table = arg;
+  }
+  else {
+    console.warn("Unused parameter " + arg);
+  }
+}
+
+if ( ! table ) {
+  usage(me, 1);
+}
+
+global.environment = require('../config/environments/' + ENV);
+
+// _after_ setting global.environment
+var serverOptions = require('../lib/cartodb/server_options');
+
+var host = global.environment.host;
+var port = global.environment.port;
+var re = ''+serverOptions.re_userFromHost;
+var hostname = re.replace(/^\/\^/, '')
+                 .replace(/\/$$/, '')
+                 .replace(/\\/g,'')
+                 .replace(/\([^)]*\)/,username)
+              ;
+//console.log("re: " + re);
+//console.log("hostname: " + hostname);
+
+var url = 'http://' + host + ':' + port + '/tiles/' + table + '/flush_cache';
+request.del({ url: url, headers: { host: hostname } },
+function(err, res, body) {
+  if ( err ) throw err;
+  console.log(res.body);
+});

tools/instanciate_template

@@ -49,5 +49,5 @@ if test x${verbose} = xyes; then
   echo "${res}"
 fi
 
-tok=`echo "$res" | sed 's/.*"template_id":"\([^"]*\)".*/\1/'`
+tok=`echo "$res" | sed 's/.*"layergroupid":"\([^"]*\)".*/\1/'`
 echo $tok

tools/list_templates

@@ -0,0 +1,45 @@
+#!/bin/sh
+
+verbose=no
+tiler_url=http://dev.localhost.lan:8181/tiles/template
+apikey=${CDB_APIKEY}
+
+while test -n "$1"; do
+  if test "$1" = "-v"; then
+    verbose=yes
+  elif test "$1" = "-k"; then
+    shift
+    apikey="$1"
+  elif test "$1" = "-u"; then
+    shift
+    tiler_url="$1"
+  elif test "$1" = "-h" -o "$1" = "-?"; then
+    echo "Usage: $0 [-v] [-k <api_key>] [-u <tiler_url>]" >&2
+    echo "Default <tiler_url> is ${tiler_url}" >&2
+    echo "Default <api_key> is read from CDB_APIKEY env variable" >&2
+    exit 0
+  else
+    echo "Unused parameter $1" >&2
+  fi
+  shift
+done
+
+
+cmd="curl -X GET -sk ${tiler_url}?api_key=${apikey}"
+if test x${verbose} = xyes; then
+  cmd="${cmd} -v"
+fi
+res=`${cmd}`
+if test $? -gt 0; then
+  echo "curl command failed: ${cmd}"
+fi
+
+if test x${verbose} = xyes; then
+  echo "${res}"
+fi
+
+node <<EOF
+var parsed = JSON.parse('$res');
+console.dir(parsed);
+EOF
+

tools/munin/windshaft

@@ -68,7 +68,7 @@ for pid in ${pids}; do
   log=$(grep "${pid}" "${tmpreport}" | grep -w 1w | awk '{print $9}')
   if test -e "${log}"; then
     kill -USR2 "${pid}"
-    cnt=$(tac ${log} | sed -n -e '/ItemKey/p;/^RenderCache/q' | wc -l)
+    cnt=$(tac ${log} | sed -n -e '/ItemKey/p;/ RenderCache /q' | wc -l)
     if test $cnt -gt $maxcache; then maxcache=$cnt; fi
   else
     # report the error...

tools/show_style

@@ -38,7 +38,8 @@ if ( ! username ) usage(me, 1);
 console.log("Using environment " + ENV);
 
 global.environment = require('../config/environments/' + ENV);
-var serverOptions = require('../lib/cartodb/server_options'); // _after_ setting global.environment
+// _after_ setting global.environment
+var serverOptions = require('../lib/cartodb/server_options')();
 
 var client;
 var dbname;