Release 1.8.5

.gitignore

@@ -5,5 +5,3 @@ config/environments/*.js
 tools/munin/windshaft.conf
 logs/
 pids/
-redis.pid
-test.log

.travis.yml

@@ -1,29 +1,15 @@
 before_install:
-  - sudo mv /etc/apt/sources.list.d/pgdg-source.list* /tmp
-  - sudo apt-get -qq purge postgis* postgresql*
-  - sudo apt-add-repository --yes ppa:cartodb/postgresql-9.3
-  - sudo apt-add-repository --yes ppa:cartodb/gis
-  - sudo rm -Rf /var/lib/postgresql /etc/postgresql
-  - sudo apt-add-repository --yes ppa:mapnik/nightly-2.3
-  - sudo apt-get update
-  - sudo apt-get install -y postgresql-9.3-postgis-2.1
-  - sudo apt-get install -y postgresql-contrib-9.3
-  - sudo apt-get install -q libprotobuf-dev protobuf-compiler
+  - sudo apt-add-repository --yes ppa:mapnik/v2.1.0
+  - sudo apt-get update -q
   - sudo apt-get install -q libmapnik-dev
-  - sudo apt-get install -q mapnik-input-plugin-gdal mapnik-input-plugin-ogr mapnik-input-plugin-postgis
-  - sudo apt-get install -y gdal-bin
-  - echo -e "local\tall\tall\ttrust\nhost\tall\tall\t127.0.0.1/32\ttrust\nhost\tall\tall\t::1/128\ttrust" |sudo tee /etc/postgresql/9.3/main/pg_hba.conf
-  - sudo service postgresql restart
   - createdb template_postgis
   - psql -c "CREATE EXTENSION postgis" template_postgis
-
-before_script:
   # Tell npm to use known registrars:
   # see http://blog.npmjs.org/post/78085451721/npms-self-signed-certificate-is-no-more
   - npm config set ca ""
 
 env:
-  - NPROCS=1 JOBS=1 PGUSER=postgres
+  - NPROCS=1 JOBS=1
 
 language: node_js
 node_js:

HOWTO_RELEASE

@@ -1,20 +1,13 @@
-1. Test (make clean all check), fix if broken before proceeding
-2. Ensure proper version in package.json 
-3. Ensure NEWS section exists for the new version, review it, add release date
-4. Drop npm-shrinkwrap.json
-5. Run npm shrinkwrap to recreate npm-shrinkwrap.json
-6. Commit package.json, npm-shrinwrap.json, NEWS
-7. git tag -a Major.Minor.Patch # use NEWS section as content
-8. Announce on cartodb@googlegroups.com
-9. Stub NEWS/package for next version
+1. Ensure proper version in package.json 
+2. Ensure NEWS section exists for the new version, review it, add release date
+3. Drop npm-shrinkwrap.json
+4. Run npm install
+5. Test (make check or npm test), fix if broken before proceeding
+6. Run npm shrinkwrap
+7. Set "from" in npm-shrinkwrap.json for known packages
+   (windshaft, node-varnish, grainstore...)
+8. Commit package.json, npm-shrinwrap.json, NEWS
+9. git tag -a Major.Minor.Patch # use NEWS section as content
+10. Announce
+11. Stub NEWS/package for next version
 
-Versions:
-
-Bugfix releases increment Patch component of version.
-Feature releases increment Minor and set Patch to zero.
-If backward compatibility is broken, increment Major and
-set to zero Minor and Patch.
-
-Branches named 'b<Major>.<Minor>' are kept for any critical
-fix that might need to be shipped before next feature release
-is ready.

LICENCE

@@ -0,0 +1,27 @@
+Copyright (c) 2011, Vizzuality 
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+3. All advertising materials mentioning features or use of this software
+   must display the following acknowledgement:
+   This product includes software developed by Vizzuality.
+4. Neither the name of Vizzuality nor the
+   names of its contributors may be used to endorse or promote products
+   derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS "AS IS" AND ANY
+EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

LICENSE

@@ -1,27 +0,0 @@
-Copyright (c) 2014, Vizzuality
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without 
-modification, are permitted provided that the following conditions are met:
-
-1. Redistributions of source code must retain the above copyright notice, this 
-list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright notice, 
-this list of conditions and the following disclaimer in the documentation 
-and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its contributors 
-may be used to endorse or promote products derived from this software without 
-specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

NEWS.md

@@ -1,195 +1,3 @@
-1.18.2 -- 2014-10-13
---------------------
-
-Bug fixes:
- - Defaults resultSet to object if undefined in QueryTablesApi
-
-Announcements:
- - Upgrades windshaft to 0.28.1
-
-
-1.18.1 -- 2014-10-13
---------------------
-
-New features:
- - Allow to add more node.js' threadpool workers via process.env.UV_THREADPOOL_SIZE
-
-
-1.18.0 -- 2014-10-03
---------------------
-
-Announcements:
- - Comes back to use mapnik 2.3.x based on cartodb/node-mapnik@1.4.15-cdb from windshaft@0.28.0
-
-
-1.17.2 -- 2014-10-01
---------------------
-
-Announcements:
- - Upgrades windshaft to 0.27.2 which downgrades node-mapnik to 0.7.26-cdb1
-
-
-1.17.1 -- 2014-09-30
---------------------
-
-Announcements:
- - Upgrades windshaft to 0.27.1 which downgrades node-mapnik to 1.4.10
-
-Enhancements:
- - TTL for template locks so they are not kept forever
- - Upgrades mocha
-
-
-1.17.0 -- 2014-09-25
---------------------
-
-New features:
- - Starts using mapnik 2.3.x
-
-Enhancements:
- - Upgrades windshaft and cartodb-redis
- - Supports `!scale_denominator!` dynamic param in SQL queries
- - Metrics revamp: removes and adds some metrics
- - Adds poolSize configuration for mapnik
-
-1.16.1 -- 2014-08-19
---------------------
-
-Enhancements:
- - Upgrades cartodb-redis
-
-1.16.0 -- 2014-08-18
---------------------
-
-New features:
- - Configurable QueryTablesAPI to call directly postgresql using cartodb-psql
-   or to keep using a request to the SQL API
-
-Enhancements:
- - Removes mapnik dependency as it now relies on Windshaft to check mapnik version
- - Upgrades dependencies:
-    - underscore
-    - lzma
-    - log4js
-    - rollbar
-    - windshaft
-    - request
-
-1.15.0 -- 2014-08-13
---------------------
-Enhancements:
- - Upgrades dependencies:
-    - redis-mpool
-    - cartodb-redis
-    - windshaft
- - Specifies name in the redis pool
- - Slow pool configuration in example configurations
-
-
-1.14.0 -- 2014-08-07
---------------------
-
-Enhancements:
- - SQL API requests moved to its own entity
-
-New features:
- - Affected tables and last updated time for a query are performed in a single
-   request to the SQL API
- - Allow specifying the tile format, upgrades windshaft and grainstore
-   dependencies for this matter
-
-
-1.13.1 -- 2014-08-04
---------------------
-
-Enhancements:
- - Profiler header sent as JSON string
-
-
-1.13.0 -- 2014-07-30
---------------------
-
-New features:
- - Support for postgresql schemas
- - Use public user from redis
- - Support for several auth tokens
-
-1.12.1 -- 2014-06-24
---------------------
-
-Enhancements:
- - Caches layergroup and sets X-Cache-Channel in GET requests also in named maps
-
-1.12.0 -- 2014-06-24
---------------------
-
-New features:
- - Caches layergroup and sets X-Cache-Channel in GET requests
-
-1.11.1 -- 2014-05-07
---------------------
-
-Enhancements:
-
- - Upgrade Windshaft to 0.21.0, see
- http://github.com/CartoDB/Windshaft/blob/0.21.0/NEWS
-
-1.11.0 -- 2014-04-28
---------------------
-
-New features:
-
- - Add support for log_filename directive
- - Reopen log file on SIGHUP, for better logrotate integration
-
-Enhancements:
-
- - Set default PostgreSQL application name to "cartodb_tiler"
-
-1.10.2 -- 2014-04-08
---------------------
-
-Bug fixes:
-
- - Fix show_style tool broken since 1.8.1
- - Fix X-Cache-Channel of tiles accessed via signed token (#188)
-
-1.10.1 -- 2014-03-21
---------------------
-
-Bug fixes:
-
- - Do not cache non-success jsonp responses (#186)
-
-1.10.0 -- 2014-03-20
---------------------
-
-New features:
-
- - Add optional support for rollbar (#150)
-
-Enhancements:
-
- - Do not send connection details to client (#183)
- - Upgrade node-varnish to 0.3.0
- - Upgrade Windshaft to 0.20.0, see
-   http://github.com/CartoDB/Windshaft/blob/0.20.0/NEWS
- - Include tiler version in startup log
- - Install an uncaught exception handler
- - Require own fork of node-mapnik, with temptative fix
-   for libxml usage (glibc detected corruptions)
-
-Other changes:
-
- - Switch to 3-clause BSD license (#184)
-
-1.9.0 -- 2014-03-10
--------------------
-
-New features:
-
- - Allow to set server related configuration in serverMetadata (#182)
-
 1.8.5 -- 2014-03-10
 -------------------
 

app.js

@@ -7,10 +7,6 @@
  * environments: [development, production]
  */
 
-var path = require('path'),
-    fs = require('fs')
-;
-
 
 if ( process.argv[2] ) ENV = process.argv[2];
 else if ( process.env['NODE_ENV'] ) ENV = process.env['NODE_ENV'];
@@ -25,49 +21,22 @@ if (ENV != 'development' && ENV != 'production' && ENV != 'staging' ){
     process.exit(1);
 }
 
-var _ = require('underscore');
-
+var _ = require('underscore')
+    , Step       = require('step')
+    ;
 // set environment specific variables
+global.settings     = require(__dirname + '/config/settings');
 global.environment  = require(__dirname + '/config/environments/' + ENV);
-global.environment.api_hostname = require('os').hostname().split('.')[0];
+_.extend(global.settings, global.environment);
 
 global.log4js = require('log4js')
-log4js_config = {
-  appenders: [],
-  replaceConsole:true
-};
-
-if (global.environment.uv_threadpool_size) {
-    process.env.UV_THREADPOOL_SIZE = global.environment.uv_threadpool_size;
-}
-
-if ( global.environment.log_filename ) {
-  var logdir = path.dirname(global.environment.log_filename);
-  // See cwd inlog4js.configure call below
-  logdir = path.resolve(__dirname, logdir);
-  if ( ! fs.existsSync(logdir) ) {
-    console.error("Log filename directory does not exist: " + logdir);
-    process.exit(1);
-  }
-  console.log("Logs will be written to " + global.environment.log_filename);
-  log4js_config.appenders.push(
-    { type: "file", filename: global.environment.log_filename }
-  );
-} else {
-  log4js_config.appenders.push(
+log4js.configure({
+  appenders: [
     { type: "console", layout: { type:'basic' } }
-  );
-}
+  ],
+  replaceConsole:true
+});
 
-if ( global.environment.rollbar ) {
-  log4js_config.appenders.push({
-    type: __dirname + "/lib/cartodb/log4js_rollbar.js",
-    options: global.environment.rollbar
-  });
-}
-
-log4js.configure(log4js_config, { cwd: __dirname });
-global.logger = log4js.getLogger();
 
 // Include cartodb_windshaft only _after_ the "global" variable is set
 // See https://github.com/Vizzuality/Windshaft-cartodb/issues/28
@@ -84,10 +53,8 @@ ws.maxConnections = global.environment.maxConnections || 128;
 
 ws.listen(global.environment.port, global.environment.host);
 
-var version = require("./package").version;
-
 ws.on('listening', function() {
-  console.log("Windshaft tileserver " + version + " started on "
+  console.log("Windshaft tileserver started on "
               + global.environment.host + ':' + global.environment.port
               + " (" + ENV + ")");
 });
@@ -102,11 +69,3 @@ process.on('SIGUSR2', function() {
   ws.dumpCacheStats();
 });
 
-process.on('SIGHUP', function() {
-  log4js.configure(log4js_config);
-  console.log('Log files reloaded');
-});
-
-process.on('uncaughtException', function(err) {
-  logger.error('Uncaught exception: ' + err.stack); 
-});

config/environments/development.js.example

@@ -2,7 +2,6 @@ var config = {
      environment: 'development'
     ,port: 8181
     ,host: '127.0.0.1'
-    ,uv_threadpool_size: undefined
     // Regular expression pattern to extract username
     // from hostname. Must have a single grabbing block.
     ,user_from_host: '^(.*)\\.localhost'
@@ -32,15 +31,11 @@ var config = {
     // to be able to navigate the map without a reload ?
     // Defaults to 7200 (2 hours)
     ,mapConfigTTL: 7200
-    // idle socket timeout, in milliseconds 
+    // idle socket timeout, in miliseconds 
     ,socket_timeout: 600000
     ,enable_cors: true
     ,cache_enabled: false
     ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
-    // If log_filename is given logs will be written
-    // there, in append mode. Otherwise stdout is used (default).
-    // Log file will be re-opened on receiving the HUP signal
-    ,log_filename: 'logs/node-windshaft.log'
     // Templated database username for authorized user
     // Supported labels: 'user_id' (read from redis)
     ,postgres_auth_user: 'development_cartodb_user_<%= user_id %>'
@@ -63,18 +58,9 @@ var config = {
         */
         row_limit: 65535,
         simplify_geometries: true,
-        /*
-         * Set persist_connection to false if you want
-         * database connections to be closed on renderer
-         * expiration (1 minute after last use).
-         * Setting to true (the default) would never
-         * close any connection for the server's lifetime
-         */
-        persist_connection: false,
         max_size: 500
     }
     ,mapnik_version: undefined
-    ,mapnik_tile_format: 'png8:m=h'
     ,statsd: {
         host: 'localhost',
         port: 8125,
@@ -98,17 +84,13 @@ var config = {
         // Max number of connections in each pool.
         // Users will be put on a queue when the limit is hit.
         // Set to maxConnection to have no possible queues.
-        // There are currently 2 pools involved in serving
+        // There are currently 3 pools involved in serving
         // windshaft-cartodb requests so multiply this number
-        // by 2 to know how many possible connections will be
+        // by 3 to know how many possible connections will be
         // kept open by the server. The default is 50.
         max: 50,
         idleTimeoutMillis: 1, // idle time before dropping connection
-        reapIntervalMillis: 1, // time between cleanups
-        slowPool: {
-            log: true, // whether a slow acquire must be logged or not
-            elapsedThreshold: 25 // the threshold to determine an slow acquire must be reported or not
-        }
+        reapIntervalMillis: 1 // time between cleanups
     }
     ,sqlapi: {
         protocol: 'http',

config/environments/production.js.example

@@ -2,7 +2,6 @@ var config = {
     environment: 'production'
     ,port: 8181
     ,host: '127.0.0.1'
-    ,uv_threadpool_size: undefined
     // Regular expression pattern to extract username
     // from hostname. Must have a single grabbing block.
     ,user_from_host: '^(.*)\\.cartodb\\.com$'
@@ -32,15 +31,11 @@ var config = {
     // to be able to navigate the map without a reload ?
     // Defaults to 7200 (2 hours)
     ,mapConfigTTL: 7200
-    // idle socket timeout, in milliseconds
+    // idle socket timeout, in miliseconds
     ,socket_timeout: 600000 
     ,enable_cors: true
     ,cache_enabled: true
     ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
-    // If log_filename is given logs will be written
-    // there, in append mode. Otherwise stdout is used (default).
-    // Log file will be re-opened on receiving the HUP signal
-    ,log_filename: 'logs/node-windshaft.log'
     // Templated database username for authorized user
     // Supported labels: 'user_id' (read from redis)
     ,postgres_auth_user: 'cartodb_user_<%= user_id %>'
@@ -56,19 +51,10 @@ var config = {
         port: 6432,
         extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
         row_limit: 65535,
-        /*
-         * Set persist_connection to false if you want
-         * database connections to be closed on renderer
-         * expiration (1 minute after last use).
-         * Setting to true (the default) would never
-         * close any connection for the server's lifetime
-         */
-        persist_connection: false,
         simplify_geometries: true,
         max_size: 500
     }
     ,mapnik_version: undefined
-    ,mapnik_tile_format: 'png8:m=h'
     ,statsd: {
         host: 'localhost',
         port: 8125,
@@ -92,17 +78,13 @@ var config = {
         // Max number of connections in each pool.
         // Users will be put on a queue when the limit is hit.
         // Set to maxConnection to have no possible queues.
-        // There are currently 2 pools involved in serving
+        // There are currently 3 pools involved in serving
         // windshaft-cartodb requests so multiply this number
-        // by 2 to know how many possible connections will be
+        // by 3 to know how many possible connections will be
         // kept open by the server. The default is 50.
         max: 50,
         idleTimeoutMillis: 30000, // idle time before dropping connection
-        reapIntervalMillis: 1000, // time between cleanups
-        slowPool: {
-            log: true, // whether a slow acquire must be logged or not
-            elapsedThreshold: 25 // the threshold to determine an slow acquire must be reported or not
-        }
+        reapIntervalMillis: 1000 // time between cleanups
     }
     ,sqlapi: {
         protocol: 'https',
@@ -134,21 +116,6 @@ var config = {
     // X-Tiler-Profile header containing elapsed timing for various 
     // steps taken for producing the response.
     ,useProfiler:false
-    ,serverMetadata: {
-      cdn_url: {
-        http: 'api.cartocdn.com',
-        https: 'cartocdn.global.ssl.fastly.net'
-      }
-    }
-    // Optional rollbar support
-    ,rollbar: {
-      token: 'secret',
-      // See http://github.com/rollbar/node_rollbar#configuration-reference
-      options: {
-        endpoint: 'https://api.rollbar.com/api/1/',
-        handler: 'inline'
-      }
-    }
 };
 
 module.exports = config;

config/environments/staging.js.example

@@ -2,7 +2,6 @@ var config = {
     environment: 'production'
     ,port: 8181
     ,host: '127.0.0.1'
-    ,uv_threadpool_size: undefined
     // Regular expression pattern to extract username
     // from hostname. Must have a single grabbing block.
     ,user_from_host: '^(.*)\\.cartodb\\.com$'
@@ -32,15 +31,11 @@ var config = {
     // to be able to navigate the map without a reload ?
     // Defaults to 7200 (2 hours)
     ,mapConfigTTL: 7200
-    // idle socket timeout, in milliseconds
+    // idle socket timeout, in miliseconds
     ,socket_timeout: 600000 
     ,enable_cors: true
     ,cache_enabled: true
     ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms (:res[X-Tiler-Profiler]) -> :res[Content-Type]'
-    // If log_filename is given logs will be written
-    // there, in append mode. Otherwise stdout is used (default).
-    // Log file will be re-opened on receiving the HUP signal
-    ,log_filename: 'logs/node-windshaft.log'
     // Templated database username for authorized user
     // Supported labels: 'user_id' (read from redis)
     ,postgres_auth_user: 'cartodb_staging_user_<%= user_id %>'
@@ -57,18 +52,9 @@ var config = {
         extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
         row_limit: 65535,
         simplify_geometries: true,
-        /*
-         * Set persist_connection to false if you want
-         * database connections to be closed on renderer
-         * expiration (1 minute after last use).
-         * Setting to true (the default) would never
-         * close any connection for the server's lifetime
-         */
-        persist_connection: false,
         max_size: 500
     }
     ,mapnik_version: undefined
-    ,mapnik_tile_format: 'png8:m=h'
     ,statsd: {
         host: 'localhost',
         port: 8125,
@@ -92,17 +78,13 @@ var config = {
         // Max number of connections in each pool.
         // Users will be put on a queue when the limit is hit.
         // Set to maxConnection to have no possible queues.
-        // There are currently 2 pools involved in serving
+        // There are currently 3 pools involved in serving
         // windshaft-cartodb requests so multiply this number
-        // by 2 to know how many possible connections will be
+        // by 3 to know how many possible connections will be
         // kept open by the server. The default is 50.
         max: 50,
         idleTimeoutMillis: 30000, // idle time before dropping connection
-        reapIntervalMillis: 1000, // time between cleanups
-        slowPool: {
-            log: true, // whether a slow acquire must be logged or not
-            elapsedThreshold: 25 // the threshold to determine an slow acquire must be reported or not
-        }
+        reapIntervalMillis: 1000 // time between cleanups
     }
     ,sqlapi: {
         protocol: 'https',
@@ -134,21 +116,6 @@ var config = {
     // X-Tiler-Profile header containing elapsed timing for various 
     // steps taken for producing the response.
     ,useProfiler:true
-    ,serverMetadata: {
-      cdn_url: {
-        http: 'api.cartocdn.com',
-        https: 'cartocdn.global.ssl.fastly.net'
-      }
-    }
-    // Optional rollbar support
-    ,rollbar: {
-      token: 'secret',
-      // See http://github.com/rollbar/node_rollbar#configuration-reference
-      options: {
-        endpoint: 'https://api.rollbar.com/api/1/',
-        handler: 'inline'
-      }
-    }
 };
 
 module.exports = config;

config/environments/test.js.example

@@ -2,7 +2,6 @@ var config = {
      environment: 'test'
     ,port: 8888
     ,host: '127.0.0.1'
-    ,uv_threadpool_size: undefined
     // Regular expression pattern to extract username
     // from hostname. Must have a single grabbing block.
     ,user_from_host: '(.*)'
@@ -32,15 +31,11 @@ var config = {
     // to be able to navigate the map without a reload ?
     // Defaults to 7200 (2 hours)
     ,mapConfigTTL: 7200
-    // idle socket timeout, in milliseconds
+    // idle socket timeout, in miliseconds
     ,socket_timeout: 600000 
     ,enable_cors: true
     ,cache_enabled: false
     ,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
-    // If log_filename is given logs will be written
-    // there, in append mode. Otherwise stdout is used (default).
-    // Log file will be re-opened on receiving the HUP signal
-    //,log_filename: 'logs/node-windshaft.log'
     // Templated database username for authorized user
     // Supported labels: 'user_id' (read from redis)
     ,postgres_auth_user: 'test_windshaft_cartodb_user_<%= user_id %>'
@@ -57,18 +52,9 @@ var config = {
         extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
         row_limit: 65535,
         simplify_geometries: true,
-        /*
-         * Set persist_connection to false if you want
-         * database connections to be closed on renderer
-         * expiration (1 minute after last use).
-         * Setting to true (the default) would never
-         * close any connection for the server's lifetime
-         */
-        persist_connection: false,
         max_size: 500
     }
     ,mapnik_version: ''
-    ,mapnik_tile_format: 'png8:m=h'
     ,statsd: {
         host: 'localhost',
         port: 8125,
@@ -92,17 +78,13 @@ var config = {
         // Max number of connections in each pool.
         // Users will be put on a queue when the limit is hit.
         // Set to maxConnection to have no possible queues.
-        // There are currently 2 pools involved in serving
+        // There are currently 3 pools involved in serving
         // windshaft-cartodb requests so multiply this number
-        // by 2 to know how many possible connections will be
+        // by 3 to know how many possible connections will be
         // kept open by the server. The default is 50.
         max: 50,
         idleTimeoutMillis: 1, // idle time before dropping connection
-        reapIntervalMillis: 1, // time between cleanups
-        slowPool: {
-            log: true, // whether a slow acquire must be logged or not
-            elapsedThreshold: 25 // the threshold to determine an slow acquire must be reported or not
-        }
+        reapIntervalMillis: 1 // time between cleanups
     }
     ,sqlapi: {
         protocol: 'http',

config/settings.js

@@ -0,0 +1 @@
+module.exports.oneDay   = 86400000;

docs/Map-API-internal.md

@@ -1,111 +0,0 @@
-# Kind of maps
-
-Windshaft-CartoDB supports these kind of maps:
-
- - [Temporary maps](#temporary-maps) (created by anyone)
-   - [Detached maps](#detached-maps)
-   - [Inline maps](#inline-maps) (legacy)
- - [Persistent maps](#peristent-maps) (created by CartDB user)
-   - [Template maps](#template-maps)
-   - [Table maps](#table-maps) (legacy, deprecated) 
-
-## Temporary maps
-
-Temporary maps have no owners and are anonymous in nature.
-There are two kind of temporary maps:
-
- - Detached maps (aka MultiLayer-API)
- - Inline maps 
-
-### Detached maps
-
-Detached maps are maps which are configured with a request
-obtaining a temporary token and then used by referencing
-the obtained token. The token expires automatically when unused.
-
-Anyone can create detached maps, but users will need read access
-to the data source of the map layers.
-
-The configuration format is a [MapConfig]
-(http://github.com/CartoDB/Windshaft/wiki/MapConfig-specification) document.
-
-The HTTP endpoints for creating the map and using it are described [here]
-(http://github.com/CartoDB/Windshaft-cartodb/wiki/MultiLayer-API)
-
-*TODO* cleanup the referenced document
-
-### Inline maps
-
-Inline maps are maps that only exist for a single request,
-being the request for a specific map resource (tile).
-
-Inline maps are always bound to a table, and can only be
-obtained by those having read access to the that table.
-Additionally, users need to have access to any datasource
-specified as part of the configuration.
-
-Inline maps only support PNG and UTF8GRID tiles.
-
-The configuration consist in a set of parameters, to be
-specified in the query string of the tile request:
-
- * sql - the query to run as datasource, can be an array
- * style - the CartoCSS style for the datasource, can be an array
- * style_version - version of the CartoCSS style, can be an array
- * interactivity - only for fetching UTF8GRID, 
-
-If the style is not provided, style of the associated table is
-used; if the sql is not provided, all records of the associated
-table are used as the datasource; the two possibilities result
-in a mix between _inline_ maps and [Table maps][].
-
-*TODO* specify (or link) api endpoints
-
-## Persistent maps
-
-Persistent maps can only be created by a CartoDB user who has full
-responsibility over editing and deleting them. There are two
-kind of persistent maps:
-
- - Template maps 
- - Table maps (legacy, deprecated) 
-
-### Templated maps
-
-Templated maps are templated [MapConfig]
-(http://github.com/CartoDB/Windshaft/wiki/MapConfig-specification) documents
-associated with an authorization certificate.
-
-The authorization certificate determines who can instanciate the
-template and use the resulting map. Authorized users of the instanciated
-maps will have the same database access privilege of the template owner.
-
-The HTTP endpoints for creating and using templated maps are described [here]
-(http://github.com/CartoDB/Windshaft-cartodb/wiki/Template-maps).
-
-*TODO* cleanup the referenced document
-
-### Table maps
-
-Table maps are maps associated with a table.
-Configuration of such maps is limited to the CartoCSS style.
-
- * style - the CartoCSS style for the datasource, can be an array
- * style_version - version of the CartoCSS style, can be an array
-
-You can only fetch PNG or UTF8GRID tiles from these maps.
-
-Access method is the same as the one for [Inline maps](#inline-maps)
-
-# Endpoints description 
-
-- **/api/maps/** (same interface than https://github.com/CartoDB/Windshaft/wiki/Multilayer-API)
-- **/api/maps/named** (same interface than https://github.com/CartoDB/Windshaft-cartodb/wiki/Template-maps)
-
-
-NOTE: in case Multilayer-API does not contain this info yet, the
-      endpoint for fetching attributes is this:
-
-- **/api/maps/:map_id/:layer_index/attributes/:feature_id** 
-   - would return { c: 1, d: 2 }
-

docs/Map-API.md

@@ -1,637 +1,111 @@
-## Maps API
-
-The CartoDB Maps API allows you to generate maps based on data hosted in your CartoDB account and style them using CartoCSS. The API generates a XYZ based URL to fetch Web Mercator projected tiles using web clients like Leaflet, Google Maps, OpenLayers.
-
-You can create two types of maps with the Maps API:
-
-- **Anonymous maps**  
-  Maps that can be created using your CartoDB public data. Any client can change the read-only SQL and CartoCSS parameters that generate the map tiles. These maps can be created from a JavaScript application alone and no authenticated calls are needed. See [this CartoDB.js example]({{ '/cartodb-platform/cartodb-js.html' | prepend: site.baseurl }}).
-
-- **Named maps**  
-  Maps that access to your private data. These maps require an owner to setup and modify any SQL and CartoCSS parameters and are not modifiable without new setup calls. 
-
-## Quickstart
+# Kind of maps
 
-### Anonymous maps
+Windshaft-CartoDB supports these kind of maps:
 
-Here is an example of how to create an anonymous map with JavaScript:
+ - [Temporary maps](#temporary-maps) (created by anyone)
+   - [Detached maps](#detached-maps)
+   - [Inline maps](#inline-maps) (legacy)
+ - [Persistent maps](#peristent-maps) (created by CartDB user)
+   - [Template maps](#template-maps)
+   - [Table maps](#table-maps) (legacy, deprecated) 
 
-{% highlight javascript %}
-var mapconfig = {
-  "version": "1.0.1",
-  "layers": [{
-    "type": "cartodb",
-    "options": {
-      "cartocss_version": "2.1.1",
-      "cartocss": "#layer { polygon-fill: #FFF; }",
-      "sql": "select * from european_countries_e"
-    }
-  }]
-}
+## Temporary maps
 
-$.ajax({
-  crossOrigin: true,
-  type: 'POST',
-  dataType: 'json',
-  contentType: 'application/json',
-  url: 'http://documentation.cartodb.com/api/v1/map',
-  data: JSON.stringify(mapconfig),
-  success: function(data) {
-    var templateUrl = 'http://documentation.cartodb.com/api/v1/map/' + data.layergroupid + '{z}/{x}/{y}.png'
-    console.log(templateUrl);
-  }
-})
-{% endhighlight %}
+Temporary maps have no owners and are anonymous in nature.
+There are two kind of temporary maps:
 
-### Named maps
+ - Detached maps (aka MultiLayer-API)
+ - Inline maps 
 
-Let's create a named map using some private tables in a CartoDB account.
-The following API call creates a map of European countries that have a white fill color:
+### Detached maps
 
-{% highlight javascript %}
-// mapconfig.json
-{
-  "version": "0.0.1",
-  "name": "test",
-  "auth": {
-    "method": "open"
-  },
-  "layergroup": {
-    "layers": [{
-      "type": "cartodb",
-      "options": {
-        "cartocss_version": "2.1.1",
-        "cartocss": "#layer { polygon-fill: #FFF; }",
-        "sql": "select * from european_countries_e"
-      }
-    }]
-  }
-}
-{% endhighlight %}
+Detached maps are maps which are configured with a request
+obtaining a temporary token and then used by referencing
+the obtained token. The token expires automatically when unused.
 
-The map config needs to be sent to CartoDB's Map API using an authenticated call. Here we use a command line tool called `curl`. For more info about this tool see [this blog post](http://quickleft.com/blog/command-line-tutorials-curl) or type ``man curl`` in bash. Using `curl` the call would look like:
+Anyone can create detached maps, but users will need read access
+to the data source of the map layers.
 
-<div class="code-title notitle code-request"></div>
-{% highlight bash %}
-curl 'https://{account}.cartodb.com/api/v1/map/named?api_key=APIKEY' -H 'Content-Type: application/json' -d @mapconfig.json
-{% endhighlight %}
+The configuration format is a [MapConfig]
+(http://github.com/CartoDB/Windshaft/wiki/MapConfig-specification) document.
 
-To get the `URL` to fetch the tiles you need to instantiate the map.
+The HTTP endpoints for creating the map and using it are described [here]
+(http://github.com/CartoDB/Windshaft-cartodb/wiki/MultiLayer-API)
 
-<div class="code-title notitle code-request"></div>
-{% highlight bash %}
-curl 'http://{account}.cartodb.com/api/v1/map/named/test' -H 'Content-Type: application/json'
-{% endhighlight %}
+*TODO* cleanup the referenced document
 
-The response will return JSON with properties for the `layergroupid` and the timestamp (`last_updated`) of the last data modification. 
+### Inline maps
 
-Here is an example response:
+Inline maps are maps that only exist for a single request,
+being the request for a specific map resource (tile).
 
-{% highlight javascript %}
-{
-  "layergroupid": "c01a54877c62831bb51720263f91fb33:0",
-  "last_updated": "1970-01-01T00:00:00.000Z"
-}
-{% endhighlight %}
+Inline maps are always bound to a table, and can only be
+obtained by those having read access to the that table.
+Additionally, users need to have access to any datasource
+specified as part of the configuration.
 
-You can use the `layergroupid` to instantiate a URL template for accessing tiles on the client. Here we use the `layergroupid` from the example response above in this URL template:
+Inline maps only support PNG and UTF8GRID tiles.
 
-{% highlight bash %}
-http://documentation.cartodb.com/tiles/layergroup/c01a54877c62831bb51720263f91fb33:0/{z}/{x}/{y}.png
-{% endhighlight %}
+The configuration consist in a set of parameters, to be
+specified in the query string of the tile request:
 
-## General Concepts
+ * sql - the query to run as datasource, can be an array
+ * style - the CartoCSS style for the datasource, can be an array
+ * style_version - version of the CartoCSS style, can be an array
+ * interactivity - only for fetching UTF8GRID, 
 
-The following concepts are the same for every endpoint in the API except when it's noted explicitly.
+If the style is not provided, style of the associated table is
+used; if the sql is not provided, all records of the associated
+table are used as the datasource; the two possibilities result
+in a mix between _inline_ maps and [Table maps][].
 
-### Auth
+*TODO* specify (or link) api endpoints
 
-By default, users do not have access to private tables in CartoDB. In order to instantiate a map from private table data an API Key is required. Additionally, to include some endpoints an API Key must be included (e.g. creating a named map).
+## Persistent maps
 
-To execute an authorized request, api_key=YOURAPIKEY should be added to the request URL. The param can be also passed as POST param. We **strongly advise** using HTTPS when you are performing requests that include your `api_key`.
+Persistent maps can only be created by a CartoDB user who has full
+responsibility over editing and deleting them. There are two
+kind of persistent maps:
 
-### Errors
+ - Template maps 
+ - Table maps (legacy, deprecated) 
 
-Errors are reported using standard HTTP codes and extended information encoded in JSON with this format:
+### Templated maps
 
-{% highlight javascript %}
-{
-  "errors": [
-    "access forbidden to table TABLE"
-  ]
-}
-{% endhighlight %}
+Templated maps are templated [MapConfig]
+(http://github.com/CartoDB/Windshaft/wiki/MapConfig-specification) documents
+associated with an authorization certificate.
 
-If you use JSONP, the 200 HTTP code is always returned so the JavaScript client can receive errors from the JSON object.
+The authorization certificate determines who can instanciate the
+template and use the resulting map. Authorized users of the instanciated
+maps will have the same database access privilege of the template owner.
 
-### CORS support
+The HTTP endpoints for creating and using templated maps are described [here]
+(http://github.com/CartoDB/Windshaft-cartodb/wiki/Template-maps).
 
-All the endpoints which might be accessed using a web browser add CORS headers and allow OPTIONS method.
+*TODO* cleanup the referenced document
 
-## Anonymous Maps
+### Table maps
 
-Anonymous maps allows you to instantiate a map given SQL and CartoCSS. It also allows you to add interaction capabilities using [UTF Grid.](https://github.com/mapbox/utfgrid-spec)
+Table maps are maps associated with a table.
+Configuration of such maps is limited to the CartoCSS style.
 
-### Instantiate
+ * style - the CartoCSS style for the datasource, can be an array
+ * style_version - version of the CartoCSS style, can be an array
 
-#### Definition
+You can only fetch PNG or UTF8GRID tiles from these maps.
 
-<div class="code-title notitle code-request"></div>
-{% highlight html %}
-POST /api/v1/map
-{% endhighlight %}
+Access method is the same as the one for [Inline maps](#inline-maps)
 
-#### Params
+# Endpoints description 
 
-{% highlight javascript %}
-{
-  "version": "1.0.1",
-  "layers": [{
-    "type": "cartodb",
-    "options": {
-      "cartocss_version": "2.1.1", 
-      "cartocss": "#layer { polygon-fill: #FFF; }",
-      "sql": "select * from european_countries_e",
-      "interactivity": ["cartodb_id", "iso3"]
-    }
-  }]
-}
-{% endhighlight %}
+- **/api/maps/** (same interface than https://github.com/CartoDB/Windshaft/wiki/Multilayer-API)
+- **/api/maps/named** (same interface than https://github.com/CartoDB/Windshaft-cartodb/wiki/Template-maps)
 
-Should be a [Mapconfig](https://github.com/CartoDB/Windshaft/blob/0.19.1/doc/MapConfig-1.1.0.md).
 
-#### Response
+NOTE: in case Multilayer-API does not contain this info yet, the
+      endpoint for fetching attributes is this:
 
-The response includes:
+- **/api/maps/:map_id/:layer_index/attributes/:feature_id** 
+   - would return { c: 1, d: 2 }
 
-- **layergroupid**  
-  The ID for that map, used to compose the URL for the tiles. The final URL is:
-
-  {% highlight html %}
-  http://{account}.cartodb.com/api/v1/map/:layergroupid/{z}/{x}/{y}.png
-  {% endhighlight %}
-
-- **updated_at**  
-  The ISO date of the last time the data involved in the query was updated.
-
-- **metadata** *(optional)*  
-  Includes information about the layers. Some layers may not have metadata.
-
-- **cdn_url**  
-  URLs to fetch the data using the best CDN for your zone.
-
-#### Example
-
-<div class="code-title code-request with-result">REQUEST</div>
-{% highlight bash %}
-curl 'http://documentation.cartodb.com/api/v1/map' -H 'Content-Type: application/json' -d @mapconfig.json
-{% endhighlight %}
-
-<div class="code-title">RESPONSE</div>
-{% highlight javascript %}
-{
-  "layergroupid":"c01a54877c62831bb51720263f91fb33:0",
-  "last_updated":"1970-01-01T00:00:00.000Z"
-  "cdn_url": {
-    "http": "http://cdb.com",
-    "https": "https://cdb.com"
-  }
-}
-{% endhighlight %}
-
-The tiles can be accessed using:
-
-{% highlight bash %}
-http://documentation.cartodb.com/api/v1/map/c01a54877c62831bb51720263f91fb33:0/{z}/{x}/{y}.png
-{% endhighlight %}
-
-For UTF grid tiles:
-
-{% highlight bash %}
-http://documentation.cartodb.com/api/v1/map/c01a54877c62831bb51720263f91fb33:0/:layer/{z}/{x}/{y}.grid.json
-{% endhighlight %}
-
-For attributes defined in `attributes` section:
-
-{% highlight bash %}
-http://documentation.cartodb.com/api/v1/map/c01a54877c62831bb51720263f91fb33:0/:layer/attributes/:feature_id
-{% endhighlight %}
-
-Which returns JSON with the attributes defined, like:
-
-{% highlight javascript %}
-{ c: 1, d: 2 }
-{% endhighlight %}
-
-Notice UTF Grid and attributes endpoints need an intenger parameter, ``layer``. That number is the 0-based index of the layer inside the mapconfig. So in this case 0 returns the UTF grid tiles/attributes for layer 0, the only layer in the example mapconfig. If a second layer was available it could be returned with 1, a third layer with 2, etc.
-
-### Create JSONP
-
-The JSONP endpoint is provided in order to allow web browsers access which don't support CORS.
-
-#### Definition
-
-<div class="code-title notitle code-request"></div>
-{% highlight bash %}
-GET /api/v1/map?callback=method
-{% endhighlight %}
-
-#### Params
-
-- **auth_token** *(optional)*  
-  If the named map needs authorization.
-
-- **config**  
-  Encoded JSON with the params for creating named maps (the variables defined in the template).
-
-- **lmza**  
-  This attribute contains the same as config but LZMA compressed. It cannot be used at the same time as `config`.
-
-- **callback**  
-  JSON callback name.
-
-#### Example
-
-<div class="code-title code-request with-result">REQUEST</div>
-{% highlight bash %}
-curl http://...
-{% endhighlight %}
-
-<div class="code-title">RESPONSE</div>
-{% highlight javascript %}
-{
-}
-{% endhighlight %}
-
-### Remove
-
-Anonymous maps cannot be removed by an API call. They will expire after about five minutes but sometimes longer. If an anonymous map expires and tiles are requested from it, an error will be raised. This could happen if a user leaves a map open and after time returns to the map an attempts to interact with it in a way that requires new tiles (e.g. zoom). The client will need to go through the steps of creating the map again to fix the problem.
-
-## Named Maps
-
-Named maps are essentially the same as anonymous maps but the mapconfig is stored in the server and given a unique name. Two other big differences are that you can created named maps from private data and that users without an API Key can see them even though they are from that private data. 
-
-The main two differences compared to anonymous maps are:
-
-- **auth layer**  
-  This allows you to control who is able to see the map based on a token auth
-
-- **templates**  
-  Since the mapconfig is static it can contain some variables so the client con modify the map appearance using those variables.
-
-Template maps are persistent with no preset expiration. They can only be created or deleted by a CartoDB user with a valid API_KEY (see auth section).
-
-### Create
-
-#### Definition
-
-<div class="code-title notitle code-request"></div>
-{% highlight html %}
-POST /api/v1/map/named
-{% endhighlight %}
-
-#### Params
-
-<div class="code-title">template.json</div>
-{% highlight javascript %}
-{
-  "version": "0.0.1",
-  "name": "template_name",
-  "auth": {
-    "method": "token",
-    "valid_tokens": [
-      "auth_token1",
-      "auth_token2"
-    ]
-  },
-  "placeholders": {
-    "color": {
-      "type": "css_color",
-      "default": "red"
-    },
-    "cartodb_id": {
-      "type": "number",
-      "default": 1
-    }
-  },
-  "layergroup": {
-    "version": "1.0.1",
-    "layers": [
-      {
-        "type": "cartodb",
-        "options": {
-          "cartocss_version": "2.1.1",
-          "cartocss": "#layer { polygon-fill: <%= color %>; }",
-          "sql": "select * from european_countries_e WHERE cartodb_id = <%= cartodb_id %>"
-        }
-      }
-    ]
-  }
-}
-{% endhighlight %}
-
-##### Arguments
-
-- **name**: there can be at most 1 template with the same name for any user valid names start with a letter and only contains letter, numbers or underscores
-- **auth**:
-  - **method** `"token"` or `"open"` (the default if no `"method"` is given)
-- **placeholders**: Variables not listed here are not substituted. Variable not provided at instantiation time trigger an error. A default is required for optional variables. Type specification is used for quoting, to avoid injections see template format section below.
-- **layergroup**: the layer list definition. This is the MapConfig explained in anonymous maps see https://github.com/CartoDB/Windshaft/blob/master/doc/MapConfig-1.1.0.md
-
-#### Template Format
-
-A templated `layergroup` allows using placeholders in the "cartocss" and "sql" elements of the "option" object in any "layer" of a layergroup configuration
-
-Valid placeholder names start with a letter and can only contain letters, numbers or underscores. They have to be written between `<%=` and `%>` strings in order to be replaced.
-
-##### Example
-
-{% highlight javascript %}
-<%= my_color %>
-{% endhighlight %}
-
-The set of supported placeholders for a template will need to be explicitly defined with a specific type and default value for each.
-
-#### Placeholder Types
-
-The placeholder type will determine the kind of escaping for the associated value. Supported types are:
-
-- **sql_literal** internal single-quotes will be sql-escaped
-- **sql_ident** internal double-quotes will be sql-escaped
-- **number** can only contain numerical representation
-- **css_color** can only contain color names or hex-values
-
-Placeholder default values will be used whenever new values are not provided as options at the time of creation on the client. They can also be used to test the template by creating a default version with now options provided.
-
-When using templates, be very careful about your selections as they can give broad access to your data if they are defined losely.
-
-<div class="code-title code-request with-result">REQUEST</div>
-{% highlight html %}
-curl -X POST \
-   -H 'Content-Type: application/json' \
-   -d @template.json \
-   'https://docs.cartodb.com/api/v1/map/named?api_key=APIKEY'
-{% endhighlight %}
-
-<div class="code-title">RESPONSE</div>
-{% highlight javascript %}
-{
-  "templateid":"name",
-}
-{% endhighlight %}
-
-### Instantiate
-
-Instantiating a map allows you to get the information needed to fetch tiles. That temporal map is an anonymous map.
-
-#### Definition
-
-<div class="code-title notitle code-request"></div>
-{% highlight html %}
-POST /api/v1/map/named/:template_name
-{% endhighlight %}
-
-#### Param
-
-{% highlight javascript %}
-// params.json
-{
- color: "#ff0000",
- cartodb_id: 3
-}
-{% endhighlight %}
-
-The fields you pass as `params.json` depend on the variables allowed by the named map. If there are variables missing it will raise an error (HTTP 400)
-
-- **auth_token** *optional* if the named map needs auth
-
-#### Example
-
-You can initialize a template map by passing all of the required parameters in a POST to `/api/v1/map/named/:template_name`.
-
-Valid credentials will be needed if required by the template.
-
-<div class="code-title code-request with-result">REQUEST</div>
-{% highlight bash %}
-curl -X POST \
-  -H 'Content-Type: application/json' \
-  -d @params.json \
-  'https://docs.cartodb.com/api/v1/template/@template_name?auth_token=AUTH_TOKEN'
-{% endhighlight %}
-
-<div class="code-title">Response</div>
-{% highlight javascript %}
-{
-  "layergroupid": "docs@fd2861af@c01a54877c62831bb51720263f91fb33:123456788",
-  "last_updated": "2013-11-14T11:20:15.000Z"
-}
-{% endhighlight %}
-
-<div class="code-title">Error</div>
-{% highlight javascript %}
-{
-  "error": "Some error string here"
-}
-{% endhighlight %}
-
-You can then use the `layergroupid` for fetching tiles and grids as you would normally (see anonymous map section).  However, you'll need to show the `auth_token`, if required by the template.
-
-### Using JSONP
-
-There is also a special endpoint to be able to initialize a map using JSONP (for old browsers).
-
-#### Definition
-
-<div class="code-title notitle code-request"></div>
-{% highlight bash %}
-GET /api/v1/map/named/:template_name/jsonp
-{% endhighlight %}
-
-#### Params
-
-- **auth_token** *(optional)* If the named map needs auth
-- **config** Encoded JSON with the params for creating named maps (the variables defined in the template)
-- **lmza** This attribute contains the same as config but LZMA compressed. It cannot be used at the same time than `config`.
-- **callback:** JSON callback name
-
-<div class="code-title code-request with-result">REQUEST</div>
-{% highlight bash %}
-curl 'https://docs.cartodb.com/api/v1/map/named/:template_name/jsonp?auth_token=AUTH_TOKEN&callback=function_name&config=template_params_json'
-{% endhighlight %}
-
-<div class="code-title">RESPONSE</div>
-{% highlight javascript %}
-callback(
-  "layergroupid":"c01a54877c62831bb51720263f91fb33:0",
-  "last_updated":"1970-01-01T00:00:00.000Z"
-  "cdn_url": {
-    "http": "http://cdb.com",
-    "https": "https://cdb.com"
-  }
-)
-{% endhighlight %}
-
-This takes the `callback` function (required), `auth_token` if the template needs auth, and `config` which is the variable for the template (in cases where it has variables). 
-
-{% highlight javascript %}
-url += "config=" + encodeURIComponent(
-JSON.stringify({ color: 'red' });
-{% endhighlight %}
-
-The response is in this format:
-
-{% highlight javascript %}
-jQuery17205720721024554223_1390996319118({
-  layergroupid: "dev@744bd0ed9b047f953fae673d56a47b4d:1390844463021.1401",
-  last_updated: "2014-01-27T17:41:03.021Z"
-})
-{% endhighlight %}
-
-### Update
-
-#### Definition
-
-<div class="code-title notitle code-request"></div>
-{% highlight bash %}
-PUT /api/v1/map/:map_name
-{% endhighlight %}
-
-#### Params
-
-Same params used to create a map.
-
-#### Response
-
-Same as updating a map.
-
-#### Other Info
-
-Updating a named map removes all the named map instances so they need to be initialized again.
-
-#### Example
-
-<div class="code-title code-request with-result">REQUEST</div>
-{% highlight bash %}
-curl -X PUT \
-  -H 'Content-Type: application/json' \
-  -d @template.json \
-  'https://docs.cartodb.com/tiles/template/:template_name?api_key=APIKEY'
-{% endhighlight %}
-
-<div class="code-title">RESPONSE</div>
-{% highlight javascript %}
-{
-  "template_id": "@template_name"
-}
-{% endhighlight %}
-
-If any template has the same name, it will be updated.
-
-If a template with the same name does NOT exist, a 400 HTTP response is generated with an error in this format:
-
-{% highlight javascript %}
-{
-  "error": "error string here"
-}
-{% endhighlight %}
-
-Updating a template map will also remove all signatures from previously initialized maps. 
-
-### Delete 
-
-Delete the specified template map from the server and disables any previously initialized versions of the map.
-
-#### Definition
-
-<div class="code-title notitle code-request"></div>
-{% highlight bash %}
-DELETE /template/:template_name
-{% endhighlight %}
-
-#### Example
-
-<div class="code-title code-request">REQUEST</div>
-{% highlight bash %}
-curl -X DELETE 'https://docs.cartodb.com/tiles/template/@template_name?auth_token=AUTH_TOKEN'
-{% endhighlight %}
-
-<div class="code-title">RESPONSE</div>
-{% highlight javascript %}
-{
-  "error": "Some error string here"
-}
-{% endhighlight %}
-
-On success, a 204 (No Content) response would be issued. Otherwise a 4xx response with with an error will be returned:
-
-### Listing Available Templates
-
-This allows you to get a list of all available templates. 
-
-#### Definition
-
-<div class="code-title notitle code-request"></div>
-{% highlight bash %}
-GET /api/v1/map/named/
-{% endhighlight %}
-
-#### Params
-
-- **api_key** is required
-
-#### Example
-
-<div class="code-title code-request with-result">REQUEST</div>
-{% highlight bash %}
-curl -X GET 'https://docs.cartodb.com/tiles/template?api_key=APIKEY'
-{% endhighlight %}
-
-<div class="code-title with-result">RESPONSE</div>
-{% highlight javascript %}
-{
-   "template_ids": ["@template_name1","@template_name2"]
-}
-{% endhighlight %}
-
-<div class="code-title">ERROR</div>
-{% highlight javascript %}
-{
-   "error": "Some error string here"
-}
-{% endhighlight %}
-
-### Getting a Specific Template
-
-This gets the definition of a template
-
-#### Definition
-
-<div class="code-title notitle code-request"></div>
-{% highlight bash %}
-GET /api/v1/map/named/:template_name
-{% endhighlight %}
-
-#### Params
-
-- **api_key** is required
-
-#### Example
-
-<div class="code-title code-request with-result">REQUEST</div>
-{% highlight bash %}
-curl -X GET 'https://docs.cartodb.com/tiles/template/@template_name?auth_token=AUTH_TOKEN'
-{% endhighlight %}
-
-<div class="code-title with-result">RESPONSE</div>
-{% highlight javascript %}
-{
-  "template": {...} // see template.json above
-}
-{% endhighlight %}
-
-<div class="code-title">ERROR</div>
-{% highlight javascript %}
-{
-  "error": "Some error string here"
-}
-{% endhighlight %}

docs/MultiLayer-API.md

@@ -1,6 +1,6 @@
 The Windshaft-CartoDB MultiLayer API extends the [Windshaft MultiLayer API](https://github.com/Vizzuality/Windshaft/wiki/Multilayer-API) in a few ways.
 
-## Last modification timestamp embedded in the token
+## Last modification timestamps
 
 It encodes a timestamp of 'last modification time' into the map token (token:EPOCH) returned to the client.
 It accepts tokens with encoded timestamp from the client considering the token suffix as a cache_buster value.
@@ -8,21 +8,8 @@ It accepts tokens with encoded timestamp from the client considering the token s
 Clients don't need to be aware of the extension but rather use the API as they would use the base one.
 The only difference will be that the _same_ layergroup configuration may result in different tokens if source data was modified between the mapview requests. 
 
-## Additional attributes in the response object
-
-Windshaft-CartoDB adds the following attributes in the response object
-
-- ``last_update`` field with ISO format (2013-11-30T12:23:10).
-- ``cdn_url`` object containing CDN url client should use (not mandatory) to access the tiles. It's in the form:
-
-   ```json
-   {
-     http: 'http://cdn_url.com/'
-     https: 'https://secure.cdn_url.com/'
-   }
-   ```
-
+Also Windshaft-CartoDB adds a ``last_update`` field with ISO format (2013-11-30T12:23:10).
 
 ## Stats tag
 
-Windshaft-CartoDB adds support for a ``stat_tag`` element in the multilayer configuration to help [stats](Redis-stats-format) gathering.
+Windshaft-CartoDB adds support for a ``stat_tag`` element in the multilayer configuration to help [stats](Redis-stats-format) gathering.

docs/Template-maps.md

@@ -49,17 +49,16 @@ certificate that would be used to sign any instance of the template.
 ```js
 // template.json 
 {
-  version: "0.0.1",
+  version: '0.0.1',
   // there can be at most 1 template with the same name for any user 
   // valid names start with a letter and only contains letter, numbers
   // or underscores
-  name: "template_name", 
+  name: 'template_name', 
   // embedded authorization certificate
   auth: {
    // See https://github.com/CartoDB/Windshaft-cartodb/wiki/Signed-maps
-   method: "token", // or "open" (the default if no "method" is given)
-   // only (required and non empty) for "token" method
-   valid_tokens: ["auth_token1","auth_token2"]
+   method: 'token', // or "open" (the default if no "method" is given)
+   valid_tokens: ['auth_token1','auth_token2'] // only (required and non empty) for 'token' method
   },
   // Variables not listed here are not substituted
   // Variable not provided at instantiation time trigger an error
@@ -67,11 +66,11 @@ certificate that would be used to sign any instance of the template.
   // Type specification is used for quoting, to avoid injections
   placeholders: {
       color: {
-        type:"css_color",
-        default:"red"
+        type:'css_color',
+        default:'red'
       },
       cartodb_id: {
-        type:"number",
+        type:'number',
         default: 1
       }
   },

docs/metrics.md

@@ -1,46 +0,0 @@
-Windshaft-cartodb metrics
-=========================
-See [Windshaft metrics documentation](https://github.com/CartoDB/Windshaft/blob/master/doc/metrics.md) to understand the full picture.
-
-The next list includes the API endpoints, each endpoint may have several inner timers, some of them are displayed within this list as subitems. Find the description for them in the Inner timers section.
-## Timers
-- **windshaft-cartodb.flush_cache**: time to flush the tile and sql cache
-- **windshaft-cartodb.get_template**: time to retrieve an specific template
-- **windshaft-cartodb.delete_template**: time to delete an specific template
-- **windshaft-cartodb.get_template_list**: time to retrieve the list of owned templates
-- **windshaft-cartodb.instance_template_post**: time to create a template via HTTP POST
-- **windshaft-cartodb.instance_template_get**: time to create a template via HTTP GET
-    + TemplateMaps_instance
-    + createLayergroup
-
-There are some endpoints that are not being tracked:
-- Adding a template
-- Updating a template
-
-### Inner timers
-Again, each inner timer may have several inner timers.
-
-- **addCacheChannel**: time to add X-Cache-Channel header based on table last modifications
-- **LZMA decompress**: time to decompress request params with LZMA
-- **TemplateMaps_instance**: time to retrieve a map template instance, see *getTemplate* and *authorizedByCert*
-- **affectedTables**: time to check what are the affected tables for adding the cache channel, see *addCacheChannel*
-- **authorize**: time to authorize a request, see *authorizedByAPIKey*, *authorizedByCert*, *authorizedBySigner*
-- **authorizedByAPIKey**: time to authorize using an API KEY
-- **authorizedByCert**: time to authorize a request by a cert, see [signed map](https://github.com/CartoDB/Windshaft-cartodb/wiki/Signed-maps)
-- **authorizedBySigner**: time to authorize a request for a [signed map](https://github.com/CartoDB/Windshaft-cartodb/wiki/Signed-maps)
-- **findLastUpdated**: time to retrieve the last update time for a list of tables, see *affectedTables*
-- **fingerPrint**: time to create a fingerprint for a signed map
-- **generateCacheChannel**: time to generate the headers for the cache channel based on the request, see *addCacheChannel*
-- **getSignerMapKey**: time to retrieve from redis the authorized key for a signed map
-- **getTablePrivacy**: time to retrieve from redis the privacy of a table
-- **getTemplate**: time to retrieve from redis the template for a map
-- **getUserMapKey**: time to retrieve from redis the user key for a map
-- **incMapviewCount**: time to incremenent in redis the map views
-- **mapStore_load**: time to retrieve from redis a map configuration
-- **req2params.setup**: time to prepare the params from a request, see *req2params* in Windshaft documentation
-- **setDBAuth**: time to retrieve from redis and set db user and db password from a user
-- **setDBConn**: time to retrieve from redis and set db host and db name from a user
-- **setDBParams**: time to prepare all db params to be able to connect/query a database, see *setDBAuth* and *setDBConn*
-- **signMap**: time to sign in redis layergroup for a map, see signed maps
-- **tablePrivacy_getUserDBName**: time to retrieve from redis the database for a user
-

lib/cartodb/api/query_tables_api.js

@@ -1,122 +0,0 @@
-var sqlApi = require('../sql/sql_api'),
-    PSQL   = require('cartodb-psql');
-
-function QueryTablesApi() {
-}
-
-var affectedTableRegexCache = {
-    bbox: /!bbox!/g,
-    scale_denominator: /!scale_denominator!/g,
-    pixel_width: /!pixel_width!/g,
-    pixel_height: /!pixel_height!/g
-};
-
-module.exports = QueryTablesApi;
-
-QueryTablesApi.prototype.getLastUpdatedTime = function (username, api_key, tableNames, callback) {
-    var sql = 'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max FROM CDB_TableMetadata m WHERE m.tabname = any (ARRAY['+
-        tableNames.map(function(t) { return "'" + t + "'::regclass"; }).join(',') +
-        '])';
-
-    // call sql api
-    sqlApi.query(username, api_key, sql, function(err, rows){
-        if (err){
-            var msg = err.message ? err.message : err;
-            callback(new Error('could not find last updated timestamp: ' + msg));
-            return;
-        }
-        // when the table has not updated_at means it hasn't been changed so a default last_updated is set
-        var last_updated = 0;
-        if(rows.length !== 0) {
-            last_updated = rows[0].max || 0;
-        }
-
-        callback(null, last_updated*1000);
-    });
-};
-
-QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, options, sql, callback) {
-
-    var query = 'SELECT CDB_QueryTables($windshaft$' + prepareSql(sql) + '$windshaft$)';
-
-    runQuery(username, options, query, handleAffectedTablesInQueryRows, callback);
-};
-
-function handleAffectedTablesInQueryRows(err, rows, callback) {
-    if (err){
-        var msg = err.message ? err.message : err;
-        callback(new Error('could not fetch source tables: ' + msg));
-        return;
-    }
-    var qtables = rows[0].cdb_querytables;
-    var tableNames = qtables.split(/^\{(.*)\}$/)[1];
-    tableNames = tableNames ? tableNames.split(',') : [];
-    callback(null, tableNames);
-}
-
-QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (username, options, sql, callback) {
-
-    var query = [
-        'WITH querytables AS (',
-            'SELECT * FROM CDB_QueryTables($windshaft$' + prepareSql(sql) + '$windshaft$) as tablenames',
-        ')',
-        'SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max',
-        'FROM CDB_TableMetadata m',
-        'WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])'
-    ].join(' ');
-
-    runQuery(username, options, query, handleAffectedTablesAndLastUpdatedTimeRows, callback);
-};
-
-function handleAffectedTablesAndLastUpdatedTimeRows(err, rows, callback) {
-    if (err || rows.length === 0) {
-        var msg = err.message ? err.message : err;
-        callback(new Error('could not fetch affected tables and last updated time: ' + msg));
-        return;
-    }
-
-    var result = rows[0];
-
-    var tableNames = result.tablenames.split(/^\{(.*)\}$/)[1];
-    tableNames = tableNames ? tableNames.split(',') : [];
-
-    var lastUpdatedTime = result.max || 0;
-
-    callback(null, {
-        affectedTables: tableNames,
-        lastUpdatedTime: lastUpdatedTime * 1000
-    });
-}
-
-
-function runQuery(username, options, query, queryHandler, callback) {
-    if (shouldQueryPostgresDirectly()) {
-        var psql = new PSQL(options);
-        psql.query(query, function(err, resultSet) {
-            resultSet = resultSet || {};
-            var rows = resultSet.rows || [];
-            queryHandler(err, rows, callback);
-        });
-    } else {
-        sqlApi.query(username, options.api_key, query, function(err, rows) {
-            queryHandler(err, rows, callback);
-        });
-    }
-}
-
-
-function prepareSql(sql) {
-    return sql
-        .replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)')
-        .replace(affectedTableRegexCache.scale_denominator, '0')
-        .replace(affectedTableRegexCache.pixel_width, '1')
-        .replace(affectedTableRegexCache.pixel_height, '1')
-    ;
-}
-
-
-function shouldQueryPostgresDirectly() {
-    return global.environment
-        && global.environment.enabledFeatures
-        && global.environment.enabledFeatures.cdbQueryTablesFromPostgres;
-}

lib/cartodb/cartodb_windshaft.js

@@ -2,7 +2,7 @@
 var _ = require('underscore')
     , Step       = require('step')
     , Windshaft = require('windshaft')
-    , redisPool = require('redis-mpool')(_.extend(global.environment.redis, {name: 'windshaft:cartodb'}))
+    , redisPool = new require('redis-mpool')(global.environment.redis)
     // TODO: instanciate cartoData with redisPool
     , cartoData  = require('cartodb-redis')(global.environment.redis)
     , SignedMaps = require('./signed_maps.js')
@@ -11,9 +11,6 @@ var _ = require('underscore')
     , os = require('os')
 ;
 
-if ( ! process.env['PGAPPNAME'] )
-  process.env['PGAPPNAME']='cartodb_tiler';
-
 var CartodbWindshaft = function(serverOptions) {
    var debug = global.environment.debug;
 
@@ -79,14 +76,9 @@ var CartodbWindshaft = function(serverOptions) {
       var that = this;
       var thatArgs = arguments;
       var statusCode;
-      if ( res._windshaftStatusCode ) {
-        // Added by our override of sendError
-        statusCode = res._windshaftStatusCode;
-      } else {
-        if ( args.length > 2 ) statusCode = args[2];
-        else {
-          statusCode = args[1] || 200;
-        }
+      if ( args.length > 2 ) statusCode = args[2];
+      else {
+        statusCode = args[1] || 200;
       }
       var req = res.req;
       Step (
@@ -125,18 +117,13 @@ var CartodbWindshaft = function(serverOptions) {
       );
     };
 
-    var ws_sendError = ws.sendError;
-    ws.sendError = function() {
-      var res = arguments[0];
-      var statusCode = arguments[2];
-      res._windshaftStatusCode = statusCode;
-      ws_sendError.apply(this, arguments);
-    };
-
     /**
      * Helper to allow access to the layer to be used in the maps infowindow popup.
      */
     ws.get(serverOptions.base_url + '/infowindow', function(req, res){
+        if ( req.profiler && req.profiler.statsd_client ) {
+          req.profiler.start('windshaft-cartodb.get_infowindow');
+        }
         ws.doCORS(res);
         Step(
             function(){
@@ -158,6 +145,9 @@ var CartodbWindshaft = function(serverOptions) {
      * Helper to allow access to metadata to be used in embedded maps.
      */
     ws.get(serverOptions.base_url + '/map_metadata', function(req, res){
+        if ( req.profiler && req.profiler.statsd_client ) {
+          req.profiler.start('windshaft-cartodb.get_map_metadata');
+        }
         ws.doCORS(res);
         Step(
             function(){
@@ -234,7 +224,8 @@ var CartodbWindshaft = function(serverOptions) {
         },
         function finish(err, response){
             if ( req.profiler ) {
-              res.header('X-Tiler-Profiler', req.profiler.toJSONString());
+              var report = req.profiler.toString();
+              res.header('X-Tiler-Profiler', report);
             }
             if (err){
                 response = { error: ''+err };
@@ -290,7 +281,8 @@ var CartodbWindshaft = function(serverOptions) {
         },
         function finish(err, response){
             if ( req.profiler ) {
-              res.header('X-Tiler-Profiler', req.profiler.toJSONString());
+              var report = req.profiler.toString();
+              res.header('X-Tiler-Profiler', report);
             }
             if (err){
                 var statusCode = 400;
@@ -486,6 +478,7 @@ var CartodbWindshaft = function(serverOptions) {
     // Instantiate a template
     function instanciateTemplate(req, res, template_params, callback) {
       ws.doCORS(res);
+      if ( req.profiler ) req.profiler.done('cors');
       var that = this;
       var response = {};
       var template;
@@ -548,8 +541,6 @@ var CartodbWindshaft = function(serverOptions) {
           if ( err ) throw err;
           layergroup = instance;
           fakereq = { query: {}, params: {}, headers: _.clone(req.headers),
-            method: req.method,
-            res: res,
             profiler: req.profiler
           };
           ws.setDBParams(cdbuser, fakereq.params, this);
@@ -600,7 +591,8 @@ var CartodbWindshaft = function(serverOptions) {
 
     function finish_instanciation(err, response, res, req) {
         if ( req.profiler ) {
-          res.header('X-Tiler-Profiler', req.profiler.toJSONString());
+          var report = req.profiler.toString();
+          res.header('X-Tiler-Profiler', report);
         }
         if (err) {
             var statusCode = 400;

lib/cartodb/log4js_rollbar.js

@@ -1,49 +0,0 @@
-var rollbar = require("rollbar");
-
-/**
- * Rollbar Appender. Sends logging events to Rollbar using node-rollbar 
- *
- * @param config object with rollbar configuration data
- * {
- *   token: 'your-secret-token',
- *   options: node-rollbar options
- * }
- */
-function rollbarAppender(config) {
-
-  var opt = config.options;
-	rollbar.init(opt.token, opt.options);
-	
-	return function(loggingEvent) {
-/*
-For logger.trace('one','two','three'):
-{ startTime: Wed Mar 12 2014 16:27:40 GMT+0100 (CET),
-  categoryName: '[default]',
-  data: [ 'one', 'two', 'three' ],
-  level: { level: 5000, levelStr: 'TRACE' },
-  logger: { category: '[default]', _events: { log: [Object] } } }
-*/
-
-    // Levels:
-    // TRACE  5000
-    // DEBUG 10000
-    // INFO  20000
-    // WARN  30000
-    // ERROR 40000
-    // FATAL 50000
-    //
-    // We only log error and higher errors
-    //
-    if ( loggingEvent.level.level < 40000 ) return;
-
-    rollbar.reportMessage(loggingEvent.data);
-	};
-}
-
-function configure(config) {
-	return rollbarAppender(config);
-}
-
-exports.name      = "rollbar";
-exports.appender  = rollbarAppender;
-exports.configure = configure;

lib/cartodb/server_options.js

@@ -1,10 +1,11 @@
 var   _          = require('underscore')
     , Step       = require('step')
     , cartoData  = require('cartodb-redis')(global.environment.redis)
-    , Cache      = require('./cache_validator')
-    , QueryTablesApi   = require('./api/query_tables_api')
+    , Cache = require('./cache_validator')
+    , mapnik     = require('mapnik')
     , crypto     = require('crypto')
-    , LZMA       = require('lzma').LZMA;
+    , request    = require('request')
+    , LZMA       = require('lzma/lzma_worker.js').LZMA
   ;
 
 // This is for backward compatibility with 1.3.3
@@ -18,10 +19,6 @@ if ( _.isUndefined(global.environment.sqlapi.domain) ) {
 
 module.exports = function(){
 
-    var lzmaWorker = new LZMA();
-
-    var queryTablesApi = new QueryTablesApi();
-
     var rendererConfig = _.defaults(global.environment.renderer || {}, {
       cache_ttl: 60000, // milliseconds
       metatile: 4,
@@ -49,13 +46,11 @@ module.exports = function(){
           },
           datasource: global.environment.postgres,
           cachedir: global.environment.millstone.cache_basedir,
-          mapnik_version: global.environment.mapnik_version,
-          mapnik_tile_format: global.environment.mapnik_tile_format || 'png',
+          mapnik_version: global.environment.mapnik_version || mapnik.versions.mapnik,
           default_layergroup_ttl: global.environment.mapConfigTTL || 7200, 
           gc_prob: 0.01 // @deprecated since Windshaft-1.8.0
         },
         mapnik: {
-          poolSize: rendererConfig.poolSize,
           metatile: rendererConfig.metatile,
           bufferSize: rendererConfig.bufferSize
         },
@@ -77,6 +72,14 @@ module.exports = function(){
     // See http://github.com/CartoDB/Windshaft-cartodb/issues/161
     me.redis.unwatchOnRelease = false;
 
+    // Be nice and warn if configured mapnik version
+    // is != instaled mapnik version
+    if ( mapnik.versions.mapnik != me.grainstore.mapnik_version ) {
+      console.warn("WARNING: detected mapnik version ("
+        + mapnik.versions.mapnik + ") != configured mapnik version ("
+        + me.grainstore.mapnik_version + ")");
+    }
+
 /* This whole block is about generating X-Cache-Channel { */
 
     // TODO: review lifetime of elements of this cache
@@ -85,6 +88,120 @@ module.exports = function(){
     //       we have no SQL after layer creation.
     me.channelCache = {};
 
+    // Run a query through the SQL api
+    me.sqlQuery = function (username, api_key, sql, callback) {
+      var api = global.environment.sqlapi;
+
+      // build up api string
+      var sqlapihostname = username;
+      if ( api.domain ) sqlapihostname += '.' + api.domain;
+
+      var sqlapi = api.protocol + '://';
+      if ( api.host && api.host != api.domain ) sqlapi += api.host;
+      else sqlapi += sqlapihostname;
+      sqlapi += ':' + api.port + '/api/' + api.version + '/sql';
+
+      var qs  = { q: sql }
+
+      // add api_key if given
+      if (_.isString(api_key) && api_key != '') { qs.api_key = api_key; }
+
+      // call sql api
+      //
+      // NOTE: using POST to avoid size limits:
+      // See http://github.com/CartoDB/Windshaft-cartodb/issues/111
+      //
+      // NOTE: uses "host" header to allow IP based specification
+      //       of sqlapi address (and avoid a DNS lookup)
+      //
+      // NOTE: allows for keeping up to "maxConnections" concurrent
+      //       sockets opened per SQL-API host.
+      // See http://nodejs.org/api/http.html#http_agent_maxsockets
+      //
+      var maxSockets = global.environment.maxConnections || 128;
+      var maxGetLen = api.max_get_sql_length || 2048;
+      var maxSQLTime = api.timeout || 100; // 1/10 of a second by default
+      var reqSpec = {
+            url:sqlapi,
+            json:true,
+            headers:{host: sqlapihostname}
+            // http://nodejs.org/api/http.html#http_agent_maxsockets
+            ,pool:{maxSockets:maxSockets}
+            // timeout in milliseconds
+            ,timeout:maxSQLTime
+      }
+      if ( sql.length > maxGetLen ) {
+        reqSpec.method = 'POST';
+        reqSpec.body = qs;
+      } else {
+        reqSpec.method = 'GET';
+        reqSpec.qs = qs;
+      }
+      request(reqSpec, function(err, res, body) {
+          if (err){
+            console.log('ERROR connecting to SQL API on ' + sqlapi + ': ' + err);
+            callback(err);
+            return;
+          }
+          if (res.statusCode != 200) {
+              var msg = res.body.error ? res.body.error : res.body;
+              callback(new Error(msg));
+              console.log('unexpected response status (' + res.statusCode + ') for sql query: ' + sql + ': ' + msg);
+              return;
+          }
+          callback(null, body.rows);
+      });
+    };
+
+    //
+    // Invoke callback with number of milliseconds since
+    // last update in any of the given tables
+    //
+    me.findLastUpdated = function (username, api_key, tableNames, callback) {
+      var sql = 'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max FROM CDB_TableMetadata m WHERE m.tabname::name = any (\'{'
+        + tableNames.join(',') + '}\')';
+
+      // call sql api
+      me.sqlQuery(username, api_key, sql, function(err, rows){
+          if (err){
+            var msg = err.message ? err.message : err;
+            callback(new Error('could not find last updated timestamp: ' + msg));
+            return;
+          }
+          // when the table has not updated_at means it hasn't been changed so a default last_updated is set
+          var last_updated = 0;
+          if(rows.length !== 0) {
+            last_updated = rows[0].max || 0;
+          }
+          callback(null, last_updated*1000);
+      });
+    };
+
+    me.affectedTables = function (username, api_key, sql, callback) {
+
+      // Replace mapnik tokens
+      sql = sql.replace(RegExp('!bbox!', 'g'), 'ST_MakeEnvelope(0,0,0,0)')
+               .replace(RegExp('!pixel_width!', 'g'), '1')
+               .replace(RegExp('!pixel_height!', 'g'), '1')
+            ;
+
+      // Pass to CDB_QueryTables
+      sql = 'SELECT CDB_QueryTables($windshaft$' + sql + '$windshaft$)';
+
+      // call sql api
+      me.sqlQuery(username, api_key, sql, function(err, rows){
+          if (err){
+            var msg = err.message ? err.message : err;
+            callback(new Error('could not fetch source tables: ' + msg));
+            return;
+          }
+          var qtables = rows[0].cdb_querytables;
+          var tableNames = qtables.split(/^\{(.*)\}$/)[1];
+          tableNames = tableNames ? tableNames.split(',') : [];
+          callback(null, tableNames);
+      });
+    };
+
     me.buildCacheChannel = function (dbName, tableNames){
       return dbName + ':' + tableNames.join(',');
     };
@@ -93,7 +210,7 @@ module.exports = function(){
         var hash = crypto.createHash('md5');
         hash.update(data);
         return hash.digest('hex');
-    };
+    }
 
     me.generateCacheChannel = function(app, req, callback){
 
@@ -123,6 +240,7 @@ module.exports = function(){
             // See http://github.com/CartoDB/Windshaft-cartodb/issues/152
             if ( ! app.mapStore ) {
               throw new Error('missing channel cache for token ' + req.params.token);
+              return;
             }
             var next = this;
             var mapStore = app.mapStore;
@@ -167,38 +285,8 @@ module.exports = function(){
             }
             return [req.params.table];
           }
-          var user, key;
-          var next = this;
-          Step (
-            function findUserKey() {
-              if ( req.params.hasOwnProperty('_authorizedBySigner') ) {
-                user = req.params._authorizedBySigner;
-                cartoData.getUserMapKey(user, this);
-              } else {
-                user = that.userByReq(req);
-                key = req.params.map_key || req.params.api_key;
-                return null;
-              }
-            },
-            function getAffected(err, data) {
-              if ( err ) throw err;
-              if ( data ) {
-                if ( req.profiler ) req.profiler.done('getSignerMapKey');
-                key = data;
-              }
-              queryTablesApi.getAffectedTablesInQuery(user, {
-                  user: req.params.dbuser,
-                  pass: req.params.dbpass,
-                  host: req.params.dbhost,
-                  port: req.params.dbport,
-                  dbname: req.params.dbname,
-                  api_key: key
-              }, sql, this); // in addCacheChannel
-            },
-            function finish(err, data) {
-              next(err,data);
-            }
-          );
+          var username   = that.userByReq(req);
+          me.affectedTables(username, req.params.map_key, sql, this);
         },
         function buildCacheChannel(err, tableNames) {
           if ( err ) throw err;
@@ -285,14 +373,6 @@ module.exports = function(){
             err = errors.length ? new Error(errors.join('\n')) : null;
             callback(err);
           }
-        };
-
-        // include in layergroup response the variables in serverMedata
-        // those variables are useful to send to the client information
-        // about how to reach this server or information about it
-        var serverMetadata = global.environment.serverMetadata;
-        if (serverMetadata) {
-          _.extend(response, serverMetadata);
         }
 
         // Don't wait for the mapview count increment to
@@ -315,44 +395,33 @@ module.exports = function(){
         var key    = req.params.map_key || req.params.api_key;
 
         var cacheKey = dbName + ':' + token;
+        var tabNames;
 
         Step(
-            function getAffectedTablesAndLastUpdatedTime() {
-                queryTablesApi.getAffectedTablesAndLastUpdatedTime(usr, {
-                    user: req.params.dbuser,
-                    pass: req.params.dbpass,
-                    host: req.params.dbhost,
-                    port: req.params.dbport,
-                    dbname: req.params.dbname,
-                    api_key: key
-                }, sql, this);
-            },
-            function handleAffectedTablesAndLastUpdatedTime(err, result) {
-                if (req.profiler) req.profiler.done('queryTablesAndLastUpdated');
-                if ( err ) throw err;
-                var cacheChannel = me.buildCacheChannel(dbName, result.affectedTables);
-                me.channelCache[cacheKey] = cacheChannel;
-
-                if (req.res && req.method == 'GET') {
-                    var res = req.res;
-                    if ( req.query && req.query.cache_policy == 'persist' ) {
-                        res.header('Cache-Control', 'public,max-age=31536000'); // 1 year
-                    } else {
-                        var ttl = global.environment.varnish.ttl || 86400;
-                        res.header('Cache-Control', 'public,max-age='+ttl+',must-revalidate');
-                    }
-                    res.header('Last-Modified', (new Date()).toUTCString());
-                    res.header('X-Cache-Channel', cacheChannel);
-                }
-
-                // last update for layergroup cache buster
-                response.layergroupid = response.layergroupid + ':' + result.lastUpdatedTime;
-                response.last_updated = new Date(result.lastUpdatedTime).toISOString();
-                return null;
-            },
-            function finish(err) {
-                done(err);
-            }
+          function getTables() {
+            me.affectedTables(usr, key, sql, this);
+          },
+          function getLastupdated(err, tableNames) {
+            if (req.profiler) req.profiler.done('affectedTables');
+            if ( err ) throw err;
+            var cacheChannel = me.buildCacheChannel(dbName,tableNames);
+            // store for caching from me.afterLayergroupCreate
+            me.channelCache[cacheKey] = cacheChannel;
+            // find last updated
+            if ( ! tableNames.length ) return 0; // skip for no affected tables
+            tabNames = tableNames;
+            me.findLastUpdated(usr, key, tableNames, this);
+          },
+          function(err, lastUpdated) {
+            if ( err ) throw err;
+            if (req.profiler && tabNames) req.profiler.done('findLastUpdated');
+            response.layergroupid = response.layergroupid + ':' + lastUpdated; // use epoch
+            response.last_updated = new Date(lastUpdated).toISOString(); 
+            return null;
+          },
+          function finish(err) {
+            done(err);
+          }
         );
     };
 
@@ -378,7 +447,7 @@ module.exports = function(){
         return;
       }
       return mat[1];
-    };
+    }
 
     // Set db authentication parameters to those of the given username
     //
@@ -447,20 +516,21 @@ module.exports = function(){
         dbport: global.environment.postgres.port
       });
       Step(
-        function getConnectionParams() {
-            cartoData.getUserDBConnectionParams(dbowner, this);
+        function getDatabaseHost(){
+            cartoData.getUserDBHost(dbowner, this);
         },
-        function extendParams(err, dbParams){
+        function getDatabase(err, data){
+            if(err) throw err;
+            if ( data ) _.extend(params, {dbhost:data});
+            cartoData.getUserDBName(dbowner, this);
+        },
+        function extendParams(err, data){
             if (err) throw err;
-            // we don't want null values or overwrite a non public user
-            if (params.dbuser != 'publicuser' || !dbParams.dbuser) {
-                delete dbParams.dbuser;
-            }
-            if ( dbParams ) _.extend(params, dbParams);
+            if ( data ) _.extend(params, {dbname:data});
             return null;
         },
         function finish(err) {
-          callback(err);
+          callback(err); 
         }
       );
     };
@@ -593,7 +663,6 @@ module.exports = function(){
                 }
 
                 // Authorized by "signed_by" !
-                _.extend(req.params, { _authorizedBySigner: signed_by });
                 that.setDBAuth(signed_by, req.params, function(err) {
                   if (req.profiler) req.profiler.done('setDBAuth');
                   callback(err, true); // authorized (or error)
@@ -630,16 +699,16 @@ module.exports = function(){
           //console.log("type of req.query.lzma is " + typeof(req.query.lzma));
 
           // Decode (from base64)
-          var lzma = (new Buffer(req.query.lzma, 'base64').toString('binary')).split('').map(function(c) { return c.charCodeAt(0) - 128 });
+          var lzma = (new Buffer(req.query.lzma, 'base64').toString('binary')).split('').map(function(c) { return c.charCodeAt(0) - 128 })
 
           // Decompress
-          lzmaWorker.decompress(
+          LZMA.decompress(
             lzma,
             function(result) {
               if (req.profiler) req.profiler.done('LZMA decompress');
               try {
-                delete req.query.lzma;
-                _.extend(req.query, JSON.parse(result));
+                delete req.query.lzma
+                _.extend(req.query, JSON.parse(result))
                 me.req2params(req, callback);
               } catch (err) {
                 callback(new Error('Error parsing lzma as JSON: ' + err));
@@ -671,7 +740,7 @@ module.exports = function(){
             req.params.signer = tksplit.shift();
             if ( ! req.params.signer ) req.params.signer = user;
             else if ( req.params.signer != user ) {
-              var err = new Error('Cannot use map signature of user "' + req.params.signer + '" on database of user "' + user + '"');
+              var err = new Error('Cannot use map signature of user "' + req.params.signer + '" on database of user "' + user + '"')
               err.http_status = 403;
               callback(err);
               return;
@@ -719,6 +788,7 @@ module.exports = function(){
                 cartoData.getTableGeometryType(req.params.dbname, req.params.table, this);
             },
             function finishSetup(err, data){
+                if (req.profiler) req.profiler.done('cartoData.getTableGeometryType');
                 if ( err ) { callback(err, req); return; }
 
                 if (!_.isNull(data))

lib/cartodb/signed_maps.js

@@ -128,7 +128,6 @@ console.log("Cert is : "); console.dir(cert);
 
 // Check if the given certificate authorizes waiver of "auth"
 o.authorizedByCert = function(cert, auth) {
-  auth = _.isArray(auth) ? auth : [auth];
 
   var err = this.checkInvalidCertificate(cert);
   if ( err ) throw err;
@@ -140,7 +139,12 @@ o.authorizedByCert = function(cert, auth) {
 
   // Token based authentication requires valid token
   if ( method === 'token' ) {
-    return _.intersection(cert.auth.valid_tokens, auth).length > 0;
+    var found = cert.auth.valid_tokens.indexOf(auth);
+    //if ( found !== -1 ) {
+      //console.log("Token " + auth + " is found at position " + found + " in valid tokens " + cert.auth.valid_tokens);
+    //  return true;
+    //} else return false;
+    return cert.auth.valid_tokens.indexOf(auth) !== -1;
   }
 
   throw new Error("Unsupported authentication method: " + cert.auth.method);

lib/cartodb/sql/sql_api.js

@@ -1,66 +0,0 @@
-var _       = require('underscore'),
-    request = require('request');
-
-module.exports.query = function (username, api_key, sql, callback) {
-    var api = global.environment.sqlapi;
-
-    // build up api string
-    var sqlapihostname = username;
-    if ( api.domain ) sqlapihostname += '.' + api.domain;
-
-    var sqlapi = api.protocol + '://';
-    if ( api.host && api.host != api.domain ) sqlapi += api.host;
-    else sqlapi += sqlapihostname;
-    sqlapi += ':' + api.port + '/api/' + api.version + '/sql';
-
-    var qs  = { q: sql };
-
-    // add api_key if given
-    if (_.isString(api_key) && api_key != '') { qs.api_key = api_key; }
-
-    // call sql api
-    //
-    // NOTE: using POST to avoid size limits:
-    // See http://github.com/CartoDB/Windshaft-cartodb/issues/111
-    //
-    // NOTE: uses "host" header to allow IP based specification
-    //       of sqlapi address (and avoid a DNS lookup)
-    //
-    // NOTE: allows for keeping up to "maxConnections" concurrent
-    //       sockets opened per SQL-API host.
-    // See http://nodejs.org/api/http.html#http_agent_maxsockets
-    //
-    var maxSockets = global.environment.maxConnections || 128;
-    var maxGetLen = api.max_get_sql_length || 2048;
-    var maxSQLTime = api.timeout || 100; // 1/10 of a second by default
-    var reqSpec = {
-        url:sqlapi,
-        json:true,
-        headers:{host: sqlapihostname}
-        // http://nodejs.org/api/http.html#http_agent_maxsockets
-        ,pool:{maxSockets:maxSockets}
-        // timeout in milliseconds
-        ,timeout:maxSQLTime
-    };
-    if ( sql.length > maxGetLen ) {
-        reqSpec.method = 'POST';
-        reqSpec.body = qs;
-    } else {
-        reqSpec.method = 'GET';
-        reqSpec.qs = qs;
-    }
-    request(reqSpec, function(err, res, body) {
-        if (err){
-            console.log('ERROR connecting to SQL API on ' + sqlapi + ': ' + err);
-            callback(err);
-            return;
-        }
-        if (res.statusCode != 200) {
-            var msg = res.body.error ? res.body.error : res.body;
-            callback(new Error(msg));
-            console.log('unexpected response status (' + res.statusCode + ') for sql query: ' + sql + ': ' + msg);
-            return;
-        }
-        callback(null, body.rows);
-    });
-};

lib/cartodb/template_maps.js

@@ -1,7 +1,10 @@
-var crypto  = require('crypto'),
-    Step    = require('step'),
-    _       = require('underscore'),
-    dot     = require('dot');
+var crypto    = require('crypto');
+var Step      = require('step');
+var _         = require('underscore');
+
+// Templates in this hash (keyed as <username>@<template_name>)
+// are being worked on.
+var user_template_locks = {};
 
 // Class handling map templates
 //
@@ -35,18 +38,17 @@ function TemplateMaps(redis_pool, signed_maps, opts) {
   // 
   // We have the following datastores:
   //
-  //  1. User templates: set of per-user map templates
+  //  1. User teplates: set of per-user map templates
   //     NOTE: each template would have an associated auth
   //           reference, see signed_maps.js
 
   // User templates (HASH:tpl_id->tpl_val)
-  this.key_usr_tpl = dot.template("map_tpl|{{=it.owner}}");
+  this.key_usr_tpl = "map_tpl|<%= owner %>";
 
   // User template locks (HASH:tpl_id->ctime)
-  this.key_usr_tpl_lck = dot.template("map_tpl|{{=it.owner}}|locks");
+  this.key_usr_tpl_lck = "map_tpl|<%= owner %>|locks";
 
-  this.lock_ttl = this.opts['lock_ttl'] || 5000;
-}
+};
 
 var o = TemplateMaps.prototype;
 
@@ -95,34 +97,36 @@ o._redisCmd = function(redisFunc, redisArgs, callback) {
 
 // @param callback function(err, obtained)
 o._obtainTemplateLock = function(owner, tpl_id, callback) {
-    var that = this,
-        lockKey = this.key_usr_tpl_lck({owner:owner});
-    Step (
-        function obtainLock() {
-            that._redisCmd('HGET', [lockKey, tpl_id], this);
-        },
-        function checkLock(err, lockTime) {
-            if (err) { throw err; }
-
-            var _newLockTime = Date.now();
-            if (!lockTime || ((_newLockTime - lockTime) > that.lock_ttl)) {
-                that._redisCmd('HSET', [lockKey, tpl_id, _newLockTime], this);
-            } else {
-                throw new Error("Template '" + tpl_id + "' of user '" + owner + "' is locked");
-            }
-        },
-        function finish(err, hsetValue) {
-            callback(err, !!hsetValue);
-        }
-    );
+  var usr_tpl_lck_key = _.template(this.key_usr_tpl_lck, {owner:owner});
+  var that = this;
+  var gotLock = false;
+  Step (
+    function obtainLock() {
+      var ctime = Date.now();
+      that._redisCmd('HSETNX', [usr_tpl_lck_key, tpl_id, ctime], this);
+    },
+    function checkLock(err, locked) {
+      if ( err ) throw err;
+      if ( ! locked ) {
+        // Already locked
+        // TODO: unlock if expired ?
+        throw new Error("Template '" + tpl_id + "' of user '" + owner + "' is locked");
+      }
+      return gotLock = true;
+    },
+    function finish(err) {
+      callback(err, gotLock);
+    }
+  );
 };
 
 // @param callback function(err, deleted)
 o._releaseTemplateLock = function(owner, tpl_id, callback) {
-    this._redisCmd('HDEL', [this.key_usr_tpl_lck({owner:owner}), tpl_id], callback);
+  var usr_tpl_lck_key = _.template(this.key_usr_tpl_lck, {owner:owner});
+  this._redisCmd('HDEL', [usr_tpl_lck_key, tpl_id], callback);
 };
 
-var _reValidIdentifier = /^[a-zA-Z][0-9a-zA-Z_]*$/;
+o._reValidIdentifier = /^[a-zA-Z][0-9a-zA-Z_]*$/;
 o._checkInvalidTemplate = function(template) {
   if ( template.version != '0.0.1' ) {
     return new Error("Unsupported template version " + template.version);
@@ -131,26 +135,22 @@ o._checkInvalidTemplate = function(template) {
   if ( ! tplname ) {
     return new Error("Missing template name");
   }
-  if ( ! tplname.match(_reValidIdentifier) ) {
+  if ( ! tplname.match(this._reValidIdentifier) ) {
     return new Error("Invalid characters in template name '" + tplname + "'");
   }
 
-  var placeholders = template.placeholders || {};
-
-  var placeholderKeys = Object.keys(placeholders);
-  for (var i = 0, len = placeholderKeys.length; i < len; i++) {
-      var placeholderKey = placeholderKeys[i];
-
-      if (!placeholderKey.match(_reValidIdentifier)) {
-          return new Error("Invalid characters in placeholder name '" + placeholderKey + "'");
-      }
-      if ( ! placeholders[placeholderKey].hasOwnProperty('default') ) {
-          return new Error("Missing default for placeholder '" + placeholderKey + "'");
-      }
-      if ( ! placeholders[placeholderKey].hasOwnProperty('type') ) {
-          return new Error("Missing type for placeholder '" + placeholderKey + "'");
-      }
-  }
+  var phold = template.placeholders;
+  for (var k in phold) {
+    if ( ! k.match(this._reValidIdentifier) ) {
+      return new Error("Invalid characters in placeholder name '" + k + "'");
+    }
+    if ( ! phold[k].hasOwnProperty('default') ) {
+      return new Error("Missing default for placeholder '" + k + "'");
+    }
+    if ( ! phold[k].hasOwnProperty('type') ) {
+      return new Error("Missing type for placeholder '" + k + "'");
+    }
+  };
 
   // Check certificate validity
   var cert = this.getTemplateCertificate(template);
@@ -168,11 +168,12 @@ o._checkInvalidTemplate = function(template) {
 // SignedMaps.addCertificate or SignedMaps.authorizedByCert
 //
 o.getTemplateCertificate = function(template) {
-  return {
-      version: '0.0.1',
-      template_id: template.name,
-      auth: template.auth
+  var cert = {
+    version: '0.0.1',
+    template_id: template.name,
+    auth: template.auth
   };
+  return cert;
 };
 
 // Add a template
@@ -208,7 +209,7 @@ o.addTemplate = function(owner, template, callback) {
   //
   //
 
-  var usr_tpl_key = this.key_usr_tpl({owner:owner});
+  var usr_tpl_key = _.template(this.key_usr_tpl, {owner:owner});
   var gotLock = false;
   var that = this;
   var limit = that._userTemplateLimit();
@@ -292,7 +293,7 @@ o.addTemplate = function(owner, template, callback) {
 // @param callback function(err)
 //
 o.delTemplate = function(owner, tpl_id, callback) {
-  var usr_tpl_key = this.key_usr_tpl({owner:owner});
+  var usr_tpl_key = _.template(this.key_usr_tpl, {owner:owner});
   var gotLock = false;
   var that = this;
   Step(
@@ -401,7 +402,7 @@ o.updTemplate = function(owner, tpl_id, template, callback) {
     return;
   }
 
-  var usr_tpl_key = this.key_usr_tpl({owner:owner});
+  var usr_tpl_key = _.template(this.key_usr_tpl, {owner:owner});
   var gotLock = false;
   var that = this;
   Step(
@@ -495,7 +496,8 @@ o.updTemplate = function(owner, tpl_id, template, callback) {
 //        Returns a list of template identifiers
 // 
 o.listTemplates = function(owner, callback) {
-    this._redisCmd('HKEYS', [ this.key_usr_tpl({owner:owner}) ], callback);
+  var usr_tpl_key = _.template(this.key_usr_tpl, {owner:owner});
+  this._redisCmd('HKEYS', [ usr_tpl_key ], callback);
 };
 
 // Get a templates
@@ -509,15 +511,17 @@ o.listTemplates = function(owner, callback) {
 //        Return full template definition
 //
 o.getTemplate = function(owner, tpl_id, callback) {
+  var usr_tpl_key = _.template(this.key_usr_tpl, {owner:owner});
   var that = this;
   Step(
     function getTemplate() {
-      that._redisCmd('HGET', [ that.key_usr_tpl({owner:owner}), tpl_id ], this);
+      that._redisCmd('HGET', [ usr_tpl_key, tpl_id ], this);
     },
     function parseTemplate(err, tpl_val) {
       if ( err ) throw err;
+      var tpl = JSON.parse(tpl_val);
       // Should we strip auth_id ?
-      return JSON.parse(tpl_val);
+      return tpl;
     },
     function finish(err, tpl) {
       callback(err, tpl);
@@ -537,22 +541,25 @@ o.getTemplate = function(owner, tpl_id, callback) {
 //
 // @throws Error on malformed template or parameter
 //
-var _reNumber = /^([-+]?[\d\.]?\d+([eE][+-]?\d+)?)$/,
-    _reCSSColorName = /^[a-zA-Z]+$/,
-    _reCSSColorVal = /^#[0-9a-fA-F]{3,6}$/;
-
-_replaceVars = function(str, params) {
+o._reNumber = /^([-+]?[\d\.]?\d+([eE][+-]?\d+)?)$/;
+o._reCSSColorName = /^[a-zA-Z]+$/;
+o._reCSSColorVal = /^#[0-9a-fA-F]{3,6}$/;
+o._replaceVars = function(str, params) {
   //return _.template(str, params); // lazy way, possibly dangerous 
   // Construct regular expressions for each param
-    Object.keys(params).forEach(function(k) {
-        str = str.replace(new RegExp("<%=\\s*" + k + "\\s*%>", "g"), params[k]);
-    });
-    return str;
+  if ( ! params._re ) {
+    params._re = {};
+    for (var k in params) {
+      params._re[k] = RegExp("<%=\\s*" + k + "\\s*%>", "g");
+    }
+  }
+  for (var k in params) str = str.replace(params._re[k], params[k]);
+  return str;
 };
 o.instance = function(template, params) {
   var all_params = {};
-  var phold = template.placeholders || {};
-  Object.keys(phold).forEach(function(k) {
+  var phold = template.placeholders;
+  for (var k in phold) {
     var val = params.hasOwnProperty(k) ? params[k] : phold[k].default;
     var type = phold[k].type;
     // properly escape
@@ -566,7 +573,7 @@ o.instance = function(template, params) {
     }
     else if ( type === 'number' ) {
       // check it's a number
-      if ( typeof(val) !== 'number' && ! val.match(_reNumber) ) {
+      if ( typeof(val) !== 'number' && ! val.match(this._reNumber) ) {
         throw new Error("Invalid number value for template parameter '"
           + k + "': " + val);
       }
@@ -574,7 +581,7 @@ o.instance = function(template, params) {
     else if ( type === 'css_color' ) {
       // check it only contains letters or
       // starts with # and only contains hexdigits
-      if ( ! val.match(_reCSSColorName) && ! val.match(_reCSSColorVal) ) {
+      if ( ! val.match(this._reCSSColorName) && ! val.match(this._reCSSColorVal) ) {
         throw new Error("Invalid css_color value for template parameter '"
           + k + "': " + val);
       }
@@ -584,14 +591,14 @@ o.instance = function(template, params) {
       throw new Error("Invalid placeholder type '" + type + "'");
     }
     all_params[k] = val;
-  });
+  }
 
   // NOTE: we're deep-cloning the layergroup here
   var layergroup = JSON.parse(JSON.stringify(template.layergroup));
   for (var i=0; i<layergroup.layers.length; ++i) {
     var lyropt = layergroup.layers[i].options;
-    if ( lyropt.cartocss ) lyropt.cartocss = _replaceVars(lyropt.cartocss, all_params);
-    if ( lyropt.sql) lyropt.sql = _replaceVars(lyropt.sql, all_params);
+    if ( lyropt.cartocss ) lyropt.cartocss = this._replaceVars(lyropt.cartocss, all_params);
+    if ( lyropt.sql) lyropt.sql = this._replaceVars(lyropt.sql, all_params);
     // Anything else ?
   }
   return layergroup;

package.json

@@ -1,7 +1,7 @@
 {
     "private": true,
     "name": "windshaft-cartodb",
-    "version": "1.18.2",
+    "version": "1.8.5",
     "description": "A map tile server for CartoDB",
     "keywords": [
       "cartodb"
@@ -22,24 +22,22 @@
       "Sandro Santilli <strk@vizzuality.com>"
     ],
     "dependencies": {
-        "node-varnish": "https://github.com/Vizzuality/node-varnish/tarball/0.3.0",
-        "underscore"  : "~1.6.0",
-        "dot": "~1.0.2",
-        "windshaft": "https://github.com/CartoDB/Windshaft/tarball/0.28.1",
-        "step": "~0.0.5",
-        "request": "~2.9.203",
-        "cartodb-redis": "https://github.com/CartoDB/node-cartodb-redis/tarball/0.11.0",
-        "cartodb-psql": "https://github.com/CartoDB/node-cartodb-psql/tarball/0.4.0",
-        "redis-mpool": "https://github.com/CartoDB/node-redis-mpool/tarball/0.1.0",
-        "lzma": "~1.3.7",
-        "log4js": "~0.6.17",
-        "rollbar": "~0.3.13"
+        "node-varnish": "http://github.com/Vizzuality/node-varnish/tarball/v0.2.0",
+        "underscore"  : "~1.3.3",
+        "windshaft"  : "http://github.com/CartoDB/Windshaft/tarball/0.19.4",
+        "step": "0.0.x",
+        "request": "2.9.202",
+        "cartodb-redis": "~0.3.0",
+        "redis-mpool": "http://github.com/CartoDB/node-redis-mpool/tarball/0.0.4",
+        "mapnik": "~0.7.22",
+        "lzma": "~1.2.3",
+        "log4js": "~0.6.10"
     },
     "devDependencies": {
-        "mocha": "~1.21.4",
-        "redis": "~0.8.6",
-        "strftime": "~0.8.2",
-        "semver": "~1.1.4"
+        "mocha": "1.14.0",
+        "redis": "~0.8.3",
+        "strftime": "~0.6.0",
+        "semver": "~1.1.0"
     },
     "scripts": {
         "test": "make check"

run_tests.sh

@@ -5,8 +5,6 @@ OPT_CREATE_PGSQL=yes # create the PostgreSQL test environment
 OPT_DROP_REDIS=yes   # drop the redis test environment
 OPT_DROP_PGSQL=yes   # drop the PostgreSQL test environment
 
-export PGAPPNAME=cartodb_tiler_tester
-
 cd $(dirname $0)
 BASEDIR=$(pwd)
 cd -

test/acceptance/multilayer.js

@@ -4,6 +4,7 @@ var _           = require('underscore');
 var redis       = require('redis');
 var querystring = require('querystring');
 var semver      = require('semver');
+var mapnik      = require('mapnik');
 var Step        = require('step');
 var strftime    = require('strftime');
 var SQLAPIEmu   = require(__dirname + '/../support/SQLAPIEmu.js');
@@ -13,20 +14,13 @@ var helper = require(__dirname + '/../support/test_helper');
 
 var windshaft_fixtures = __dirname + '/../../node_modules/windshaft/test/fixtures';
 
-var IMAGE_EQUALS_TOLERANCE_PER_MIL = 20;
-var IMAGE_EQUALS_HIGHER_TOLERANCE_PER_MIL = 25;
-
 var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft');
 var ServerOptions = require(__dirname + '/../../lib/cartodb/server_options');
 serverOptions = ServerOptions();
 var server = new CartodbWindshaft(serverOptions);
 server.setMaxListeners(0);
 
-[true, false].forEach(function(cdbQueryTablesFromPostgresEnabledValue) {
-
-global.environment.enabledFeatures = {cdbQueryTablesFromPostgres: cdbQueryTablesFromPostgresEnabledValue};
-
-suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function() {
+suite('multilayer', function() {
 
     var redis_client = redis.createClient(global.environment.redis.port);
     var sqlapi_server;
@@ -114,15 +108,12 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
               assert.equal(cc.substring(0, dbname.length), dbname);
               var jsonquery = cc.substring(dbname.length+1);
               var sentquery = JSON.parse(jsonquery);
-              var expectedQuery = [layergroup.layers[0].options.sql, ';', layergroup.layers[1].options.sql].join('');
-              assert.equal(sentquery.q, 'WITH querytables AS ( SELECT * FROM CDB_QueryTables($windshaft$'
-                + expectedQuery
-                + '$windshaft$) as tablenames )'
-                + ' SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max'
-                + ' FROM CDB_TableMetadata m'
-                + ' WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])');
+              assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$'
+                + layergroup.layers[0].options.sql + ';'
+                + layergroup.layers[1].options.sql 
+                + '$windshaft$)');
 
-              assert.imageEqualsFile(res.body, 'test/fixtures/test_table_0_0_0_multilayer1.png', IMAGE_EQUALS_HIGHER_TOLERANCE_PER_MIL,
+              assert.imageEqualsFile(res.body, 'test/fixtures/test_table_0_0_0_multilayer1.png', 2,
                 function(err, similarity) {
                   next(err);
               });
@@ -157,7 +148,7 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
               method: 'GET'
           }, {}, function(res) {
               assert.equal(res.statusCode, 200, res.body);
-              assert.equal(res.headers['content-type'], "application/json; charset=utf-8");
+              assert.equal(res.headers['content-type'], "text/javascript; charset=utf-8; charset=utf-8");
               assert.utfgridEqualsFile(res.body, 'test/fixtures/test_table_0_0_0_multilayer1.layer0.grid.json', 2,
                 function(err, similarity) {
                   next(err);
@@ -175,7 +166,7 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
               method: 'GET'
           }, {}, function(res) {
               assert.equal(res.statusCode, 200, res.body);
-              assert.equal(res.headers['content-type'], "application/json; charset=utf-8");
+              assert.equal(res.headers['content-type'], "text/javascript; charset=utf-8; charset=utf-8");
               assert.utfgridEqualsFile(res.body, 'test/fixtures/test_table_0_0_0_multilayer1.layer1.grid.json', 2,
                 function(err, similarity) {
                   next(err);
@@ -201,41 +192,9 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
       );
     });
 
-
-    test("should include serverMedata in the response", function(done) {
-      global.environment.serverMetadata = { cdn_url : { http:'test', https: 'tests' } }
-      var layergroup =  {
-        version: '1.0.0',
-        layers: [
-           { options: {
-               sql: 'select cartodb_id, ST_Translate(the_geom_webmercator, 5e6, 0) as the_geom_webmercator from test_table limit 2',
-               cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', 
-               cartocss_version: '2.0.1'
-             } }
-        ]
-      };
-
-      var expected_token; 
-      Step(
-        function do_create_get()
-        {
-          var next = this;
-          assert.response(server, {
-              url: '/tiles/layergroup?config=' + encodeURIComponent(JSON.stringify(layergroup)),
-              method: 'GET',
-              headers: {host: 'localhost'} 
-          }, {}, function(res, err) { next(err, res); });
-        },
-        function do_check_create(err, res) {
-          var parsed = JSON.parse(res.body);
-          assert.ok(_.isEqual(parsed.cdn_url, global.environment.serverMetadata.cdn_url));
-          done();
-        }
-      )
-    });
-
-
-    test("get creation requests has cache", function(done) {
+    // See https://github.com/CartoDB/Windshaft-cartodb/issues/176
+    // NOTE: another test like this is in templates.js
+    test("get creation requests no cache", function(done) {
 
       var layergroup =  {
         version: '1.0.0',
@@ -264,7 +223,7 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
           assert.equal(res.statusCode, 200, res.body);
           var parsedBody = JSON.parse(res.body);
           expected_token = parsedBody.layergroupid.split(':')[0];
-          helper.checkCache(res);
+          helper.checkNoCache(res);
           return null;
         },
         function finish(err) {
@@ -286,49 +245,6 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
       );
     });
 
-    test("get creation has no cache if sql is bogus", function(done) {
-        var layergroup =  {
-            version: '1.0.0',
-            layers: [
-                { options: {
-                    sql: 'select bogus(0,0) as the_geom_webmercator',
-                    cartocss: '#layer { polygon-fill: red; }',
-                    cartocss_version: '2.0.1'
-                } }
-            ]
-        };
-        assert.response(server, {
-            url: '/tiles/layergroup?config=' + encodeURIComponent(JSON.stringify(layergroup)),
-            method: 'GET',
-            headers: {host: 'localhost'}
-        }, {}, function(res) {
-            assert.notEqual(res.statusCode, 200);
-            helper.checkNoCache(res);
-            done();
-        });
-    });
-
-    test("get creation has no cache if cartocss is not valid", function(done) {
-        var layergroup =  {
-            version: '1.0.0',
-            layers: [
-                { options: {
-                    sql: 'select cartodb_id, ST_Translate(the_geom_webmercator, 5e6, 0) as the_geom_webmercator from test_table limit 2',
-                    cartocss: '#layer { invalid-rule:red; }',
-                    cartocss_version: '2.0.1'
-                } }
-            ]
-        };
-        assert.response(server, {
-            url: '/tiles/layergroup?config=' + encodeURIComponent(JSON.stringify(layergroup)),
-            method: 'GET',
-            headers: {host: 'localhost'}
-        }, {}, function(res) {
-            assert.notEqual(res.statusCode, 200);
-            helper.checkNoCache(res);
-            done();
-        });
-    });
 
     test("layergroup can hold substitution tokens", function(done) {
 
@@ -393,18 +309,14 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
               assert.equal(cc.substring(0, dbname.length), dbname);
               var jsonquery = cc.substring(dbname.length+1);
               var sentquery = JSON.parse(jsonquery);
-              var expectedQuery = layergroup.layers[0].options.sql
-                  .replace(/!bbox!/g, 'ST_MakeEnvelope(0,0,0,0)')
-                  .replace(/!pixel_width!/g, '1')
-                  .replace(/!pixel_height!/g, '1');
-              assert.equal(sentquery.q, 'WITH querytables AS ( SELECT * FROM CDB_QueryTables($windshaft$'
-                  + expectedQuery
-                  + '$windshaft$) as tablenames )'
-                  + ' SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max'
-                  + ' FROM CDB_TableMetadata m'
-                  + ' WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])');
+              assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$'
+                + layergroup.layers[0].options.sql
+                    .replace(RegExp('!bbox!', 'g'), 'ST_MakeEnvelope(0,0,0,0)')
+                    .replace(RegExp('!pixel_width!', 'g'), '1')
+                    .replace(RegExp('!pixel_height!', 'g'), '1')
+                + '$windshaft$)');
 
-              assert.imageEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.png', IMAGE_EQUALS_TOLERANCE_PER_MIL,
+              assert.imageEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.png', 2,
                 function(err, similarity) {
                   next(err);
               });
@@ -430,18 +342,14 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
               assert.equal(cc.substring(0, dbname.length), dbname);
               var jsonquery = cc.substring(dbname.length+1);
               var sentquery = JSON.parse(jsonquery);
-              var expectedQuery = layergroup.layers[0].options.sql
-                  .replace('!bbox!', 'ST_MakeEnvelope(0,0,0,0)')
-                  .replace('!pixel_width!', '1')
-                  .replace('!pixel_height!', '1');
-              assert.equal(sentquery.q, 'WITH querytables AS ( SELECT * FROM CDB_QueryTables($windshaft$'
-                  + expectedQuery
-                  + '$windshaft$) as tablenames )'
-                  + ' SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max'
-                  + ' FROM CDB_TableMetadata m'
-                  + ' WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])');
+              assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$'
+                + layergroup.layers[0].options.sql
+                    .replace('!bbox!', 'ST_MakeEnvelope(0,0,0,0)')
+                    .replace('!pixel_width!', '1')
+                    .replace('!pixel_height!', '1')
+                + '$windshaft$)');
 
-              assert.imageEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.png', IMAGE_EQUALS_TOLERANCE_PER_MIL,
+              assert.imageEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.png', 2,
                 function(err, similarity) {
                   next(err);
               });
@@ -458,7 +366,7 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
               method: 'GET'
           }, {}, function(res) {
               assert.equal(res.statusCode, 200, res.body);
-              assert.equal(res.headers['content-type'], "application/json; charset=utf-8");
+              assert.equal(res.headers['content-type'], "text/javascript; charset=utf-8; charset=utf-8");
               assert.utfgridEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.grid.json', 2,
                 function(err, similarity) {
                   next(err);
@@ -476,7 +384,7 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
               method: 'GET'
           }, {}, function(res) {
               assert.equal(res.statusCode, 200, res.body);
-              assert.equal(res.headers['content-type'], "application/json; charset=utf-8");
+              assert.equal(res.headers['content-type'], "text/javascript; charset=utf-8; charset=utf-8");
               assert.utfgridEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.grid.json', 2,
                 function(err, similarity) {
                   next(err);
@@ -752,7 +660,7 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
               method: 'GET'
           }, {}, function(res) {
               assert.equal(res.statusCode, 200, res.body);
-              assert.equal(res.headers['content-type'], "application/json; charset=utf-8");
+              assert.equal(res.headers['content-type'], "text/javascript; charset=utf-8; charset=utf-8");
               next(err);
           });
         },
@@ -1070,7 +978,7 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
           }, {}, function(res) {
               assert.equal(res.statusCode, 200, res.body);
               assert.equal(res.headers['content-type'], "image/png");
-              assert.imageEqualsFile(res.body, windshaft_fixtures + '/test_default_mapnik_point.png', IMAGE_EQUALS_TOLERANCE_PER_MIL,
+              assert.imageEqualsFile(res.body, windshaft_fixtures + '/test_default_mapnik_point.png', 2,
                 function(err, similarity) {
                   next(err);
               });
@@ -1345,4 +1253,3 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
     
 });
 
-});

test/acceptance/server.js

@@ -4,31 +4,24 @@ var _           = require('underscore');
 var redis       = require('redis');
 var querystring = require('querystring');
 var semver      = require('semver');
+var mapnik      = require('mapnik');
 var Step        = require('step');
 var http        = require('http');
 var SQLAPIEmu   = require(__dirname + '/../support/SQLAPIEmu.js');
 
 var helper = require(__dirname + '/../support/test_helper');
 
-var IMAGE_EQUALS_TOLERANCE_PER_MIL = 20,
-    IMAGE_EQUALS_ZERO_TOLERANCE_PER_MIL = 0;
-
 var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft');
 var serverOptions = require(__dirname + '/../../lib/cartodb/server_options')();
 var server = new CartodbWindshaft(serverOptions);
 server.setMaxListeners(0);
 
-[true, false].forEach(function(cdbQueryTablesFromPostgresEnabledValue) {
-
-global.environment.enabledFeatures = {cdbQueryTablesFromPostgres: cdbQueryTablesFromPostgresEnabledValue};
-
-suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function() {
-
+suite('server', function() {
 
     var redis_client = redis.createClient(global.environment.redis.port);
     var sqlapi_server;
 
-    var mapnik_version = global.environment.mapnik_version || server.getVersion().mapnik;
+    var mapnik_version = global.environment.mapnik_version || mapnik.versions.mapnik;
     var test_database = _.template(global.environment.postgres_auth_user, {user_id:1}) + '_db';
     var default_style;
     if ( semver.satisfies(mapnik_version, '<2.1.0') ) {
@@ -586,7 +579,7 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
             method: 'GET'
         },{
             status: 200,
-            headers: { 'Content-Type': 'application/json; charset=utf-8',
+            headers: { 'Content-Type': 'text/javascript; charset=utf-8; charset=utf-8',
                        'X-Cache-Channel': test_database+':gadm4' }
         }, function() { done(); });
     });
@@ -598,7 +591,7 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
             method: 'GET'
         },{
             status: 200,
-            headers: { 'Content-Type': 'application/json; charset=utf-8' }
+            headers: { 'Content-Type': 'text/javascript; charset=utf-8; charset=utf-8' }
         }, function() { done(); });
     });
     
@@ -610,7 +603,7 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
             method: 'GET'
         },{
             status: 200,
-            headers: { 'Content-Type': 'application/json; charset=utf-8' }
+            headers: { 'Content-Type': 'text/javascript; charset=utf-8; charset=utf-8' }
         }, function() { done(); });
     });
 
@@ -627,24 +620,6 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
         });
     });
 
-    // See http://github.com/CartoDB/Windshaft-cartodb/issues/186
-    test("get'ing the grid of a private table should fail when unauthenticated (jsonp)",
-    function(done) {
-        assert.response(server, {
-            headers: {host: 'localhost'},
-            url: '/tiles/test_table_private_1/6/31/24.grid.json?callback=x',
-            method: 'GET'
-        },{}, function(res) {
-          // It's forbidden, but jsonp calls for status = 200
-          assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
-          // Still, we do NOT want to add caching headers here
-          // See https://github.com/CartoDB/Windshaft-cartodb/issues/186
-          assert.ok(!res.headers.hasOwnProperty('cache-control'),
-            "Unexpected Cache-Control: " + res.headers['cache-control']);
-          done();
-        });
-    });
-
     // See http://github.com/Vizzuality/Windshaft-cartodb/issues/55
     test("get'ing grid of private table should fail on unknown username",
     function(done) {
@@ -849,7 +824,7 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
           assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
           var ct = res.headers['content-type'];
           assert.equal(ct, 'image/png');
-          assert.imageEqualsFile(res.body, './test/fixtures/test_table_15_16046_12354_styled_black.png', IMAGE_EQUALS_TOLERANCE_PER_MIL,
+          assert.imageEqualsFile(res.body, './test/fixtures/test_table_15_16046_12354_styled_black.png',  2,
             function(err, similarity) {
               if (err) throw err;
               done();
@@ -880,7 +855,7 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
             assert.equal(ct, 'image/png');
             assert.imageEqualsFile(res.body,
               './test/fixtures/test_table_15_16046_12354_styled_black.png',
-              IMAGE_EQUALS_TOLERANCE_PER_MIL, this);
+              2, this);
           },
           function checkImage(err, similarity) {
               if (err) throw err;
@@ -896,8 +871,8 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
     // See https://github.com/CartoDB/Windshaft-cartodb/issues/89
     test("get'ing a tile with a user-specific database password",  function(done){
         var style = querystring.stringify({style: test_style_black_200, style_version: '2.0.0'});
-        var backupDBPass = global.environment.postgres_auth_pass;
-        global.environment.postgres_auth_pass = '<%= user_password %>';
+        var backupDBPass = global.settings.postgres_auth_pass;
+        global.settings.postgres_auth_pass = '<%= user_password %>';
         Step (
           function() {
             var next = this;
@@ -917,14 +892,14 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
             assert.equal(ct, 'image/png');
             assert.imageEqualsFile(res.body,
               './test/fixtures/test_table_15_16046_12354_styled_black.png',
-              IMAGE_EQUALS_TOLERANCE_PER_MIL, this);
+              2, this);
           },
           function checkImage(err, similarity) {
               if (err) throw err;
               return null
           },
           function finish(err) {
-            global.environment.postgres_auth_pass = backupDBPass;
+            global.settings.postgres_auth_pass = backupDBPass;
             done(err);
           }
         );
@@ -941,7 +916,7 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
           assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
           var ct = res.headers['content-type'];
           assert.equal(ct, 'image/png');
-          assert.imageEqualsFile(res.body, './test/fixtures/test_table_15_16046_12354_styled_black.png', IMAGE_EQUALS_TOLERANCE_PER_MIL,
+          assert.imageEqualsFile(res.body, './test/fixtures/test_table_15_16046_12354_styled_black.png',  2,
             function(err, similarity) {
               if (err) throw err;
               done();
@@ -978,7 +953,7 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
             assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
             var ct = res.headers['content-type'];
             assert.equal(ct, 'image/png');
-            assert.imageEqualsFile(res.body, './test/fixtures/test_table_15_16046_12354_styled_black.png', IMAGE_EQUALS_TOLERANCE_PER_MIL,
+            assert.imageEqualsFile(res.body, './test/fixtures/test_table_15_16046_12354_styled_black.png',  2,
               function(err, similarity) {
                 next(err);
             });
@@ -1018,7 +993,7 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
             assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
             var ct = res.headers['content-type'];
             assert.equal(ct, 'image/png');
-            assert.imageEqualsFile(res.body, './test/fixtures/blank.png', IMAGE_EQUALS_ZERO_TOLERANCE_PER_MIL,
+            assert.imageEqualsFile(res.body, './test/fixtures/blank.png',  0,
               function(err, similarity) {
                 if (err) next(err); 
                 else next();
@@ -1038,7 +1013,7 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
             assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
             var ct = res.headers['content-type'];
             assert.equal(ct, 'image/png');
-            assert.imageEqualsFile(res.body, './test/fixtures/blank.png', IMAGE_EQUALS_ZERO_TOLERANCE_PER_MIL,
+            assert.imageEqualsFile(res.body, './test/fixtures/blank.png',  0,
               function(err, similarity) {
                 if (err) next(err); 
                 else next();
@@ -1075,7 +1050,7 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
             assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
             var ct = res.headers['content-type'];
             assert.equal(ct, 'image/png');
-            assert.imageEqualsFile(res.body, './test/fixtures/test_table_15_16046_12354_styled_black.png', IMAGE_EQUALS_TOLERANCE_PER_MIL,
+            assert.imageEqualsFile(res.body, './test/fixtures/test_table_15_16046_12354_styled_black.png',  2,
               function(err, similarity) {
                 // NOTE: we expect them to be EQUAL here
                 if (err) { next(err); return; }
@@ -1112,7 +1087,7 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
             assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
             var ct = res.headers['content-type'];
             assert.equal(ct, 'image/png');
-            assert.imageEqualsFile(res.body, './test/fixtures/test_table_15_16046_12354_styled_black.png', IMAGE_EQUALS_TOLERANCE_PER_MIL,
+            assert.imageEqualsFile(res.body, './test/fixtures/test_table_15_16046_12354_styled_black.png',  2,
               function(err, similarity) {
                 // NOTE: we expect them to be different here
                 if (err) next(); 
@@ -1393,4 +1368,3 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function(
     
 });
 
-});

test/acceptance/templates.js

@@ -3,6 +3,7 @@ var _           = require('underscore');
 var redis       = require('redis');
 var querystring = require('querystring');
 var semver      = require('semver');
+var mapnik      = require('mapnik');
 var Step        = require('step');
 var strftime    = require('strftime');
 var SQLAPIEmu   = require(__dirname + '/../support/SQLAPIEmu.js');
@@ -19,16 +20,11 @@ var helper = require(__dirname + '/../support/test_helper');
 var windshaft_fixtures = __dirname + '/../../node_modules/windshaft/test/fixtures';
 
 var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft');
-var ServerOptions = require(__dirname + '/../../lib/cartodb/server_options');
-var serverOptions = ServerOptions();
+var serverOptions = require(__dirname + '/../../lib/cartodb/server_options')();
 var server = new CartodbWindshaft(serverOptions);
 server.setMaxListeners(0);
 
-[true, false].forEach(function(cdbQueryTablesFromPostgresEnabledValue) {
-
-global.environment.enabledFeatures = {cdbQueryTablesFromPostgres: cdbQueryTablesFromPostgresEnabledValue};
-
-suite('template_api:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function() {
+suite('template_api', function() {
 
     var redis_client = redis.createClient(global.environment.redis.port);
     var sqlapi_server;
@@ -312,52 +308,6 @@ suite('template_api:postgres=' + cdbQueryTablesFromPostgresEnabledValue, functio
       });
     });
 
-    test("instance endpoint should return server metadata", function(done){
-      global.environment.serverMetadata = { cdn_url : { http:'test', https: 'tests' } }
-      var tmpl = _.clone(template_acceptance1)
-      tmpl.name = "rambotemplate2"
-
-      Step(function postTemplate1(err, res) {
-          var next = this;
-          var post_request = {
-              url: '/tiles/template?api_key=1234',
-              method: 'POST',
-              headers: {host: 'localhost', 'Content-Type': 'application/json' },
-              data: JSON.stringify(tmpl)
-          };
-          assert.response(server, post_request, {}, function(res) { 
-            next(null, res); 
-          });
-        },
-        function testCORS() {
-          var next = this;
-          assert.response(server, {
-              url: '/tiles/template/' + tmpl.name,
-              method: 'POST',
-              headers: {host: 'localhost', 'Content-Type': 'application/json' },
-          },{
-              status: 200
-          }, function(res) { 
-            var parsed = JSON.parse(res.body);
-            assert.ok(_.isEqual(parsed.cdn_url, global.environment.serverMetadata.cdn_url));
-            next(null); 
-          });
-        },
-        function deleteTemplate(err) {
-          if ( err ) throw err;
-          var del_request = {
-              url: '/tiles/template/' + tmpl.name + '?api_key=1234', 
-              method: 'DELETE',
-              headers: {host: 'localhost', 'Content-Type': 'application/json' }
-          }
-          var next = this;
-          assert.response(server, del_request, {},
-            function(res) { done(); });
-        }
-      );
-    });
-
-
 
     test("can list templates", function(done) {
 
@@ -1171,37 +1121,12 @@ suite('template_api:postgres=' + cdbQueryTablesFromPostgresEnabledValue, functio
           assert.response(server, get_request, {},
             function(res) { next(null, res); });
         },
-        function checkTile_fetchOnRestart(err, res) {
+        function checkTile(err, res) {
           if ( err ) throw err;
           assert.equal(res.statusCode, 200, 
             'Unexpected error for authorized instance: '
             + res.statusCode + ' -- ' + res.body);
           assert.equal(res.headers['content-type'], "application/json; charset=utf-8");
-          var cc = res.headers['x-cache-channel'];
-          assert.ok(cc);
-          assert.ok(cc.match, /ciao/, cc);
-          // hack simulating restart...
-          serverOptions = ServerOptions(); // need to clean channel cache
-          server = new CartodbWindshaft(serverOptions);
-          var get_request = {
-              url: '/tiles/layergroup/' + layergroupid + ':cb1/0/0/0/1.json.torque?auth_token=valid1',
-              method: 'GET',
-              headers: {host: 'localhost' },
-              encoding: 'binary'
-          }
-          var next = this;
-          assert.response(server, get_request, {},
-            function(res) { next(null, res); });
-        },
-        function checkCacheChannel(err, res) {
-          if ( err ) throw err;
-          assert.equal(res.statusCode, 200, 
-            'Unexpected error for authorized instance: '
-            + res.statusCode + ' -- ' + res.body);
-          assert.equal(res.headers['content-type'], "application/json; charset=utf-8");
-          var cc = res.headers['x-cache-channel'];
-          assert.ok(cc, "Missing X-Cache-Channel on fetch-after-restart");
-          assert.ok(cc.match, /ciao/, cc);
           return null;
         },
         function deleteTemplate(err)
@@ -1596,7 +1521,7 @@ suite('template_api:postgres=' + cdbQueryTablesFromPostgresEnabledValue, functio
           if ( err ) throw err;
           assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
           // See https://github.com/CartoDB/Windshaft-cartodb/issues/176
-          helper.checkCache(res);
+          helper.checkNoCache(res);
           return null;
         },
         function finish(err) {
@@ -1951,4 +1876,3 @@ suite('template_api:postgres=' + cdbQueryTablesFromPostgresEnabledValue, functio
     
 });
 
-});

test/support/SQLAPIEmu.js

@@ -2,7 +2,7 @@ var http        = require('http');
 var url         = require('url');
 var _ = require('underscore');
 
-var SQLAPIEmulator = function(port, cb) {
+var o = function(port, cb) {
 
   this.queries = [];
   var that = this;
@@ -37,45 +37,47 @@ var SQLAPIEmulator = function(port, cb) {
    }).listen(port, cb);
 };
 
-SQLAPIEmulator.prototype.handleQuery = function(query, res) {
+o.prototype.handleQuery = function(query, res) {
     this.queries.push(query);
     if ( query.q.match('SQLAPIERROR') ) {
       res.statusCode = 400;
       res.write(JSON.stringify({'error':'Some error occurred'}));
     } else if ( query.q.match('SQLAPINOANSWER') ) {
-        console.log("SQLAPIEmulator will never respond, on request");
-        return;
-    } else if (query.q.match('tablenames')) {
-        var tableNames = JSON.stringify(query);
-        res.write(queryResult({tablenames: '{' + tableNames + '}', max: 1234567890.123}));
+      console.log("SQLAPIEmulator will never respond, on request");
+      return;
     } else if ( query.q.match('EPOCH.* as max') ) {
       // This is the structure of the known query sent by tiler
-        res.write(queryResult({max: 1234567890.123}));
+      var row = {
+        'max': 1234567890.123
+      };
+      res.write(JSON.stringify({rows: [ row ]}));
     } else {
       if ( query.q.match('_private_') && query.api_key === undefined) {
         res.statusCode = 403;
         res.write(JSON.stringify({'error':'forbidden: ' + JSON.stringify(query)}));
       } else {
         var qs = JSON.stringify(query);
-        res.write(queryResult({cdb_querytables: '{' + qs + '}', max: 1234567890.123}));
+        var row = {
+          // This is the structure of the known query sent by tiler
+          'cdb_querytables': '{' + qs + '}',
+          'max': qs
+        };
+        var out_obj = {rows: [ row ]};
+        var out = JSON.stringify(out_obj);
+        res.write(out);
       }
     }
     res.end();
   };
 
-SQLAPIEmulator.prototype.close = function(cb) {
+
+o.prototype.close = function(cb) {
   this.sqlapi_server.close(cb);
 };
 
-SQLAPIEmulator.prototype.getLastRequest = function() {
+o.prototype.getLastRequest = function() {
   return this.requests.pop();
 };
 
-function queryResult(row) {
-    return JSON.stringify({
-        rows: [row]
-    });
-}
-
-module.exports = SQLAPIEmulator;
+module.exports = o;
 

test/support/assert.js

@@ -1,11 +1,10 @@
 // Cribbed from the ever prolific Konstantin Kaefer
 // https://github.com/mapbox/tilelive-mapnik/blob/master/test/support/assert.js
 
-var exec = require('child_process').exec,
-    fs = require('fs'),
-    http = require('http'),
-    path = require('path'),
-    util = require('util');
+var fs = require('fs');
+var http = require('http');
+var path = require('path');
+var exec = require('child_process').exec;
 
 var assert = module.exports = exports = require('assert');
 
@@ -67,51 +66,35 @@ assert.utfgridEqualsFile = function(buffer, file_b, tolerance, callback) {
     callback(err);
 };
 
-/**
- * Takes an image data as an input and an image path and compare them using ImageMagick fuzz algorithm, if case the
- * similarity is not within the tolerance limit it will callback with an error.
- *
- * @param buffer The image data to compare from
- * @param {string} referenceImageRelativeFilePath The relative file to compare against
- * @param {number} tolerance tolerated mean color distance, as a per mil (‰)
- * @param {function} callback Will call to home with null in case there is no error, otherwise with the error itself
- * @see FUZZY in http://www.imagemagick.org/script/command-line-options.php#metric
- */
-assert.imageEqualsFile = function(buffer, referenceImageRelativeFilePath, tolerance, callback) {
+//
+// @param tol tolerated color distance as a percent over max channel value
+//            by default this is zero. For meaningful values, see
+//            http://www.imagemagick.org/script/command-line-options.php#metric
+//
+assert.imageEqualsFile = function(buffer, file_b, tol, callback) {
     if (!callback) callback = function(err) { if (err) throw err; };
-    var referenceImageFilePath = path.resolve(referenceImageRelativeFilePath),
-        testImageFilePath = '/tmp/windshaft-test-image-' + (Math.random() * 1e16); // TODO: make predictable
-    var err = fs.writeFileSync(testImageFilePath, buffer, 'binary');
+    file_b = path.resolve(file_b);
+    var file_a = '/tmp/windshaft-test-image-test.png'; // + (Math.random() * 1e16); // TODO: make predictable 
+    var err = fs.writeFileSync(file_a, buffer, 'binary');
     if (err) throw err;
 
-    var imageMagickCmd = util.format(
-        'compare -metric fuzz "%s" "%s" /dev/null',
-        testImageFilePath, referenceImageFilePath
-    );
-
-    exec(imageMagickCmd, function(err, stdout, stderr) {
+    var fuzz = tol + '%';
+    exec('compare -fuzz ' + fuzz + ' -metric AE "' + file_a + '" "' +
+            file_b + '" /dev/null', function(err, stdout, stderr) {
         if (err) {
-            fs.unlinkSync(testImageFilePath);
+            fs.unlinkSync(file_a);
             callback(err);
         } else {
             stderr = stderr.trim();
-            var metrics = stderr.match(/([0-9]*) \((.*)\)/);
-            if ( ! metrics ) {
-                callback(new Error("No match for " + stderr));
-                return;
-            }
-            var similarity = parseFloat(metrics[2]),
-                tolerancePerMil = (tolerance / 1000);
-            if (similarity > tolerancePerMil) {
-                err = new Error(util.format(
-                        'Images %s and %s are not equal (got %d similarity, expected %d)',
-                        testImageFilePath, referenceImageFilePath, similarity, tolerancePerMil)
-                );
-                err.similarity = similarity;
-                callback(err);
+            var similarity = parseFloat(stderr);
+            if ( similarity > 0 ) {
+              var err = new Error('Images not equal(' + similarity + '): ' +
+                      file_a  + '    ' + file_b);
+              err.similarity = similarity;
+              callback(err);
             } else {
-                fs.unlinkSync(testImageFilePath);
-                callback(null);
+              fs.unlinkSync(file_a);
+              callback(null);
             }
         }
     });
@@ -226,8 +209,6 @@ assert.response = function(server, req, res, msg){
             response.on('end', function(){
                 if (timer) clearTimeout(timer);
 
-                check();
-
                 // Assert response body
                 if (res.body !== undefined) {
                     var eql = res.body instanceof RegExp
@@ -273,6 +254,7 @@ assert.response = function(server, req, res, msg){
 
                 // Callback
                 callback(response);
+                check();
             });
         });
 

test/support/prepare_db.sh

@@ -76,7 +76,7 @@ if test x"$PREPARE_PGSQL" = xyes; then
     sed "s/:PUBLICPASS/${PUBLICPASS}/" | 
     sed "s/:TESTUSER/${TESTUSER}/" | 
     sed "s/:TESTPASS/${TESTPASS}/" | 
-    psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
+    psql ${TEST_DB}
 
 fi
 
@@ -86,8 +86,7 @@ if test x"$PREPARE_REDIS" = xyes; then
 
   cat <<EOF | redis-cli -p ${REDIS_PORT} -n 5
 HMSET rails:users:localhost id ${TESTUSERID} \
-                            database_name "${TEST_DB}" \
-                            database_host localhost \
+                            database_name '${TEST_DB}' \
                             map_key 1234
 SADD rails:users:localhost:map_key 1235
 EOF

test/support/test_helper.js

@@ -7,18 +7,18 @@
 
 var _ = require('underscore');
 var assert = require('assert');
-var LZMA  = require('lzma').LZMA;
-
-var lzmaWorker = new LZMA();
+var LZMA  = require('lzma/lzma_worker.js').LZMA;
 
 // set environment specific variables
+global.settings     = require(__dirname + '/../../config/settings');
 global.environment  = require(__dirname + '/../../config/environments/test');
+_.extend(global.settings, global.environment);
 process.env.NODE_ENV = 'test';
 
 
 // Utility function to compress & encode LZMA
 function lzma_compress_to_base64(payload, mode, callback) {
-  lzmaWorker.compress(payload, mode,
+  LZMA.compress(payload, mode, 
     function(ints) {
       ints = ints.map(function(c) { return String.fromCharCode(c + 128) }).join('')
       var base64 = new Buffer(ints, 'binary').toString('base64');
@@ -39,21 +39,8 @@ function checkNoCache(res) {
 }
 
 
-/**
- * Check that the response headers do not request caching
- * @see checkNoCache
- * @param res
- */
-function checkCache(res) {
-    assert.ok(res.headers.hasOwnProperty('x-cache-channel'));
-    assert.ok(res.headers.hasOwnProperty('cache-control'));
-    assert.ok(res.headers.hasOwnProperty('last-modified'));
-}
-
-
 module.exports = {
   lzma_compress_to_base64: lzma_compress_to_base64,
-  checkNoCache: checkNoCache,
-  checkCache: checkCache
-};
+  checkNoCache: checkNoCache
+}
 

test/unit/cartodb/template_maps.test.js

@@ -12,14 +12,6 @@ suite('template_maps', function() {
   // configure redis pool instance to use in tests
   var redis_pool = RedisPool(global.environment.redis);
   var signed_maps = new SignedMaps(redis_pool);
-
-    var validTemplate = {
-        version:'0.0.1',
-        name: 'first',
-        auth: {},
-        layergroup: {}
-    };
-    var owner = 'me';
     
   test('does not accept template with unsupported version', function(done) {
     var tmap = new TemplateMaps(redis_pool, signed_maps);
@@ -510,89 +502,5 @@ suite('template_maps', function() {
       }
     );
   });
-
-    var redisCmdFunc = TemplateMaps.prototype._redisCmd;
-
-    function runWithRedisStubbed(stubbedCommands, func) {
-        TemplateMaps.prototype._redisCmd = function(redisFunc, redisArgs, callback) {
-            redisFunc = redisFunc.toLowerCase();
-            if (stubbedCommands.hasOwnProperty(redisFunc)) {
-                callback(null, stubbedCommands[redisFunc]);
-            } else {
-                throw 'Unknown command';
-            }
-        };
-
-        func();
-
-        TemplateMaps.prototype._redisCmd = redisCmdFunc;
-    }
-
-    test('_obtainTemplateLock with no previous value, happy case', function(done) {
-        runWithRedisStubbed({hget: null, hset: 1}, function() {
-            var templateMaps = new TemplateMaps(redis_pool, signed_maps);
-
-            templateMaps._obtainTemplateLock(owner, validTemplate.name, function(err, gotLock) {
-                assert.ok(!err);
-                assert.ok(gotLock);
-                done();
-            });
-        });
-    });
-
-    test('_obtainTemplateLock no lock for non expired ttl, simulates obtaining two locks at same time', function(done) {
-        runWithRedisStubbed({hget: Date.now()}, function() {
-            var templateMaps = new TemplateMaps(redis_pool, signed_maps);
-
-            templateMaps._obtainTemplateLock(owner, validTemplate.name, function(err, gotLock) {
-                assert.ok(!!err);
-                assert.equal(gotLock, false);
-                done();
-            });
-        });
-    });
-
-
-    test('_obtainTemplateLock no lock for non expired ttl, last millisecond of valid ttl', function(done) {
-        var nowValue = Date.now(),
-            nowFunc = Date.now;
-        Date.now = function() {
-            return nowValue;
-        };
-        var lockTtl = 1000;
-        runWithRedisStubbed({hget: Date.now() - lockTtl, hset: true}, function() {
-            var templateMaps = new TemplateMaps(redis_pool, signed_maps, {lock_ttl: lockTtl});
-
-            templateMaps._obtainTemplateLock(owner, validTemplate.name, function(err, gotLock) {
-                assert.ok(!!err);
-                assert.equal(gotLock, false);
-
-                Date.now = nowFunc;
-
-                done();
-            });
-        });
-    });
-
-    test('_obtainTemplateLock gets lock for expired ttl, first millisecond of invalid ttl', function(done) {
-        var nowValue = Date.now(),
-            nowFunc = Date.now;
-        Date.now = function() {
-            return nowValue;
-        };
-        var lockTtl = 1000;
-        runWithRedisStubbed({hget: Date.now() - lockTtl - 1, hset: true}, function() {
-            var templateMaps = new TemplateMaps(redis_pool, signed_maps, {lock_ttl: lockTtl});
-
-            templateMaps._obtainTemplateLock(owner, validTemplate.name, function(err, gotLock) {
-                assert.ok(!err);
-                assert.ok(gotLock);
-
-                Date.now = nowFunc;
-
-                done();
-            });
-        });
-    });
-
+    
 });

tools/examples/mapconfig_slow.js

@@ -1,11 +0,0 @@
-{"version":"1.0.1",
-	"layers":[{
-    "type":"cartodb",
-	  "options":{
-      "sql":"select 1 as id, ST_Transform(ST_SetSRID(ST_MakePoint(x/1000,x/2000),4326),3857) as the_geom_webmercator FROM generate_series(-170000,170000) x",
-	    "cartocss":"#style{ marker-width: 12;}",
-	    "cartocss_version":"2.1.1",
-      "Interactivity":"id"
-    }
-  }]
-}

tools/show_style

@@ -38,8 +38,7 @@ if ( ! username ) usage(me, 1);
 console.log("Using environment " + ENV);
 
 global.environment = require('../config/environments/' + ENV);
-// _after_ setting global.environment
-var serverOptions = require('../lib/cartodb/server_options')();
+var serverOptions = require('../lib/cartodb/server_options'); // _after_ setting global.environment
 
 var client;
 var dbname;