Release 1.6.0

Closes #100 (yet again)

.gitignore

@@ -1,3 +1,7 @@
-node_modules
+node_modules*
+config.status*
 config/environments/*.js
 .idea
+tools/munin/windshaft.conf
+logs/
+pids/

.travis.yml

@@ -0,0 +1,19 @@
+before_install:
+  - sudo apt-add-repository --yes ppa:mapnik/v2.1.0
+  - sudo apt-get update -q
+  - sudo apt-get install -q libmapnik-dev
+  - createdb template_postgis
+  - psql -c "CREATE EXTENSION postgis" template_postgis
+
+env:
+  - NPROCS=1 JOBS=1
+
+language: node_js
+node_js:
+  - "0.8"
+
+notifications:
+  irc:
+    channels:
+      - "irc.freenode.org#cartodb"
+    use_notice: true

HOWTO_RELEASE

@@ -0,0 +1,11 @@
+1. Ensure proper version in package.json 
+2. Ensure NEWS section exists for the new version, review it, add release date
+3. Drop npm-shrinkwrap.json
+4. Run npm install
+5. Test (make check or npm test), fix if broken before proceeding
+6. Run npm shrinkwrap
+7. Commit package.json, npm-shrinwrap.json, NEWS
+8. Tag Major.Minor.Patch
+9. Announce
+10. Stub NEWS/package for next version
+

Makefile

@@ -1,11 +1,35 @@
+srcdir=$(shell pwd)
+
 all:
 	npm install
 
 clean:
 	rm -rf node_modules/*
 
-config/environments/test.js: config/environments/test.js.example
-	./configure
+distclean: clean
+	rm config.status*
+
+config.status--test:
+	./configure --environment=test
+
+config/environments/test.js: config.status--test
+	./config.status--test 
+
+check-local: config/environments/test.js
+	./run_tests.sh ${RUNTESTFLAGS} \
+    test/unit/cartodb/*.js \
+    test/acceptance/*.js 
+
+check-submodules:
+	PATH="$$PATH:$(srcdir)/node_modules/.bin/"; \
+	for sub in windshaft grainstore node-varnish mapnik; do \
+	  if test -e node_modules/$${sub}; then \
+	      echo "Testing submodule $${sub}"; \
+	      make -C node_modules/$${sub} check || exit 1; \
+	  fi; \
+	done
+
+check-full: check-local check-submodules
+
+check: check-local
 
-check: config/environments/test.js
-	./run_tests.sh

NEWS.md

@@ -1,11 +1,224 @@
-1.1.0 (DD/MM/YY)
+1.6.0 -- 2014-01-10
+-------------------
+
+New features:
+
+ * Add 'user_from_host' directive to generalize username extraction (#100)
+ * Implement signed template maps (#98)
+
+Other changes:
+
+ * Update cartodb-redis dependency to "~0.3.0"
+ * Update redis-server dependency to "2.4.0+"
+
+1.5.2 -- 2013-12-05
+-------------------
+
+Bug fixes:
+
+* Fix configuration-level compatibility with versions prior to 1.5 (#96)
+* Fix use of old layergroups on mapnik upgrade (#97)
+
+1.5.1 -- 2013-11-28
+-------------------
+
+Bug fixes:
+
+* Survive presence of malformed CartoCSS in redis (#94)
+* Accept useless point-transform:scale directives (#93)
+
+1.5.0 -- 2013-11-19
+-------------------
+
+NOTE: new configuration directives `postgres_auth_pass` and
+      `postgres.password` added; see config/environments/*.example
+      for documentation.
+
+Improvements:
+
+* Add support for configuring database connection passwords
+* Optionally read user-specific database_host and database_password
+  from redis as per CartoDB-2.5.0 model (#88, #89)
+* Do not force ending dot in SQL-API hostname, for easier testing
+
+Bug fixes:
+
+* Return CORS headers when creating layergroups via GET (windshaft/#92)
+* Fix http status on database authentication error (windshaft/#94)
+* Fix text-face-name error at layergroup creation (windshaft/#93)
+
+Other changes:
+
+* CartoDB redis interaction delegated to "cartodb-redis" module
+
+
+1.4.1 -- 2013-11-08
+-------------------
+
+* Fix support for exponential notation in CartoCSS filter values (#87)
+
+1.4.0 -- 2013-10-31
+-------------------
+
+* Add Support for Mapnik-2.2.0 (#78)
+
+1.3.6 -- 2013-10-11
+-------------------
+
+* Restore support for node-0.8.9 accidentally dropped by 1.3.5
+  NOTE: needs removing node_modules/windshaft and re-running npm install
+
+1.3.5 -- 2013-10-03
+-------------------
+
+* Fixing apostrophes in CartoCSS
+* Fix "sql/table must contain zoom variable" error when using
+  "[ zoom > 3]" CartoCSS snippets (note the space)
+* Fix backward compatibility handling of sqlapi.host configuration (#82)
+* Fix error for invalid text-name in CartoCSS (#81)
+* Do not let anonymous requests use authorized renderer caches 
+
+1.3.4
+------
+
+NOTE: configuration sqlapi.host renamed to sqlapi.domain
+      (support for "sqlapi.host" is retained for backward compatibility)
+
+* Improve empty CartoCSS error message
+* Improve invalid mapnik-geometry-type CSS error message
+* Fix race condition in localization of network resources
+
+1.3.3
+------
+* Set Last-Modified header to allow for 304 responses
+* Add profiling support (needs useProfiler in env config file)
+* Fix double-checking for layergroups with no interactivity
+* Log full layergroup config at creation time (#76)
+
+1.3.2
+------
+* Set default layergroup TTL to 2 hours
+* Serve multilayer tiles and grid with persistent cache control
+
+1.3.1
+------
+* Fix deadlock on new style creation
+* Fix database authentication with multi-table layergroups
+* Add tile and grid fetching checks at layergroup creation time
+* Fix SQL error reporting to NOT split on newline
+* Fix support for CartoCSS attachments
+
+1.3.0
+------
+* Change stats format for multilayer map token request, see
+  http://github.com/Vizzuality/Windshaft-cartodb/wiki/Redis-stats-format
+
+1.2.1
+------
+* Fix multilayer post from firefox
+* Fix multilayer cartocss layer name handling
+
+1.2.0
+------
+* Multilayer API changes
+  * Layers passed by index in grid fetching url
+  * Interactivity only specified in layergroup config
+  * Embed cache_buster within token 
+  * Use ISO format for last_modified timestamp
+* Expected LZMA encoding changed to base64
+
+1.1.10
+------
+* Fix regression with default interactivity parameter (#74)
+* More verbose logging for SQL api connection errors
+* Write stats for multilayer map token request
+
+1.1.9
 -----
+* Handle SQL API errors by requesting no Varnish cache
+* Fix X-Cache-Channel for multilayer (by token) responses
+* Add last_modified field to layergroup creation response (#72)
+* Deprecate signal handler for USR1, add handler for USR2 (#71)
+* Fix support for ampersend characters in CartoCSS
+* Add support for LZMA compressed GET parameters
+* Add support for creating layergroups via GET
+
+1.1.8
+-----
+* Require Windshaft-0.9.1, to reduce harmfulness of cache_buster param
+
+1.1.7 (DD//MM//YY)
+-----
+* Do not let /etc/services confuse FD checker (munin plugin)
+* Multilayer support (#72)
+* Expose renderer settings in the environment config files
+
+1.1.6 (19//02//13)
+-----
+* Require windshaft 0.8.5, fixing some stability issues
+  and providing cache info on request
+* Require grainstore 0.10.9, fixing an issue with multi-geom markers
+* Enhance run_tests.sh to allow running single tests and skipping preparation
+* Fix async throws in getGeometryType, getInfoWindow and getMapMetadata
+* Survive connection refusals from redis
+* Add maxConnection environment configuration, default to 128
+
+1.1.5 (DD//MM//YY)
+-----
+* Fix bogus cached return of utf grid for fully contained tiles (#67)
+
+1.1.4 (DD//MM//YY)
+-----
+* Reduce default extent to allow for consistent proj4 round-tripping 
+* Enhance reset_styles script to use full configuration (#62)
+* Have reset_styles script also drop extended keys (#58)
+* Fix example postgis parameter for simplifying input geoms (#63)
+* Add row_limit to example config (#64)
+
+1.1.3 (30//11//12)
+-----
+* Fix reset_styles script to really skip extended keys
+* CartoCSS versioning
+ * Mapnik-version dependent default styles
+ * Enhance 2.0 -> 2.1 transforms:
+  * styles with conditional markers
+  * scale arrow markers by 50%
+
+1.1.2 (DD//MM//YY)
+-----
+* CartoCSS versioning
+ * Fix use of "style_version" with GET (inline styles)
+ * Enhance 2.0 -> 2.1 transforms:
+  * styles with no semicolon
+  * markers shift due to geometry clipping
+
+1.1.1 (DD//MM//YY)
+-----
+* Add support for persistent client cache headers
+* Fix crash on unknown user (#55)
+* Add /version entry point
+* CartoCSS versioning
+ * Include style_version in GET /style response
+ * Support style_version and style_convert parameters in POST /style request
+ * Support style_version in GET /:z/:x/:y request
+
+1.1.0 (30/10/12)
+=======
+* Add /version entry point
+* CartoCSS versioning
+ * Include version in GET /style response
+ * Support version and convert parameters in POST /style request
+ * Autodetect target mapnik version and let config override it
+ * Add tools/reset_styles script to batch-reset (and optionally convert) styles
 * Configurable logging format (#4)
 * Detailed error on missing user metadata 
 * Properly handle unauthenticated requests for metadata
 * Accept "api_key" in addition to "map_key",
   both in query_string and POST body (#38)
 * Add ./configure script
+* Allow listening on host IP
+* Replaced environment configs by .example ones
+* Fixed some issues with cluster2
 
 1.0.0 (03/10/12)
 -----

README.md

@@ -1,7 +1,8 @@
 Windshaft-CartoDB
 ==================
 
-NOTE: requires node-0.8.x
+[![Build Status](http://travis-ci.org/CartoDB/Windshaft-cartodb.png)]
+(http://travis-ci.org/CartoDB/Windshaft-cartodb)
 
 This is the CartoDB map tiler. It extends Windshaft with some extra
 functionality and custom filters for authentication
@@ -11,23 +12,28 @@ functionality and custom filters for authentication
 * gets the default geometry type from the cartodb redis store
 * allows tiles to be styled individually
 * provides a link to varnish high speed cache
-* provides a infowindow endpoint for windshaft
-* provides a ``map_metadata`` endpoint for windshaft
+* provides a ``infowindow`` endpoint for windshaft (DEPRECATED)
+* provides a ``map_metadata`` endpoint for windshaft (DEPRECATED)
+* provides signed template maps API
+  (http://github.com/CartoDB/Windshaft-cartodb/wiki/Template-maps)
 
 Requirements
 ------------
 
  [core]
- - node-0.6.x+
+ - node-0.8.x+
  - PostgreSQL-8.3+
  - PostGIS-1.5.0+
- - Redis 2.2.0+ (http://www.redis.io)
+ - Redis 2.4.0+ (http://www.redis.io)
  - Mapnik 2.0 or 2.1 
 
  [for cache control]
  - CartoDB-SQL-API 1.0.0+
  - CartoDB 0.9.5+ (for ``CDB_QueryTables``)
- - Varnish (https://www.varnish-cache.org)
+ - Varnish (http://www.varnish-cache.org)
+
+ [for running the testsuite]
+ - Imagemagick (http://www.imagemagick.org)
 
 Configure
 ---------
@@ -80,8 +86,16 @@ Args:
 
 * sql - plain SQL arguments
 * interactivity - specify the column to use in UTFGrid
-* cache_buster - if needed you can add a cachebuster to make sure you're
-  rendering new
+* cache_buster - Specify an identifier for the internal tile cache.
+                 Requesting tiles with the same cache_buster value may
+                 result in being served a cached version of the tile
+                 (even when requesting a tile for the first time, as tiles
+                 can be prepared in advance)
+* cache_policy - Set to "persist" to have the server send an Cache-Control
+                 header requesting caching devices to keep the response
+                 cached as much as possible. This is best used with a
+                 timestamp value in cache_buster for manual control of
+                 updates.
 * geom_type - override the cartodb default
 * style - override the default map style with Carto
 
@@ -93,6 +107,8 @@ Args:
 Args:
 
 * style - the style in CartoCSS you want to set
+* style_version - the version of the style for POST
+* style_convert - request conversion to target version (both POST and GET)
 
 
 **INFOWINDOW**

app.js

@@ -10,28 +10,45 @@
 
 // sanity check
 var ENV = process.argv[2]
-if (ENV != 'development' && ENV != 'production'){
+if (ENV != 'development' && ENV != 'production' && ENV != 'staging' ){
     console.error("\nnode app.js [environment]");
-    console.error("environments: [development, production]\n");
+    console.error("environments: [development, production, staging]\n");
     process.exit(1);
 }
 
 var _ = require('underscore')
     , Step       = require('step')
-	, CartodbWindshaft = require('./lib/cartodb/cartodb_windshaft');
-
+    ;
 // set environment specific variables
 global.settings     = require(__dirname + '/config/settings');
 global.environment  = require(__dirname + '/config/environments/' + ENV);
 _.extend(global.settings, global.environment);
 
-// Include cart_data.js only _after_ the "global" variable is set
+// Include cartodb_windshaft only _after_ the "global" variable is set
 // See https://github.com/Vizzuality/Windshaft-cartodb/issues/28
-var cartoData = require('./lib/cartodb/carto_data');
-
-var Windshaft = require('windshaft');
+var CartodbWindshaft = require('./lib/cartodb/cartodb_windshaft');
 var serverOptions = require('./lib/cartodb/server_options');
 
 ws = CartodbWindshaft(serverOptions);
-ws.listen(global.environment.port);
-console.log("Windshaft tileserver started on port " + global.environment.port);
+
+// Maximum number of connections for one process
+// 128 is a good number if you have up to 1024 filedescriptors
+// 4 is good if you have max 32 filedescriptors
+// 1 is good if you have max 16 filedescriptors
+ws.maxConnections = global.environment.maxConnections || 128;
+
+ws.listen(global.environment.port, global.environment.host);
+
+ws.on('listening', function() {
+  console.log("Windshaft tileserver started on " + global.environment.host + ':' + global.environment.port);
+});
+
+// DEPRECATED, use SIGUSR2
+process.on('SIGUSR1', function() {
+  console.log('WARNING: handling of SIGUSR1 by Windshaft-CartoDB is deprecated, please send SIGUSR2 instead');
+  ws.dumpCacheStats();
+});
+
+process.on('SIGUSR2', function() {
+  ws.dumpCacheStats();
+});

cluster.js

@@ -1,52 +0,0 @@
-/*
- * Windshaft-CartoDB
- * ===============
- *
- * ./app.js [environment]
- *
- * environments: [development, production]
- */
-
-var Cluster = require('cluster2');
-
-// sanity check
-var ENV = process.argv[2]
-if (ENV != 'development' && ENV != 'production' && ENV != 'staging'){
-    console.error("\nnode app.js [environment]");
-    console.error("environments: [development, production, staging]\n");
-    process.exit(1);
-}
-
-var _ = require('underscore')
-    , Step       = require('step')
-	, CartodbWindshaft = require('./lib/cartodb/cartodb_windshaft');
-    
-
-// set environment specific variables
-global.settings     = require(__dirname + '/config/settings');
-global.environment  = require(__dirname + '/config/environments/' + ENV);
-_.extend(global.settings, global.environment);
-
-// Include cart_data.js only _after_ the "global" variable is set
-// See https://github.com/Vizzuality/Windshaft-cartodb/issues/28
-var cartoData = require('./lib/cartodb/carto_data');
-
-var Windshaft = require('windshaft');
-var serverOptions = require('./lib/cartodb/server_options');
-
-var ws = CartodbWindshaft(serverOptions);
-
-//.use(cluster.logger('logs'))
-//.use(cluster.stats())
-//.use(cluster.pidfiles('pids'))
-var cluster = new Cluster({
-  port: global.environment.port,
-  monPort: global.environment.port+1,
-  noWorkers: 1 // .set('workers', 1)
-});
-
-cluster.listen(function(cb) {
-  cb(ws);
-});
-
-console.log("Windshaft tileserver started on port " + global.environment.port);

config/environments/development.js.example

@@ -2,35 +2,69 @@ var config = {
      environment: 'development'
     ,port: 8181
     ,host: '127.0.0.1'
+    // Regular expression pattern to extract username
+    // from hostname. Must have a single grabbing block.
+    ,user_from_host: '^(.*)\\.localhost'
+    // Maximum number of connections for one process
+    // 128 is a good value with a limit of 1024 open file descriptors
+    ,maxConnections:128
+    // idle socket timeout, in miliseconds 
+    ,socket_timeout: 600000
     ,enable_cors: true
     ,cache_enabled: false
-    ,log_format: '[:date] :req[X-Real-IP] \033[90m:method\033[0m \033[36m:req[Host]:url\033[0m \033[90m:status :response-time ms -> :res[Content-Type]\033[0m'
+    ,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
+    // Templated database username for authorized user
+    // Supported labels: 'user_id' (read from redis)
     ,postgres_auth_user: 'development_cartodb_user_<%= user_id %>'
+    // Templated database password for authorized user
+    // Supported labels: 'user_id', 'user_password' (both read from redis)
+    ,postgres_auth_pass: '<%= user_password %>'
     ,postgres: {
+        // Parameters to pass to datasource plugin of mapnik
+        // See http://github.com/mapnik/mapnik/wiki/PostGIS
         type: "postgis",
         user: "publicuser",
+        password: "public",
         host: '127.0.0.1',
         port: 5432,
-        extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188",
+        extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
         /* experimental
         geometry_field: "the_geom",
         extent: "-180,-90,180,90",
         srid: 4326,
         */
-        simplify: true
+        row_limit: 65535,
+        simplify_geometries: true,
+        max_size: 500
+    }
+    ,mapnik_version: undefined
+    ,renderer: {
+      // Milliseconds since last access before renderer cache item expires
+      cache_ttl: 60000,
+      metatile: 4,
+      bufferSize: 64
     }
     ,millstone: {
+        // Needs to be writable by server user
         cache_basedir: '/tmp/cdb-tiler-dev/millstone-dev'
     }
     ,redis: {
         host: '127.0.0.1',
         port: 6379,
-        idleTimeoutMillis: 1,
-        reapIntervalMillis: 1
+        // Max number of connections in each pool.
+        // Users will be put on a queue when the limit is hit.
+        // Set to maxConnection to have no possible queues.
+        // There are currently 3 pools involved in serving
+        // windshaft-cartodb requests so multiply this number
+        // by 3 to know how many possible connections will be
+        // kept open by the server. The default is 50.
+        max: 50,
+        idleTimeoutMillis: 1, // idle time before dropping connection
+        reapIntervalMillis: 1 // time between cleanups
     }
     ,sqlapi: {
         protocol: 'http',
-        host: 'localhost.lan',
+        domain: 'localhost.lan',
         port: 8080,
         version: 'v1'
     }
@@ -39,6 +73,10 @@ var config = {
         port: 6082,
         ttl: 86400
     }
+    // If useProfiler is true every response will be served with an
+    // X-Tiler-Profile header containing elapsed timing for various 
+    // steps taken for producing the response.
+    ,useProfiler:true
 };
 
 module.exports = config;

config/environments/production.js.example

@@ -2,27 +2,63 @@ var config = {
     environment: 'production'
     ,port: 8181
     ,host: '127.0.0.1'
+    // Regular expression pattern to extract username
+    // from hostname. Must have a single grabbing block.
+    ,user_from_host: '^(.*)\\.cartodb\\.com$'
+    // Maximum number of connections for one process
+    // 128 is a good value with a limit of 1024 open file descriptors
+    ,maxConnections:128
+    // idle socket timeout, in miliseconds
+    ,socket_timeout: 600000 
     ,enable_cors: true
     ,cache_enabled: true
-    ,log_format: '[:date] :req[X-Real-IP] \033[90m:method\033[0m \033[36m:req[Host]:url\033[0m \033[90m:status :response-time ms -> :res[Content-Type]\033[0m'
+    ,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
+    // Templated database username for authorized user
+    // Supported labels: 'user_id' (read from redis)
     ,postgres_auth_user: 'cartodb_user_<%= user_id %>'
+    // Templated database password for authorized user
+    // Supported labels: 'user_id', 'user_password' (both read from redis)
+    ,postgres_auth_pass: '<%= user_password %>'
     ,postgres: {
+        // Parameters to pass to datasource plugin of mapnik
+        // See http://github.com/mapnik/mapnik/wiki/PostGIS
         user: "publicuser",
+        password: "public",
         host: '127.0.0.1',
         port: 6432,
-        extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188",
-        simplify: true
+        extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
+        row_limit: 65535,
+        simplify_geometries: true,
+        max_size: 500
+    }
+    ,mapnik_version: undefined
+    ,renderer: {
+      // Milliseconds since last access before renderer cache item expires
+      cache_ttl: 60000,
+      metatile: 4,
+      bufferSize: 64
     }
     ,millstone: {
+        // Needs to be writable by server user
         cache_basedir: '/home/ubuntu/tile_assets/'
     }
     ,redis: {
         host: '127.0.0.1',
-        port: 6379
+        port: 6379,
+        // Max number of connections in each pool.
+        // Users will be put on a queue when the limit is hit.
+        // Set to maxConnection to have no possible queues.
+        // There are currently 3 pools involved in serving
+        // windshaft-cartodb requests so multiply this number
+        // by 3 to know how many possible connections will be
+        // kept open by the server. The default is 50.
+        max: 50,
+        idleTimeoutMillis: 30000, // idle time before dropping connection
+        reapIntervalMillis: 1000 // time between cleanups
     }
     ,sqlapi: {
         protocol: 'https',
-        host: 'cartodb.com',
+        domain: 'cartodb.com',
         port: 8080,
         version: 'v2'
     }
@@ -31,6 +67,10 @@ var config = {
         port: 6082,
         ttl: 86400
     }
+    // If useProfiler is true every response will be served with an
+    // X-Tiler-Profile header containing elapsed timing for various 
+    // steps taken for producing the response.
+    ,useProfiler:false
 };
 
 module.exports = config;

config/environments/staging.js.example

@@ -2,27 +2,63 @@ var config = {
     environment: 'production'
     ,port: 8181
     ,host: '127.0.0.1'
+    // Regular expression pattern to extract username
+    // from hostname. Must have a single grabbing block.
+    ,user_from_host: '^(.*)\\.cartodb\\.com$'
+    // Maximum number of connections for one process
+    // 128 is a good value with a limit of 1024 open file descriptors
+    ,maxConnections:128
+    // idle socket timeout, in miliseconds
+    ,socket_timeout: 600000 
     ,enable_cors: true
     ,cache_enabled: true
-    ,log_format: '[:date] :req[X-Real-IP] \033[90m:method\033[0m \033[36m:req[Host]:url\033[0m \033[90m:status :response-time ms -> :res[Content-Type]\033[0m'
+    ,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms (:res[X-Tiler-Profiler]) -> :res[Content-Type]'
+    // Templated database username for authorized user
+    // Supported labels: 'user_id' (read from redis)
     ,postgres_auth_user: 'cartodb_staging_user_<%= user_id %>'
+    // Templated database password for authorized user
+    // Supported labels: 'user_id', 'user_password' (both read from redis)
+    ,postgres_auth_pass: '<%= user_password %>'
     ,postgres: {
+        // Parameters to pass to datasource plugin of mapnik
+        // See http://github.com/mapnik/mapnik/wiki/PostGIS
         user: "publicuser",
+        password: "public",
         host: '127.0.0.1',
         port: 6432,
-        extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188",
-        simplify: true
+        extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
+        row_limit: 65535,
+        simplify_geometries: true,
+        max_size: 500
+    }
+    ,mapnik_version: undefined
+    ,renderer: {
+      // Milliseconds since last access before renderer cache item expires
+      cache_ttl: 60000,
+      metatile: 4,
+      bufferSize: 64
     }
     ,millstone: {
+        // Needs to be writable by server user
         cache_basedir: '/home/ubuntu/tile_assets/'
     }
     ,redis: {
         host: '127.0.0.1',
-        port: 6379
+        port: 6379,
+        // Max number of connections in each pool.
+        // Users will be put on a queue when the limit is hit.
+        // Set to maxConnection to have no possible queues.
+        // There are currently 3 pools involved in serving
+        // windshaft-cartodb requests so multiply this number
+        // by 3 to know how many possible connections will be
+        // kept open by the server. The default is 50.
+        max: 50,
+        idleTimeoutMillis: 30000, // idle time before dropping connection
+        reapIntervalMillis: 1000 // time between cleanups
     }
     ,sqlapi: {
         protocol: 'https',
-        host: 'cartodb.com',
+        domain: 'cartodb.com',
         port: 8080,
         version: 'v2'
     }
@@ -31,6 +67,10 @@ var config = {
         port: 6082,
         ttl: 86400
     }
+    // If useProfiler is true every response will be served with an
+    // X-Tiler-Profile header containing elapsed timing for various 
+    // steps taken for producing the response.
+    ,useProfiler:true
 };
 
 module.exports = config;

config/environments/test.js.example

@@ -2,31 +2,66 @@ var config = {
      environment: 'test'
     ,port: 8888
     ,host: '127.0.0.1'
+    // Regular expression pattern to extract username
+    // from hostname. Must have a single grabbing block.
+    ,user_from_host: '(.*)'
+    // Maximum number of connections for one process
+    // 128 is a good value with a limit of 1024 open file descriptors
+    ,maxConnections:128
+    // idle socket timeout, in miliseconds
+    ,socket_timeout: 600000 
     ,enable_cors: true
     ,cache_enabled: false
-    ,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type]'
+    ,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
+    // Templated database username for authorized user
+    // Supported labels: 'user_id' (read from redis)
     ,postgres_auth_user: 'test_cartodb_user_<%= user_id %>'
+    // Templated database password for authorized user
+    // Supported labels: 'user_id', 'user_password' (both read from redis)
+    ,postgres_auth_pass: 'test_cartodb_user_<%= user_id %>_pass'
     ,postgres: {
-        user: "publicuser",
+        // Parameters to pass to datasource plugin of mapnik
+        // See http://github.com/mapnik/mapnik/wiki/PostGIS
+        user: "testpublicuser",
+        password: "public",
         host: '127.0.0.1',
         port: 5432,
-        srid: 4326,
-        extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188",
-        simplify: true
+        extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
+        row_limit: 65535,
+        simplify_geometries: true,
+        max_size: 500
+    }
+    ,mapnik_version: '2.0.2'
+    ,renderer: {
+      // Milliseconds since last access before renderer cache item expires
+      cache_ttl: 60000,
+      metatile: 4,
+      bufferSize: 64
     }
     ,millstone: {
+        // Needs to be writable by server user
         cache_basedir: '/tmp/cdb-tiler-test/millstone'
     }
     ,redis: {
         host: '127.0.0.1',
-        port: 6333,
-        idleTimeoutMillis: 1,
-        reapIntervalMillis: 1
+        port: 6335,
+        // Max number of connections in each pool.
+        // Users will be put on a queue when the limit is hit.
+        // Set to maxConnection to have no possible queues.
+        // There are currently 3 pools involved in serving
+        // windshaft-cartodb requests so multiply this number
+        // by 3 to know how many possible connections will be
+        // kept open by the server. The default is 50.
+        max: 50,
+        idleTimeoutMillis: 1, // idle time before dropping connection
+        reapIntervalMillis: 1 // time between cleanups
     }
     ,sqlapi: {
         protocol: 'http',
-        host: 'localhost.lan',
-        port: 8080,
+        domain: '',
+        // This port will be used by "make check" for testing purposes
+        // It must be available
+        port: 1080,
         version: 'v1'
     }
     ,varnish: {
@@ -34,6 +69,10 @@ var config = {
         port: null,
         ttl: 86400
     }
+    // If useProfiler is true every response will be served with an
+    // X-Tiler-Profile header containing elapsed timing for various 
+    // steps taken for producing the response.
+    ,useProfiler:false
 };
 
 module.exports = config;

configure

@@ -17,16 +17,26 @@
 # --strk(2012-07-23)
 #
 
+ENVDIR=config/environments
+
+PGPORT=
+SQLAPI_PORT=
+MAPNIK_VERSION=
+ENVIRONMENT=development
+
+STATUS="$0 $*"
+
 usage() {
   echo "Usage: $0 [OPTION]"
   echo 
   echo "Configuration:"
-  echo "  --help                 display this help and exit"
-  echo "  --with-pgport=NUM      access PostgreSQL server on TCP port NUM"
+  echo "  --help                        display this help and exit"
+  echo "  --with-pgport=NUM             access PostgreSQL server on TCP port NUM [$PGPORT]"
+  echo "  --with-sqlapi-port=NUM        access SQL-API server on TCP port NUM [$SQLAPI_PORT]"
+  echo "  --with-mapnik-version=STRING  set mapnik version string [$MAPNIK_VERSION]"
+  echo "  --environment=STRING          set output environment name [$ENVIRONMENT]"
 }
 
-PGPORT=5432
-
 while test -n "$1"; do
   case "$1" in
     --help|-h)
@@ -36,6 +46,15 @@ while test -n "$1"; do
     --with-pgport=*)
       PGPORT=`echo "$1" | cut -d= -f2`
       ;;
+    --with-sqlapi-port=*)
+      SQLAPI_PORT=`echo "$1" | cut -d= -f2`
+      ;;
+    --with-mapnik-version=*)
+      MAPNIK_VERSION=`echo "$1" | cut -d= -f2`
+      ;;
+    --environment=*)
+      ENVIRONMENT=`echo "$1" | cut -d= -f2`
+      ;;
     *)
       echo "Unknown option '$1'" >&2
       usage >&2
@@ -44,12 +63,30 @@ while test -n "$1"; do
   shift
 done
 
-echo "PGPORT: $PGPORT"
+ENVEX=./${ENVDIR}/${ENVIRONMENT}.js.example
+
+if [ -z "$PGPORT" ]; then
+  PGPORT=`node -e "console.log(require('${ENVEX}').postgres.port)"`
+fi
+if [ -z "$SQLAPI_PORT" ]; then
+  SQLAPI_PORT=`node -e "console.log(require('${ENVEX}').sqlapi.port)"`
+fi
+
+echo "PGPORT: $PGPORT"
+echo "SQLAPI_PORT: $SQLAPI_PORT"
+echo "MAPNIK_VERSION: $MAPNIK_VERSION"
+echo "ENVIRONMENT: $ENVIRONMENT"
+
+o=`dirname "${ENVEX}"`/`basename "${ENVEX}" .example`
+echo "Writing $o"
+
+# See http://austinmatzko.com/2008/04/26/sed-multi-line-search-and-replace/
+sed -n "1h;1!H;\${;g;s/\(,postgres: {[^}]*port: *'\?\)[^',]*\('\?,\)/\1$PGPORT\2/;p;}" < "${ENVEX}" \
+  | sed "s/mapnik_version:.*/mapnik_version: '$MAPNIK_VERSION'/" \
+  | sed -n "1h;1!H;\${;g;s/\(,sqlapi: {[^}]*port: *'\?\)[^',]*\('\?,\)/\1$SQLAPI_PORT\2/;p;}" \
+  > "$o"
+
+STATUSFILE=config.status--${ENVIRONMENT}
+echo "Writing ${STATUSFILE}"
+echo ${STATUS} > ${STATUSFILE} && chmod +x ${STATUSFILE}
 
-# TODO: allow specifying configuration settings !
-for f in config/environments/*.example; do
-  o=`dirname "$f"`/`basename "$f" .example`
-  echo "Writing $o"
-  # See http://austinmatzko.com/2008/04/26/sed-multi-line-search-and-replace/
-  sed -n "1h;1!H;\${;g;s/\(,postgres: {[^}]*port: *'\?\)[^',]*\('\?,\)/\1$PGPORT\2/;p;}" < "$f" > "$o"
-done

lib/cartodb/cache_validator.js

@@ -1,8 +1,5 @@
 var _             = require('underscore'),
     Varnish       = require('node-varnish'),
-    request       = require('request'),
-    crypto      = require('crypto'),
-    channelCache  = {},
     varnish_queue = null;
 
 function init(host, port) {
@@ -18,72 +15,7 @@ function invalidate_db(dbname, table) {
     }
 }
 
-function generateCacheChannel(req, callback){
-    var cacheChannel = "";
-
-    // use key to call sql api with sql request if present, else just return dbname and table name
-    // base key
-    var tableNames = req.params.table;
-    var dbName     = req.params.dbname;
-    var username   = req.headers.host.split('.')[0];
-
-    // replace tableNames with the results of the explain if present
-    if (_.isString(req.params.sql) && req.params.sql != ''){
-        // initialise MD5 key of sql for cache lookups
-        var sql_md5 = generateMD5(req.params.sql);
-        var api = global.environment.sqlapi;
-        var qs  = {};
-
-        // use cache if present
-        if (!_.isNull(channelCache[sql_md5]) && !_.isUndefined(channelCache[sql_md5])) {
-            callback(channelCache[sql_md5]);
-        } else{
-            // strip out windshaft/mapnik inserted sql if present
-            var sql = req.params.sql.match(/^\((.*)\)\sas\scdbq$/);
-            sql = (sql != null) ? sql[1] : req.params.sql;
-
-            // build up api string
-            var sqlapi = api.protocol + '://' + username + '.' + api.host + ':' + api.port + '/api/' + api.version + '/sql'
-
-            // add query to querystring
-            qs.q = 'SELECT CDB_QueryTables($windshaft$' + sql + '$windshaft$)';
-
-            // add api_key if present in tile request (means table is private)
-            if (_.isString(req.params.map_key) && req.params.map_key != ''){
-                qs.api_key = req.params.map_key;
-            }
-
-            // call sql api
-            request.get({url:sqlapi, qs:qs, json:true}, function(err, response, body){
-                if (!err && response.statusCode == 200) {
-                    tableNames = body.rows[0].cdb_querytables.split(/^\{(.*)\}$/)[1];
-                } else {
-                    //oops, no SQL API. Just cache using fallback 'table' key
-                    tableNames = 'table';
-                }
-                cacheChannel = buildCacheChannel(dbName,tableNames);
-                channelCache[sql_md5] = cacheChannel; // store for caching
-                callback(cacheChannel);
-            });
-        }
-    } else {
-        cacheChannel = buildCacheChannel(dbName,tableNames);
-        callback(cacheChannel);
-    }
-}
-
-function buildCacheChannel(dbName, tableNames){
-    return dbName + ':' + tableNames;
-}
-
-function generateMD5(data){
-    var hash = crypto.createHash('md5');
-    hash.update(data);
-    return hash.digest('hex');
-}
-
 module.exports = {
     init: init,
-    invalidate_db: invalidate_db,
-    generateCacheChannel: generateCacheChannel
+    invalidate_db: invalidate_db
 }

lib/cartodb/carto_data.js

@@ -1,243 +0,0 @@
-/**
- * User: simon
- * Date: 30/08/2011
- * Time: 21:10
- * Desc: CartoDB helper.
- *       Retrieves dbname (based on subdomain/username)
- *       and geometry type from the redis stores of cartodb
- */
-
-var   RedisPool = require("./redis_pool")
-    , _ = require('underscore')
-    , Step = require('step');
-
-module.exports = function() {
-    var redis_pool = new RedisPool(global.environment.redis);
-
-
-    var me = {
-        user_metadata_db: 5,
-        table_metadata_db: 0,
-        user_key:  "rails:users:<%= username %>",
-        table_key: "rails:<%= database_name %>:<%= table_name %>"
-    };
-
-
-    /**
-     * Get the database name for this particular subdomain/username
-     *
-     * @param req - standard express req object. importantly contains host information
-     * @param callback - gets called with args(err, dbname) 
-     */
-    me.getDatabase = function(req, callback) {
-        // strip subdomain from header host
-        var username = req.headers.host.split('.')[0]
-        var redisKey = _.template(this.user_key, {username: username});
-
-        this.retrieve(this.user_metadata_db, redisKey, 'database_name', function(err, dbname) {
-          if ( err ) callback(err, null);
-          else if ( dbname === null ) {
-            callback(new Error("missing " + username + "'s dbname in redis (try CARTODB/script/restore_redis)"), null);
-          }
-          else callback(err, dbname);
-        });
-    };
-
-
-
-    /**
-     * Get the user id for this particular subdomain/username
-     *
-     * @param req - standard express req object. importantly contains host information
-     * @param callback
-     */
-    me.getId= function(req, callback) {
-        // strip subdomain from header host
-        var username = req.headers.host.split('.')[0];
-        var redisKey = _.template(this.user_key, {username: username});
-
-        this.retrieve(this.user_metadata_db, redisKey, 'id', function(err, dbname) {
-          if ( err ) callback(err, null);
-          else if ( dbname === null ) {
-            callback(new Error("missing " + username + "'s dbuser in redis (try CARTODB/script/restore_redis)"), null);
-          }
-          else callback(err, dbname);
-        });
-    };
-
-    /**
-     * Check the user map key for this particular subdomain/username
-     *
-     * @param req - standard express req object. importantly contains host information
-     * @param callback
-     */
-    me.checkMapKey = function(req, callback) {
-        // strip subdomain from header host
-        var username = req.headers.host.split('.')[0];
-        var redisKey = "rails:users:" + username; 
-        this.retrieve(this.user_metadata_db, redisKey, "map_key", function(err, val) {
-            var valid = 0;
-            if ( val ) {
-              if ( val == req.query.map_key ) valid = 1;
-              else if ( val == req.query.api_key ) valid = 1;
-              // check also in request body 
-              else if ( req.body && req.body.map_key && val == req.body.map_key ) valid = 1;
-              else if ( req.body && req.body.api_key && val == req.body.api_key ) valid = 1;
-            }
-            callback(err, valid);
-        });
-    };
-
-    /**
-     * Get privacy for cartodb table
-     *
-     * @param req - standard req object. Importantly contains table and host information
-     * @param callback - is the table private or not?
-     */
-    me.authorize= function(req, callback) {
-        var that = this;
-
-        Step(
-            function(){
-                that.checkMapKey(req, this);
-            },
-            function checkIfInternal(err, check_result){
-                if (err) throw err;
-                if (check_result === 1) {
-                    // authorized by key, login as db owner 
-                    that.getId(req, function(err, user_id) {
-                        if (err) throw new Error(err);
-                        var dbuser = _.template(global.settings.postgres_auth_user, {user_id: user_id});
-                        _.extend(req, {dbuser:dbuser});
-                        callback(err, true); 
-                    });
-                } else {
-                    return true; // continue to check if the table is public/private
-                }
-            }
-            ,function (err, data){
-                if (err) throw err;
-                that.getDatabase(req, this);
-            },
-            function(err, data){
-                if (err) throw err;
-                var redisKey = _.template(that.table_key, {database_name: data, table_name: req.params.table});
-
-                that.retrieve(that.table_metadata_db, redisKey, 'privacy', this);
-            },
-            function(err, data){
-                if (err) throw err;
-                callback(err, data);
-             }
-        );
-    };
-
-
-    /**
-     * Get the geometry type for this particular table;
-     * @param req - standard req object. Importantly contains table and host information
-     * @param callback
-     */
-    me.getGeometryType = function(req, callback){
-        var that = this;
-
-        Step(
-            function(){
-                that.getDatabase(req, this)
-            },
-            function(err, data){
-                if (err) throw err;
-                var redisKey = _.template(that.table_key, {database_name: data, table_name: req.params.table});
-
-                that.retrieve(that.table_metadata_db, redisKey, 'the_geom_type', this);
-            },
-            function(err, data){
-                if (err) throw err;
-                callback(err, data);
-            }
-        );
-    };
-
-
-    me.getInfowindow = function(req, callback){
-        var that = this;
-
-        Step(
-            function(){
-                that.getDatabase(req, this);
-            },
-            function(err, data) {
-                if (err) throw err;
-                var redisKey = _.template(that.table_key, {database_name: data, table_name: req.params.table});
-                that.retrieve(that.table_metadata_db, redisKey, 'infowindow', this);
-            },
-            function(err, data){
-                if (err) throw err;
-                callback(err, data);
-            }
-        );
-    };
-
-
-    me.getMapMetadata = function(req, callback){
-        var that = this;
-
-        Step(
-            function(){
-                that.getDatabase(req, this);
-            },
-            function(err, data) {
-                if (err) throw err;
-                var redisKey = _.template(that.table_key, {database_name: data, table_name: req.params.table});
-
-                that.retrieve(that.table_metadata_db, redisKey, 'map_metadata', this);
-            },
-            function(err, data){
-                if (err) throw err;
-                callback(err, data);
-            }
-        );
-    };
-
-    // Redis Hash lookup
-    // @param callback will be invoked with args (err, reply)
-    //                 note that reply is null when the key is missing
-    me.retrieve = function(db, redisKey, hashKey, callback) {
-        this.redisCmd(db,'HGET',[redisKey, hashKey], callback);
-    };
-
-    // Redis Set member check
-    me.inSet = function(db, setKey, member, callback) {
-        this.redisCmd(db,'SISMEMBER',[setKey, member], callback);
-    };
-
-    /**
-     * Use Redis
-     *
-     * @param db - redis database number
-     * @param redisFunc - the redis function to execute
-     * @param redisArgs - the arguments for the redis function in an array
-     * @param callback - function to pass results too.
-     */
-    me.redisCmd = function(db, redisFunc, redisArgs, callback) {
-        var redisClient;
-
-        Step(
-            function getRedisClient() {
-                redis_pool.acquire(db, this);
-            },
-            function executeQuery(err, data) {
-                redisClient = data;
-                redisArgs.push(this);
-                redisClient[redisFunc.toUpperCase()].apply(redisClient, redisArgs);
-            },
-            function releaseRedisClient(err, data) {
-                if (err) throw err;
-                redis_pool.release(db, redisClient);
-                callback(err, data);
-            }
-        );
-    };
-
-    return me;
-}();

lib/cartodb/cartodb_windshaft.js

@@ -2,6 +2,11 @@
 var _ = require('underscore')
     , Step       = require('step')
     , Windshaft = require('windshaft')
+    , redisPool = new require('redis-mpool')(global.environment.redis)
+    // TODO: instanciate cartoData with redisPool
+    , cartoData  = require('cartodb-redis')(global.environment.redis)
+    , SignedMaps = require('./signed_maps.js')
+    , TemplateMaps = require('./template_maps.js')
     , Cache = require('./cache_validator');
 
 var CartodbWindshaft = function(serverOptions) {
@@ -17,15 +22,26 @@ var CartodbWindshaft = function(serverOptions) {
 
     serverOptions.beforeStateChange = function(req, callback) {
         var err = null;
-        if ( ! req.hasOwnProperty('dbuser') ) {
+        if ( ! req.params.hasOwnProperty('dbuser') ) {
           err = new Error("map state cannot be changed by unauthenticated request!");
         }
         callback(err, req);
     }
 
+    serverOptions.signedMaps = new SignedMaps(redisPool);
+    var templateMaps = new TemplateMaps(redisPool, serverOptions.signedMaps);
+
     // boot
     var ws = new Windshaft.Server(serverOptions);
 
+    // Override getVersion to include cartodb-specific versions
+    var wsversion = ws.getVersion;
+    ws.getVersion = function() {
+      var version = wsversion();
+      version.windshaft_cartodb = require('../../package.json').version;
+      return version;
+    }
+
     /**
      * Helper to allow access to the layer to be used in the maps infowindow popup.
      */
@@ -37,7 +53,8 @@ var CartodbWindshaft = function(serverOptions) {
             },
             function(err, data){
                 if (err){
-                    res.send({error: err.message}, 500);
+                    ws.sendError(res, {error: err.message}, 500, 'GET INFOWINDOW');
+                    //res.send({error: err.message}, 500);
                 } else {
                     res.send({infowindow: data}, 200);
                 }
@@ -57,7 +74,8 @@ var CartodbWindshaft = function(serverOptions) {
             },
             function(err, data){
                 if (err){
-                    res.send(err.message, 500);
+                    ws.sendError(res, {error: err.message}, 500, 'GET MAP_METADATA');
+                    //res.send(err.message, 500);
                 } else {
                     res.send({map_metadata: data}, 200);
                 }
@@ -72,18 +90,389 @@ var CartodbWindshaft = function(serverOptions) {
     ws.del(serverOptions.base_url + '/flush_cache', function(req, res){
         ws.doCORS(res);
         Step(
-            function(){
+            function flushCache(){
                 serverOptions.flushCache(req, serverOptions.cache_enabled ? Cache : null, this);
             },
-            function(err, data){
+            function sendResponse(err, data){
                 if (err){
-                    res.send(500);
+                    ws.sendError(res, {error: err.message}, 500, 'DELETE CACHE');
+                    //res.send(500);
                 } else {
                     res.send({status: 'ok'}, 200);
                 }
             }
         );
     });
+
+    // ---- Template maps interface starts @{
+
+    ws.userByReq = function(req) {
+        return serverOptions.userByReq(req);
+    }
+
+    var template_baseurl = serverOptions.base_url_notable + '/template';
+
+    // Add a template
+    ws.post(template_baseurl, function(req, res) {
+      ws.doCORS(res);
+      var that = this;
+      var response = {};
+      var cdbuser = ws.userByReq(req);
+      Step(
+        function checkPerms(){
+            serverOptions.authorizedByAPIKey(req, this);
+        },
+        function addTemplate(err, authenticated) {
+          if ( err ) throw err;
+          if (authenticated !== 1) {
+            err = new Error("Only authenticated user can create templated maps");
+            err.http_status = 401;
+            throw err;
+          }
+          var next = this;
+          if ( ! req.headers['content-type'] || req.headers['content-type'].split(';')[0] != 'application/json' )
+            throw new Error('template POST data must be of type application/json');
+          var cfg = req.body;
+          templateMaps.addTemplate(cdbuser, cfg, this);
+        },
+        function prepareResponse(err, tpl_id){
+          if ( err ) throw err;
+          // NOTE: might omit "cdbuser" if == dbowner ...
+          return { template_id: cdbuser + '@' + tpl_id };
+        },
+        function finish(err, response){
+            if (err){
+                response = { error: ''+err };
+                var statusCode = 400;
+                if ( ! _.isUndefined(err.http_status) ) {
+                  statusCode = err.http_status;
+                }
+                ws.sendError(res, response, statusCode, 'POST TEMPLATE', err.message);
+            } else {
+              res.send(response, 200);
+            }
+        }
+      );
+    });
+
+    // Update a template
+    ws.put(template_baseurl + '/:template_id', function(req, res) {
+      ws.doCORS(res);
+      var that = this;
+      var response = {};
+      var cdbuser = ws.userByReq(req);
+      var template;
+      var tpl_id;
+      Step(
+        function checkPerms(){
+            serverOptions.authorizedByAPIKey(req, this);
+        },
+        function updateTemplate(err, authenticated) {
+          if ( err ) throw err;
+          if (authenticated !== 1) {
+            err = new Error("Only authenticated user can list templated maps");
+            err.http_status = 401;
+            throw err;
+          }
+          if ( ! req.headers['content-type'] || req.headers['content-type'].split(';')[0] != 'application/json' )
+            throw new Error('template PUT data must be of type application/json');
+          template = req.body;
+          tpl_id = req.params.template_id.split('@');
+          if ( tpl_id.length > 1 ) {
+            if ( tpl_id[0] != cdbuser ) {
+              err = new Error("Invalid template id '"
+                + req.params.template_id + "' for user '" + cdbuser + "'");
+              err.http_status = 404;
+              throw err;
+            }
+            tpl_id = tpl_id[1];
+          }
+          templateMaps.updTemplate(cdbuser, tpl_id, template, this);
+        },
+        function prepareResponse(err){
+          if ( err ) throw err;
+          return { template_id: cdbuser + '@' + tpl_id };
+        },
+        function finish(err, response){
+            if (err){
+                var statusCode = 400;
+                response = { error: ''+err };
+                if ( ! _.isUndefined(err.http_status) ) {
+                  statusCode = err.http_status;
+                }
+                ws.sendError(res, response, statusCode, 'PUT TEMPLATE', err.message);
+            } else {
+              res.send(response, 200);
+            }
+        }
+      );
+    });
+
+    // Get a specific template
+    ws.get(template_baseurl + '/:template_id', function(req, res) {
+      ws.doCORS(res);
+      var that = this;
+      var response = {};
+      var cdbuser = ws.userByReq(req);
+      var template;
+      var tpl_id;
+      Step(
+        function checkPerms(){
+            serverOptions.authorizedByAPIKey(req, this);
+        },
+        function updateTemplate(err, authenticated) {
+          if ( err ) throw err;
+          if (authenticated !== 1) {
+            err = new Error("Only authenticated users can get template maps");
+            err.http_status = 401;
+            throw err;
+          }
+          tpl_id = req.params.template_id.split('@');
+          if ( tpl_id.length > 1 ) {
+            if ( tpl_id[0] != cdbuser ) {
+              var err = new Error("Cannot get template id '"
+                + req.params.template_id + "' for user '" + cdbuser + "'");
+              err.http_status = 404;
+              throw err;
+            }
+            tpl_id = tpl_id[1];
+          }
+          templateMaps.getTemplate(cdbuser, tpl_id, this);
+        },
+        function prepareResponse(err, tpl_val){
+          if ( err ) throw err;
+          if ( ! tpl_val ) {
+            err = new Error("Cannot find template '" + tpl_id + "' of user '" + cdbuser + "'");
+            err.http_status = 404;
+            throw err;
+          }
+          // auth_id was added by ourselves,
+          // so we remove it before returning to the user
+          delete tpl_val.auth_id;
+          return { template: tpl_val };
+        },
+        function finish(err, response){
+            if (err){
+                var statusCode = 400;
+                response = { error: ''+err };
+                if ( ! _.isUndefined(err.http_status) ) {
+                  statusCode = err.http_status;
+                }
+                ws.sendError(res, response, statusCode, 'GET TEMPLATE', err.message);
+            } else {
+              res.send(response, 200);
+            }
+        }
+      );
+    });
+
+    // Delete a specific template
+    ws.delete(template_baseurl + '/:template_id', function(req, res) {
+      ws.doCORS(res);
+      var that = this;
+      var response = {};
+      var cdbuser = ws.userByReq(req);
+      var template;
+      var tpl_id;
+      Step(
+        function checkPerms(){
+            serverOptions.authorizedByAPIKey(req, this);
+        },
+        function updateTemplate(err, authenticated) {
+          if ( err ) throw err;
+          if (authenticated !== 1) {
+            err = new Error("Only authenticated users can delete template maps");
+            err.http_status = 401;
+            throw err;
+          }
+          tpl_id = req.params.template_id.split('@');
+          if ( tpl_id.length > 1 ) {
+            if ( tpl_id[0] != cdbuser ) {
+              var err = new Error("Cannot find template id '"
+                + req.params.template_id + "' for user '" + cdbuser + "'");
+              err.http_status = 404;
+              throw err;
+            }
+            tpl_id = tpl_id[1];
+          }
+          templateMaps.delTemplate(cdbuser, tpl_id, this);
+        },
+        function prepareResponse(err, tpl_val){
+          if ( err ) throw err;
+          return { status: 'ok' };
+        },
+        function finish(err, response){
+            if (err){
+                var statusCode = 400;
+                response = { error: ''+err };
+                if ( ! _.isUndefined(err.http_status) ) {
+                  statusCode = err.http_status;
+                }
+                ws.sendError(res, response, statusCode, 'DELETE TEMPLATE', err.message);
+            } else {
+              res.send('', 204);
+            }
+        }
+      );
+    });
+
+    // Get a list of owned templates 
+    ws.get(template_baseurl, function(req, res) {
+      ws.doCORS(res);
+      var that = this;
+      var response = {};
+      var cdbuser = ws.userByReq(req);
+      Step(
+        function checkPerms(){
+            serverOptions.authorizedByAPIKey(req, this);
+        },
+        function listTemplates(err, authenticated) {
+          if ( err ) throw err;
+          if (authenticated !== 1) {
+            err = new Error("Only authenticated user can list templated maps");
+            err.http_status = 401;
+            throw err;
+          }
+          templateMaps.listTemplates(cdbuser, this);
+        },
+        function prepareResponse(err, tpl_ids){
+          if ( err ) throw err;
+          // NOTE: might omit "cbduser" if == dbowner ...
+          var ids = _.map(tpl_ids, function(id) { return cdbuser + '@' + id; })
+          return { template_ids: ids };
+        },
+        function finish(err, response){
+            var statusCode = 200;
+            if (err){
+                response = { error: ''+err };
+                if ( ! _.isUndefined(err.http_status) ) {
+                  statusCode = err.http_status;
+                }
+                ws.sendError(res, response, statusCode, 'GET TEMPLATE LIST', err.message);
+            } else {
+              res.send(response, statusCode);
+            }
+        }
+      );
+    });
+
+    ws.setDBParams = function(cdbuser, params, callback) {
+      Step(
+        function setAuth() {
+          serverOptions.setDBAuth(cdbuser, params, this);
+        },
+        function setConn(err) {
+          if ( err ) throw err;
+          serverOptions.setDBConn(cdbuser, params, this);
+        },
+        function finish(err) {
+          callback(err);
+        }
+      );
+    };
+
+    // Instantiate a template
+    ws.post(template_baseurl + '/:template_id', function(req, res) {
+      ws.doCORS(res);
+      var that = this;
+      var response = {};
+      var template;
+      var signedMaps = serverOptions.signedMaps;
+      var layergroup;
+      var layergroupid;
+      var fakereq; // used for call to createLayergroup
+      var cdbuser = ws.userByReq(req);
+      // Format of template_id: [<template_owner>]@<template_id>
+      var tpl_id = req.params.template_id.split('@');
+      if ( tpl_id.length > 1 ) {
+        if ( tpl_id[0] ) cdbuser = tpl_id[0];
+        tpl_id = tpl_id[1];
+      }
+      var auth_token = req.query.auth_token;
+      Step(
+        function getTemplate(){
+          templateMaps.getTemplate(cdbuser, tpl_id, this);
+        },
+        function checkAuthorized(err, data) {
+          if ( err ) throw err;
+          if ( ! data ) {
+            err = new Error("Template '" + tpl_id + "' of user '" + cdbuser + "' not found");
+            err.http_status = 404;
+            throw err;
+          }
+          template = data;
+          var cert = templateMaps.getTemplateCertificate(template);
+          var authorized = false;
+          try {
+            // authorizedByCert will throw if unauthorized
+            authorized = signedMaps.authorizedByCert(cert, auth_token);
+          } catch (err) {
+            // we catch to add http_status
+            err.http_status = 401;
+            throw err;
+          }
+          if ( ! authorized ) {
+            err = new Error('Unauthorized template instanciation');
+            err.http_status = 401;
+            throw err;
+          }
+          if ( ! req.headers['content-type'] || req.headers['content-type'].split(';')[0] != 'application/json' )
+            throw new Error('template POST data must be of type application/json, it is instead ');
+          var template_params = req.body;
+          return templateMaps.instance(template, template_params);
+        },
+        function prepareParams(err, instance){
+          if ( err ) throw err;
+          layergroup = instance;
+          fakereq = { query: {}, params: {}, headers: _.clone(req.headers) };
+          ws.setDBParams(cdbuser, fakereq.params, this);
+        },
+        function createLayergroup(err) {
+          if ( err ) throw err;
+          ws.createLayergroup(layergroup, fakereq, this);
+        },
+        function signLayergroup(err, resp) {
+          if ( err ) throw err;
+          response = resp;
+          var signer = cdbuser;
+          var map_id = response.layergroupid.split(':')[0]; // dropping last_updated 
+          var crt_id = template.auth_id; // check ?
+          if ( ! crt_id ) {
+            var errmsg = "Template '" + tpl_id + "' of user '" + cdbuser + "' has no signature";
+            // Is this really illegal ?
+            // Maybe we could just return an unsigned layergroupid
+            // in this case...
+            err = new Error(errmsg);
+            err.http_status = 403; // Forbidden, we refuse to respond to this
+            throw err;
+          }
+          signedMaps.signMap(signer, map_id, crt_id, this);
+        },
+        function prepareResponse(err) {
+          if ( err ) throw err;
+          //console.log("Response from createLayergroup: "); console.dir(response);
+          // Add the signature part to the token!
+          response.layergroupid = cdbuser + '@' + response.layergroupid;
+          return response;
+        },
+        function finish(err, response){
+            if (err){
+                var statusCode = 400;
+                response = { error: ''+err };
+                if ( ! _.isUndefined(err.http_status) ) {
+                  statusCode = err.http_status;
+                }
+                ws.sendError(res, response, statusCode, 'POST INSTANCE TEMPLATE', err.message);
+            } else {
+              res.send(response, 200);
+            }
+        }
+      );
+    });
+
+
+    // ---- Template maps interface ends @}
+
     return ws;
 }
 

lib/cartodb/redis_pool.js

@@ -1,77 +0,0 @@
-/**
- * RedisPool. A database specific redis pooling lib
- *
- */
-
-var redis  = require('redis')
-  , _      = require('underscore')
-  , Pool   = require('generic-pool').Pool;
-
-// constructor.
-// 
-// - `opts` {Object} optional config for redis and pooling
-var RedisPool = function(opts){
-  var opts = opts || {};
-  var defaults = {
-    host: '127.0.0.1', 
-    port: '6379', 
-    max: 50, 
-    idleTimeoutMillis: 10000, 
-    reapIntervalMillis: 1000, 
-    log: false
-  };    
-  var options = _.defaults(opts, defaults)
-
-  var me = {
-    pools: {} // cached pools by DB name
-  };
-  
-  // Acquire resource.
-  //
-  // - `database` {String} redis database name
-  // - `callback` {Function} callback to call once acquired. Takes the form
-  //   `callback(err, resource)`  
-  me.acquire = function(database, callback) {
-      if (!this.pools[database]) {
-        this.pools[database] = this.makePool(database);            
-      }
-      this.pools[database].acquire(function(err,resource) {
-        callback(err, resource);
-      });
-  };
-  
-  // Release resource.
-  //
-  // - `database` {String} redis database name
-  // - `resource` {Object} resource object to release
-  me.release = function(database, resource) {
-      this.pools[database] && this.pools[database].release(resource);
-  };
-    
-  // Factory for pool objects.
-  me.makePool = function(database) {
-    return Pool({
-      name: database,
-      create: function(callback){
-        var client = redis.createClient(options.port, options.host);          
-        client.on('connect', function () {
-          client.send_anyway = true;
-          client.select(database);  
-          client.send_anyway = false;
-        });    
-        return callback(null, client);
-      },
-      destroy: function(client) { 
-        return client.quit(); 
-      },
-      max: options.max, 
-      idleTimeoutMillis: options.idleTimeoutMillis, 
-      reapIntervalMillis: options.reapIntervalMillis, 
-      log: options.log 
-    });
-  };
-      
-  return me;
-};
-
-module.exports = RedisPool;

lib/cartodb/server_options.js

@@ -1,21 +1,205 @@
 var   _          = require('underscore')
     , Step       = require('step')
-    , cartoData  = require('./carto_data')
-    , Cache = require('./cache_validator');
+    , cartoData  = require('cartodb-redis')(global.environment.redis)
+    , Cache = require('./cache_validator')
+    , mapnik     = require('mapnik')
+    , crypto     = require('crypto')
+    , request    = require('request')
+    , LZMA       = require('lzma/lzma_worker.js').LZMA
+  ;
+
+// This is for backward compatibility with 1.3.3
+if ( _.isUndefined(global.environment.sqlapi.domain) ) {
+  global.environment.sqlapi.domain = global.environment.sqlapi.host;
+}
 
 module.exports = function(){
+
+    var rendererConfig = _.defaults(global.environment.renderer || {}, {
+      cache_ttl: 60000, // milliseconds
+      metatile: 4,
+      bufferSize: 64
+    });
+
     var me = {
         base_url: '/tiles/:table',
+        base_url_notable: '/tiles',
         grainstore: {
           datasource: global.environment.postgres,
-          cachedir: global.environment.millstone.cache_basedir
+          cachedir: global.environment.millstone.cache_basedir,
+          mapnik_version: global.environment.mapnik_version || mapnik.versions.mapnik,
+          default_layergroup_ttl: 7200, // seconds (default is 300)
+          gc_prob: 0.01 // default is 0.01 TODO: make configurable via env config
+        },
+        mapnik: {
+          metatile: rendererConfig.metatile,
+          bufferSize: rendererConfig.bufferSize
+        },
+        renderCache: {
+          ttl: rendererConfig.cache_ttl
         },
         redis: global.environment.redis,
         enable_cors: global.environment.enable_cors,
         varnish_host: global.environment.varnish.host,
         varnish_port: global.environment.varnish.port,
         cache_enabled: global.environment.cache_enabled,
-        log_format: global.environment.log_format
+        log_format: global.environment.log_format,
+        useProfiler: global.environment.useProfiler
+    };
+
+    // Be nice and warn if configured mapnik version
+    // is != instaled mapnik version
+    if ( mapnik.versions.mapnik != me.grainstore.mapnik_version ) {
+      console.warn("WARNING: detected mapnik version ("
+        + mapnik.versions.mapnik + ") != configured mapnik version ("
+        + me.grainstore.mapnik_version + ")");
+    }
+
+/* This whole block is about generating X-Cache-Channel { */
+
+    // TODO: review lifetime of elements of this cache
+    // NOTE: by-token indices should only be dropped when
+    //       the corresponding layegroup is dropped, because
+    //       we have no SQL after layer creation.
+    me.channelCache = {};
+
+    // Run a query through the SQL api
+    me.sqlQuery = function (username, api_key, sql, callback) {
+      var api = global.environment.sqlapi;
+
+      // build up api string
+      var sqlapi = api.protocol + '://' + username;
+      if ( api.domain ) sqlapi += '.' + api.domain;
+      sqlapi += ':' + api.port + '/api/' + api.version + '/sql'
+
+      var qs  = { q: sql }
+
+      // add api_key if given
+      if (_.isString(api_key) && api_key != '') { qs.api_key = api_key; }
+
+      // call sql api
+      request.get({url:sqlapi, qs:qs, json:true}, function(err, res, body){
+          if (err){
+            console.log('ERROR connecting to SQL API on ' + sqlapi + ': ' + err);
+            callback(err);
+            return;
+          }
+          if (res.statusCode != 200) {
+              var msg = res.body.error ? res.body.error : res.body;
+              callback(new Error(msg));
+              console.log('unexpected response status (' + res.statusCode + ') for sql query: ' + sql + ': ' + msg);
+              return;
+          }
+          callback(null, body.rows);
+      });
+    };
+
+    //
+    // Invoke callback with number of milliseconds since
+    // last update in any of the given tables
+    //
+    me.findLastUpdated = function (username, api_key, tableNames, callback) {
+      var sql = 'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max FROM CDB_TableMetadata m WHERE m.tabname::name = any (\'{'
+        + tableNames.join(',') + '}\')';
+
+      // call sql api
+      me.sqlQuery(username, api_key, sql, function(err, rows){
+          if (err){
+            var msg = err.message ? err.message : err;
+            callback(new Error('could not find last updated timestamp: ' + msg));
+            return;
+          }
+          // when the table has not updated_at means it hasn't been changed so a default last_updated is set
+          var last_updated = 0;
+          if(rows.length !== 0) {
+            last_updated = rows[0].max || 0;
+          }
+          callback(null, last_updated*1000);
+      });
+    };
+
+    me.affectedTables = function (username, api_key, sql, callback) {
+
+      // Replace mapnik tokens
+      sql = sql.replace(RegExp('!bbox!', 'g'), 'ST_MakeEnvelope(0,0,0,0)')
+               .replace(RegExp('!pixel_width!', 'g'), '1')
+               .replace(RegExp('!pixel_height!', 'g'), '1')
+            ;
+
+      // Pass to CDB_QueryTables
+      sql = 'SELECT CDB_QueryTables($windshaft$' + sql + '$windshaft$)';
+
+      // call sql api
+      me.sqlQuery(username, api_key, sql, function(err, rows){
+          if (err){
+            var msg = err.message ? err.message : err;
+            callback(new Error('could not fetch source tables: ' + msg));
+            return;
+          }
+          var qtables = rows[0].cdb_querytables;
+          var tableNames = qtables.split(/^\{(.*)\}$/)[1];
+          tableNames = tableNames.split(',');
+          callback(null, tableNames);
+      });
+    };
+
+    me.buildCacheChannel = function (dbName, tableNames){
+      return dbName + ':' + tableNames.join(',');
+    };
+
+    me.generateMD5 = function(data){
+        var hash = crypto.createHash('md5');
+        hash.update(data);
+        return hash.digest('hex');
+    }
+
+    me.generateCacheChannel = function(req, callback){
+
+      // use key to call sql api with sql request if present, else
+      // just return dbname and table name base key
+      var dbName     = req.params.dbname;
+
+      var cacheKey = [ dbName ];
+      if ( req.params.token ) cacheKey.push(req.params.token);
+      else if ( req.params.sql ) cacheKey.push( me.generateMD5(req.params.sql) );
+      cacheKey = cacheKey.join(':');
+
+      if ( me.channelCache.hasOwnProperty(cacheKey) ) {
+        callback(null, me.channelCache[cacheKey]);
+        return;
+      }
+      else if ( req.params.token ) {
+        // cached cache channel for token-based access should be constructed
+        // at cache creation time
+        callback(new Error('missing channel cache for token ' + req.params.token));
+        return;
+      }
+
+      if ( ! req.params.sql && ! req.params.token ) {
+        var cacheChannel = me.buildCacheChannel(dbName, [req.params.table]);
+        // not worth caching this
+        callback(null, cacheChannel);
+        return;
+      }
+
+      if ( ! req.params.sql ) {
+        callback(new Error("this request doesn't need an X-Cache-Channel generated"));
+        return;
+      }
+
+      var dbName     = req.params.dbname;
+      var username   = this.userByReq(req);
+
+      // strip out windshaft/mapnik inserted sql if present
+      var sql = req.params.sql.match(/^\((.*)\)\sas\scdbq$/);
+      sql = (sql != null) ? sql[1] : req.params.sql;
+
+      me.affectedTables(username, req.params.map_key, sql, function(err, tableNames) {
+          if ( err ) { callback(err); return; }
+          var cacheChannel = me.buildCacheChannel(dbName,tableNames);
+          me.channelCache[cacheKey] = cacheChannel; // store for caching
+          callback(null, cacheChannel);
+      });
     };
 
     // Set the cache chanel info to invalidate the cache on the frontend server
@@ -31,15 +215,318 @@ module.exports = function(){
         // skip non-GET requests, or requests for which there's no response
         if ( req.method != 'GET' || ! req.res ) { cb(null, null); return; }
         var res = req.res;
-        var ttl = global.environment.varnish.ttl || 86400;
-        Cache.generateCacheChannel(req, function(channel){
-            res.header('X-Cache-Channel', channel);
-            res.header('Last-Modified', new Date().toUTCString());
-            res.header('Cache-Control', 'no-cache,max-age='+ttl+',must-revalidate, public');
-            cb(null, channel); // add last-modified too ?
+        var cache_policy = req.query.cache_policy;
+        if ( req.params.token ) cache_policy = 'persist';
+        if ( cache_policy == 'persist' ) {
+          res.header('Cache-Control', 'public,max-age=31536000'); // 1 year
+        } else {
+          var ttl = global.environment.varnish.ttl || 86400;
+          res.header('Cache-Control', 'no-cache,max-age='+ttl+',must-revalidate, public');
+        }
+
+        // Set Last-Modified header
+        var lastUpdated;
+        if ( req.params.cache_buster ) {
+          // Assuming cache_buster is a timestamp
+          // FIXME: store lastModified in the cache channel instead
+          lastUpdated = new Date(parseInt(req.params.cache_buster));
+        } else {
+          lastUpdated = new Date();
+        }
+        res.header('Last-Modified', lastUpdated.toUTCString());
+
+        me.generateCacheChannel(req, function(err, channel){
+            if ( ! err ) {
+              res.header('X-Cache-Channel', channel);
+              cb(null, channel);
+            } else {
+              console.log('ERROR generating cache channel: ' + ( err.message ? err.message : err ));
+              // TODO: evaluate if we should bubble up the error instead
+              cb(null, 'ERROR');
+            }
         });
+    };
+
+    me.afterLayergroupCreate = function(req, mapconfig, response, callback) {
+        var token = response.layergroupid;
+
+        var username = this.userByReq(req);
+
+        var tasksleft = 2; // redis key and affectedTables
+        var errors = [];
+
+        var done = function(err) {
+          if ( err ) {
+            errors.push('' + err);
+          }
+          if ( ! --tasksleft ) {
+            err = errors.length ? new Error(errors.join('\n')) : null;
+            callback(err);
+          }
+        }
+
+        // Don't wait for the mapview count increment to
+        // take place before proceeding. Error will be logged
+        // asyncronously
+        cartoData.incMapviewCount(username, mapconfig.stat_tag, function(err) {
+          if (req.profiler) req.profiler.done('incMapviewCount');
+          if ( err ) console.log("ERROR: failed to increment mapview count for user '" + username + "': " + err);
+          done();
+        });
+
+        var sql = [];
+        _.each(mapconfig.layers, function(lyr) {
+          sql.push(lyr.options.sql);
+        });
+        sql = sql.join(';');
+
+        var dbName = req.params.dbname;
+        var usr    = this.userByReq(req);
+        var key    = req.params.map_key;
+
+        var cacheKey = dbName + ':' + token;
+
+        me.affectedTables(usr, key, sql, function(err, tableNames) {
+            if (req.profiler) req.profiler.done('affectedTables');
+
+            if ( err ) { done(err); return; }
+            var cacheChannel = me.buildCacheChannel(dbName,tableNames);
+            me.channelCache[cacheKey] = cacheChannel; // store for caching
+            // find last updated
+            me.findLastUpdated(usr, key, tableNames, function(err, lastUpdated) {
+              if (req.profiler) req.profiler.done('findLastUpdated');
+              if ( err ) { done(err); return; }
+              response.layergroupid = response.layergroupid + ':' + lastUpdated; // use epoch
+              response.last_updated = new Date(lastUpdated).toISOString(); // TODO: use ISO format
+              done(null);
+            });
+        });
+    };
+
+/* X-Cache-Channel generation } */
+
+    me.re_userFromHost = new RegExp(
+      global.environment.user_from_host ||
+      '^([^\\.]+)\\.' // would extract "strk" from "strk.cartodb.com"
+    );
+
+    me.userByReq = function(req) {
+      var host = req.headers.host;
+      var mat = host.match(this.re_userFromHost);
+      if ( ! mat ) {
+        console.error("ERROR: user pattern '" + this.re_userFromHost
+          + "' does not match hostname '" + host + "'");
+        return;
+      }
+      // console.log("Matches: "); console.dir(mat);
+      if ( ! mat.length === 2 ) {
+        console.error("ERROR: pattern '" + this.re_userFromHost
+          + "' gave unexpected matches against '" + host + "': " + mat);
+        return;
+      }
+      return mat[1];
     }
 
+    // Set db authentication parameters to those of the given username
+    //
+    // @param username the cartodb username, mapped to a database username
+    //                 via CartodbRedis metadata records
+    //
+    // @param params the parameters to set auth options into
+    //               added params are: "dbuser" and "dbpassword"
+    //
+    // @param callback function(err) 
+    //
+    me.setDBAuth = function(username, params, callback) {
+
+      var user_params = {};
+      var auth_user = global.environment.postgres_auth_user;
+      var auth_pass = global.environment.postgres_auth_pass;
+      Step(
+        function getId() {
+          cartoData.getUserId(username, this);
+        },
+        function(err, user_id) {
+          if (err) throw err;
+          user_params['user_id'] = user_id;
+          var dbuser = _.template(auth_user, user_params);
+          _.extend(params, {dbuser:dbuser});
+
+          // skip looking up user_password if postgres_auth_pass
+          // doesn't contain the "user_password" label 
+          if (!auth_pass || ! auth_pass.match(/\buser_password\b/) ) return null;
+
+          cartoData.getUserDBPass(username, this);
+        },
+        function(err, user_password) {
+          if (err) throw err;
+          user_params['user_password'] = user_password;
+          if ( auth_pass ) {
+            var dbpass = _.template(auth_pass, user_params);
+            _.extend(params, {dbpassword:dbpass});
+          }
+          return true;
+        },
+        function finish(err) {
+          callback(err); 
+        }
+      );
+    };
+
+    // Set db connection parameters to those for the given username
+    //
+    // @param dbowner cartodb username of database owner,
+    //                mapped to a database username
+    //                via CartodbRedis metadata records
+    //
+    // @param params the parameters to set connection options into
+    //               added params are: "dbname", "dbhost"
+    //
+    // @param callback function(err) 
+    //
+    me.setDBConn = function(dbowner, params, callback) {
+      Step(
+        function getDatabaseHost(){
+            cartoData.getUserDBHost(dbowner, this);
+        },
+        function getDatabase(err, data){
+            if(err) throw err;
+            if ( data ) _.extend(params, {dbhost:data});
+            cartoData.getUserDBName(dbowner, this);
+        },
+        function getGeometryType(err, data){
+            if (err) throw err;
+            if ( data ) _.extend(params, {dbname:data});
+            return null;
+        },
+        function finish(err) {
+          callback(err); 
+        }
+      );
+    };
+
+
+    // Check if a request is authorized by a signer
+    //
+    // Any existing signature for the given request will verified
+    // for authorization to this specific request (may require auth_token)
+    // See https://github.com/CartoDB/Windshaft-cartodb/wiki/Signed-maps
+    //
+    // @param req express request object
+    // @param callback function(err, signed_by) signed_by will be
+    //                 null if the request is not signed by anyone
+    //                 or will be a string cartodb username otherwise.
+    //                 
+    me.authorizedBySigner = function(req, callback)
+    {
+      if ( ! req.params.token || ! req.params.signer ) {
+        //console.log("No signature provided"); // debugging
+        callback(null, null); // no signer requested
+        return;
+      }
+
+      var signer = req.params.signer;
+      var layergroup_id = req.params.token;
+      var auth_token = req.params.auth_token;
+
+console.log("Checking authorization from signer " + signer + " for resource " + layergroup_id + " with auth_token " + auth_token);
+
+      me.signedMaps.isAuthorized(signer, layergroup_id, auth_token,
+        function(err, authorized) {
+          callback(err, authorized ? signer : null);
+        });
+    };
+
+    // Check if a request is authorized by api_key
+    //
+    // @param req express request object
+    // @param callback function(err, authorized) 
+    //                 
+    me.authorizedByAPIKey = function(req, callback)
+    {
+        var user = me.userByReq(req);
+        Step(
+          function (){
+              cartoData.getUserMapKey(user, this);
+          },
+          function checkApiKey(err, val){
+              if (err) throw err;
+
+              var valid = 0;
+              if ( val ) {
+                if ( val == req.query.map_key ) valid = 1;
+                else if ( val == req.query.api_key ) valid = 1;
+                // check also in request body
+                else if ( req.body && req.body.map_key && val == req.body.map_key ) valid = 1;
+                else if ( req.body && req.body.api_key && val == req.body.api_key ) valid = 1;
+              }
+              return valid;
+          },
+          function finish(err, authorized) {
+              callback(err, authorized);
+          }
+        );
+    };
+
+    /**
+     * Check access authorization
+     *
+     * @param req - standard req object. Importantly contains table and host information
+     * @param callback function(err, allowed) is access allowed not?
+     */
+    me.authorize = function(req, callback) {
+        var that = this;
+        var user = me.userByReq(req);
+
+        Step(
+            function (){
+                that.authorizedByAPIKey(req, this);
+            },
+            function checkApiKey(err, authorized){
+                if (err) throw err;
+
+                // if not authorized by api_key, continue 
+                if (authorized !== 1)  {
+                  // not authorized by api_key, 
+                  // check if authorized by signer
+                  that.authorizedBySigner(req, this);
+                  return;
+                }
+
+                // authorized by api key, login as the given username and stop
+                that.setDBAuth(user, req.params, function(err) {
+                  callback(err, true); // authorized (or error)
+                });
+            },
+            function checkSignAuthorized(err, signed_by){
+                if (err) throw err;
+
+                if ( ! signed_by ) {
+                  // request not authorized by signer, continue
+                  // to check map privacy
+                  return null;
+                }
+
+                // Authorized by "signed_by" !
+                that.setDBAuth(signed_by, req.params, function(err) {
+                  callback(err, true); // authorized (or error)
+                });
+            },
+            function getDatabase(err){
+                if (err) throw err;
+                // NOTE: only used to get to table privacy
+                cartoData.getUserDBName(user, this);
+            },
+            function getPrivacy(err, dbname){
+                if (err) throw err;
+                cartoData.getTablePrivacy(dbname, req.params.table, this);
+            },
+            function(err, privacy){
+                callback(err, privacy);
+            }
+        );
+    };
+
     /**
      * Whitelist input and get database name & default geometry type from
      * subdomain/user metadata held in CartoDB Redis
@@ -48,13 +535,55 @@ module.exports = function(){
      */
     me.req2params = function(req, callback){
 
+        if ( req.query.lzma ) {
+
+          // TODO: check ?
+          //console.log("type of req.query.lzma is " + typeof(req.query.lzma));
+
+          // Decode (from base64)
+          var lzma = (new Buffer(req.query.lzma, 'base64').toString('binary')).split('').map(function(c) { return c.charCodeAt(0) - 128 })
+
+          // Decompress
+          LZMA.decompress(
+            lzma,
+            function(result) {
+              if (req.profiler) req.profiler.done('LZMA decompress');
+              try {
+                delete req.query.lzma
+                _.extend(req.query, JSON.parse(result))
+                me.req2params(req, callback);
+              } catch (err) {
+                callback(new Error('Error parsing lzma as JSON: ' + err));
+              }
+            },
+            function(percent) { // progress
+              //console.log("LZMA decompression " + percent + "%");
+            }
+          );
+          return;
+        }
+
         // Whitelist query parameters and attach format
-        var good_query = ['sql', 'geom_type', 'cache_buster','callback', 'interactivity', 'map_key', 'api_key', 'style'];
+        var good_query = ['sql', 'geom_type', 'cache_buster', 'cache_policy', 'callback', 'interactivity', 'map_key', 'api_key', 'auth_token', 'style', 'style_version', 'style_convert', 'config' ];
         var bad_query  = _.difference(_.keys(req.query), good_query);
 
         _.each(bad_query, function(key){ delete req.query[key]; });
         req.params =  _.extend({}, req.params); // shuffle things as request is a strange array/object
 
+        if ( req.params.token ) {
+          //console.log("Request parameters include token " + req.params.token);
+          var tksplit = req.params.token.split(':');
+          req.params.token = tksplit[0];
+          if ( tksplit.length > 1 ) req.params.cache_buster= tksplit[1];
+          tksplit = req.params.token.split('@');
+          if ( tksplit.length > 1 ) {
+            req.params.signer = this.userByReq(req);
+            if ( tksplit[0] ) req.params.signer = tksplit[0];
+            req.params.token = tksplit[1];
+            //console.log("Request for token " + req.params.token + " with signature from " + req.params.signer);
+          }
+        }
+
         // bring all query values onto req.params object
         _.extend(req.params, req.query);
 
@@ -62,41 +591,59 @@ module.exports = function(){
         req.params.interactivity = req.params.interactivity || 'cartodb_id';
 
         req.params.processXML = function(req, xml, callback) {
-          var dbuser = req.dbuser ? req.dbuser : global.settings.postgres.user;
-          if ( ! me.rx_dbuser ) me.rx_dbuser = /(<Parameter name="user"><!\[CDATA\[)[^\]]*(]]><\/Parameter>)/;
+
+          // Replace dbuser
+          var dbuser = req.params.dbuser || global.environment.postgres.user;
+          if ( ! me.rx_dbuser ) me.rx_dbuser = /(<Parameter name="user"><!\[CDATA\[)[^\]]*(]]><\/Parameter>)/g;
           xml = xml.replace(me.rx_dbuser, "$1" + dbuser + "$2");
+
+          // Replace dbpass
+          var dbpass = req.params.dbpassword || global.environment.postgres.password;
+          if ( ! me.rx_dbpass ) me.rx_dbpass = /(<Parameter name="password"><!\[CDATA\[)[^\]]*(]]><\/Parameter>)/g;
+          xml = xml.replace(me.rx_dbpass, "$1" + dbpass + "$2");
+
+          // Replace or set dbhost
+          var dbhost = req.params.dbhost || global.environment.postgres.host;
+          if ( ! me.rx_dbhost ) me.rx_dbhost = /(<Parameter name="host"><!\[CDATA\[)[^\]]*(]]><\/Parameter>)/g;
+          xml = xml.replace(me.rx_dbhost, "$1" + dbhost + "$2");
+
           callback(null, xml);
         }
 
         var that = this;
 
+        if (req.profiler) req.profiler.done('req2params.setup');
+
+        var user = me.userByReq(req);
+
         Step(
             function getPrivacy(){
-                cartoData.authorize(req, this);
+                me.authorize(req, this);
             },
             function gatekeep(err, data){
+                if (req.profiler) req.profiler.done('authorize');
                 if(err) throw err;
                 if(data === "0") throw new Error("Sorry, you are unauthorized (permission denied)");
                 return data;
             },
             function getDatabase(err, data){
                 if(err) throw err;
-
-                cartoData.getDatabase(req, this);
+                that.setDBConn(user, req.params, this);
             },
-            function getGeometryType(err, data){
+            function getGeometryType(err){
+                if (req.profiler) req.profiler.done('cartoData.getDatabase');
                 if (err) throw err;
-                _.extend(req.params, {dbname:data});
-
-                cartoData.getGeometryType(req, this);
+                cartoData.getTableGeometryType(req.params.dbname, req.params.table, this);
             },
             function finishSetup(err, data){
+                if (req.profiler) req.profiler.done('cartoData.getGeometryType');
                 if ( err ) { callback(err, req); return; }
 
                 if (!_.isNull(data))
                     _.extend(req.params, {geom_type: data});
 
-                that.addCacheChannel(req, function(err, chan) {
+                that.addCacheChannel(req, function(err) {
+                  if (req.profiler) req.profiler.done('addCacheChannel');
                   callback(err, req);
                 });
             }
@@ -110,14 +657,23 @@ module.exports = function(){
      */
     me.getInfowindow = function(req, callback){
         var that = this;
+        var user = me.userByReq(req);
 
         Step(
             function(){
+                // TODO: if this step really needed ?
                 that.req2params(req, this);
             },
+            function getDatabase(err){
+                if (err) throw err;
+                cartoData.getUserDBName(user, this);
+            },
+            function getInfowindow(err, dbname){
+                if (err) throw err;
+                cartoData.getTableInfowindow(dbname, req.params.table, this);
+            },
             function(err, data){
-                if (err) callback(err, null);
-                else cartoData.getInfowindow(data, callback);
+                callback(err, data);
             }
         );
     };
@@ -129,14 +685,23 @@ module.exports = function(){
      */
     me.getMapMetadata = function(req, callback){
         var that = this;
+        var user = me.userByReq(req);
 
         Step(
             function(){
+                // TODO: if this step really needed ?
                 that.req2params(req, this);
             },
+            function getDatabase(err){
+                if (err) throw err;
+                cartoData.getUserDBName(user, this);
+            },
+            function getMapMetadata(err, dbname){
+                if (err) throw err;
+                cartoData.getTableMapMetadata(dbname, req.params.table, this);
+            },
             function(err, data){
-                if (err) callback(err, null);
-                else cartoData.getMapMetadata(data, callback);
+                callback(err, data);
             }
         );
     };
@@ -150,11 +715,17 @@ module.exports = function(){
         var that = this;
 
         Step(
-            function(){
+            function getParams(){
+                // this is mostly to compute req.params.dbname
                 that.req2params(req, this);
             },
-            function(err, data){
-                if (err) throw err;
+            function flushInternalCache(err){
+                // TODO: implement this, see
+                // http://github.com/Vizzuality/Windshaft-cartodb/issues/73
+                return true;
+            },
+            function flushVarnishCache(err){
+                if (err) { callback(err); return; }
                 if(Cache) {
                   Cache.invalidate_db(req.params.dbname, req.params.table);
                 }

lib/cartodb/signed_maps.js

@@ -0,0 +1,347 @@
+var crypto    = require('crypto');
+var Step      = require('step');
+var _         = require('underscore');
+
+
+// Class handling map signatures and user certificates
+//
+// See https://github.com/CartoDB/Windshaft-cartodb/wiki/Signed-maps
+//
+// @param redis_pool an instance of a "redis-mpool"
+//        See https://github.com/CartoDB/node-redis-mpool
+//        Needs version 0.x.x of the API.
+//
+function SignedMaps(redis_pool) {
+  this.redis_pool = redis_pool;
+
+  // Database containing signatures
+  // TODO: allow configuring ?
+  // NOTE: currently it is the same as
+  //       the one containing layergroups
+  this.db_signatures = 0;
+
+  //
+  // Map signatures in redis are reference to signature certificates
+  // We have the following datastores:
+  //
+  //  1. User certificates: set of per-user authorization certificates
+  //  2. Map signatures: set of per-map certificate references 
+  //  3. Certificate applications: set of per-certificate signed maps
+
+  // User certificates (HASH:crt_id->crt_val)
+  this.key_map_crt = "map_crt|<%= signer %>";
+
+  // Map signatures (SET:crt_id)
+  this.key_map_sig = "map_sig|<%= signer %>|<%= map_id %>";
+  
+  // Certificates applications (SET:map_id)
+  //
+  // Everytime a map is signed, the map identifier (layergroup_id)
+  // is added to this set. The purpose of this set is to drop
+  // all map signatures when a certificate is removed
+  //
+  this.key_crt_sig = "crt_sig|<%= signer %>|<%= crt_id %>";
+
+};
+
+var o = SignedMaps.prototype;
+
+//--------------- PRIVATE METHODS --------------------------------
+
+o._acquireRedis = function(callback) {
+  this.redis_pool.acquire(this.db_signatures, callback);
+};
+
+o._releaseRedis = function(client) {
+  this.redis_pool.release(this.db_signatures, client);
+};
+
+/**
+ * Internal function to communicate with redis
+ *
+ * @param redisFunc - the redis function to execute
+ * @param redisArgs - the arguments for the redis function in an array
+ * @param callback - function to pass results too.
+ */
+o._redisCmd = function(redisFunc, redisArgs, callback) {
+  var redisClient;
+  var that = this;
+  var db = that.db_signatures;
+
+  Step(
+    function getRedisClient() {
+      that.redis_pool.acquire(db, this);
+    },
+    function executeQuery(err, data) {
+      if ( err ) throw err;
+      redisClient = data;
+      redisArgs.push(this);
+      redisClient[redisFunc.toUpperCase()].apply(redisClient, redisArgs);
+    },
+    function releaseRedisClient(err, data) {
+      if ( ! _.isUndefined(redisClient) ) that.redis_pool.release(db, redisClient);
+      callback(err, data);
+    }
+  );
+};
+
+//--------------- PUBLIC API -------------------------------------
+
+// Check if the given certificate authorizes waiver of "auth"
+o.authorizedByCert = function(cert, auth) {
+  //console.log("Checking cert: "); console.dir(cert);
+  if ( cert.version !== "0.0.1" ) {
+    throw new Error("Unsupported certificate version " + cert.version);
+  }
+
+  if ( ! cert.auth ) {
+    throw new Error("No certificate authorization");
+  }
+
+  if ( ! cert.auth.method ) {
+    throw new Error("No certificate authorization method");
+  }
+
+  // Open authentication certificates are always authorized
+  if ( cert.auth.method === 'open' ) return true;
+
+  // Token based authentication requires valid token
+  if ( cert.auth.method === 'token' ) {
+    var found = cert.auth.valid_tokens.indexOf(auth);
+    //if ( found !== -1 ) {
+      //console.log("Token " + auth + " is found at position " + found + " in valid tokens " + cert.auth.valid_tokens);
+    //  return true;
+    //} else return false;
+    return cert.auth.valid_tokens.indexOf(auth) !== -1;
+  }
+
+  throw new Error("Unsupported authentication method: " + cert.auth.method);
+};
+
+// Check if shown credential are authorized to access a map
+// by the given signer.
+//
+// @param signer a signer name (cartodb username)
+// @param map_id a layergroup_id
+// @param auth an authentication token, or undefined if none
+//                    (can still be authorized by signature)
+//
+// @param callback function(Error, Boolean)
+//
+o.isAuthorized = function(signer, map_id, auth, callback) {
+  var that = this;
+  var authorized = false;
+  var certificate_id_list;
+  var missing_certificates = [];
+console.log("Check auth from signer '" + signer + "' on map '" + map_id + "' with auth '" + auth + "'");
+  Step(
+    function getMapSignatures() {
+      var map_sig_key = _.template(that.key_map_sig, {signer:signer, map_id:map_id});
+      that._redisCmd('SMEMBERS', [ map_sig_key ], this);
+    },
+    function getCertificates(err, crt_lst) {
+      if ( err ) throw err;
+      console.log("Map '" + map_id + "' is signed by " + crt_lst.length + " certificates of user '" + signer + "': " + crt_lst);
+      certificate_id_list = crt_lst;
+      if ( ! crt_lst.length ) {
+        // No certs, avoid calling redis with short args list.
+        // Next step expects a list of certificate values so
+        // we directly send the empty list.
+        return crt_lst;
+      }
+      var map_crt_key = _.template(that.key_map_crt, {signer:signer});
+      that._redisCmd('HMGET', [ map_crt_key ].concat(crt_lst), this);
+    },
+    function checkCertificates(err, certs) {
+      if ( err ) throw err;
+      for (var i=0; i<certs.length; ++i) {
+        var crt_id = certificate_id_list[i];
+        if ( _.isNull(certs[i]) ) {
+          missing_certificates.push(crt_id);
+          continue;
+        }
+        var cert;
+        try {
+          //console.log("cert " + crt_id + ": " + certs[i]);
+          cert = JSON.parse(certs[i]);
+          authorized = that.authorizedByCert(cert, auth);
+        } catch (err) {
+          console.log("Certificate " + certificate_id_list[i] + " by user '" + signer + "' is malformed: " + err);
+          continue;
+        }
+        if ( authorized ) {
+          console.log("Access to map '" + map_id + "' authorized by cert '"
+            + certificate_id_list[i] + "' of user '" + signer + "'");
+          //console.dir(cert);
+          break; // no need to further check certs
+        }
+      }
+      return null;
+    },
+    function finish(err) {
+      if ( missing_certificates.length ) {
+        console.log("WARNING: map '" + map_id + "' is signed by '" + signer
+          + "' with " + missing_certificates.length
+          + " missing certificates: "
+          + missing_certificates + " (TODO: give cleanup instructions)");
+      }
+      callback(err, authorized);
+    }
+  );
+};
+
+// Add an authorization certificate from a user.
+//
+// @param signer a signer name (cartodb username)
+// @param cert certificate object, see
+//             http://github.com/CartoDB/Windshaft-cartodb/wiki/Signed-maps
+//
+// @param callback function(err, crt_id) return certificate id
+//
+// TODO: allow for requesting error when certificate already exists ?
+//
+o.addCertificate = function(signer, cert, callback) {
+  var crt_val = JSON.stringify(cert);
+  var crt_id = crypto.createHash('md5').update(crt_val).digest('hex');
+
+  var usr_crt_key = _.template(this.key_map_crt, {signer:signer});
+  this._redisCmd('HSET', [ usr_crt_key, crt_id, crt_val ], function(err, created) {
+    // NOTE: created would be 0 if the field already existed, 1 otherwise
+    callback(err, crt_id);
+  });
+};
+
+// Remove an authorization certificate of a user, also removing
+// any signature made with the certificate.
+//
+// @param signer a signer name (cartodb username)
+// @param crt_id certificate identifier, as returned by addCertificate
+// @param callback function(err)
+//
+o.delCertificate = function(signer, crt_id, callback) {
+  var db = this.db_signatures;
+  var crt_sig_key = _.template(this.key_crt_sig, {signer:signer, crt_id:crt_id});
+  var signed_map_list;
+  var redis_client;
+  var that = this;
+  Step (
+    function getRedisClient() {
+      that._acquireRedis(this);
+    },
+    function removeCertificate(err, data) {
+      if ( err ) throw err;
+      redis_client = data;
+      // Remove the certificate (would be enough to stop authorizing uses)
+      var usr_crt_key = _.template(that.key_map_crt, {signer:signer});
+      redis_client.HDEL(usr_crt_key, crt_id, this);
+    },
+    function getMapSignatures(err, deleted) {
+      if ( err ) throw err;
+      if ( ! deleted ) {
+        // debugging (how can this be possible?)
+        console.log("WARNING: authorization certificate '" + crt_id
+          + "' by user '" + signer + "' did not exist on delete request");
+      }
+      // Get all signatures by this certificate
+      redis_client.SMEMBERS(crt_sig_key, this);
+    },
+    function delMapSignaturesReference(err, map_id_list) {
+      if ( err ) throw err;
+      signed_map_list = map_id_list;
+      console.log("Certificate '" + crt_id + "' from user '" + signer
+        + "' was used to sign " + signed_map_list.length + " maps");
+      redis_client.DEL(crt_sig_key, this);
+    },
+    function delMapSignatures(err) {
+      if ( err ) throw err;
+      var crt_sig_key = _.template(that.key_crt_sig, {signer:signer, crt_id:crt_id});
+      var tx = redis_client.MULTI();
+      for (var i=0; i<signed_map_list.length; ++i) {
+        var map_id = signed_map_list[i];
+        var map_sig_key = _.template(that.key_map_sig, {signer:signer, map_id:map_id});
+        //console.log("Queuing removal of '" + crt_id + "' from '" + map_sig_key + "'");
+        tx.SREM( map_sig_key, crt_id )
+      }
+      tx.EXEC(this);
+    },
+    function reportTransaction(err, rets) {
+      if ( err ) throw err;
+      for (var i=0; i<signed_map_list.length; ++i) {
+        var ret = rets[i];
+        if ( ! ret ) {
+          console.log("No signature with certificate '" + crt_id
+            + "' of user '" + signer + "' found in map '"
+            + signed_map_list[i] + "'");
+        } else {
+          console.log("Signature with certificate '" + crt_id
+            + "' of user '" + signer + "' removed from map '"
+            + signed_map_list[i] + "'");
+        }
+      }
+      return null;
+    },
+    function finish(err) {
+      if ( ! _.isUndefined(redis_client) ) {
+        that._releaseRedis(redis_client);
+      }
+      callback(err);
+    }
+  );
+};
+
+// Sign a map with a certificate reference
+//
+// @param signer a signer name (cartodb username)
+// @param map_id a layergroup_id
+// @param crt_id signature certificate identifier
+//
+// @param callback function(Error) 
+//
+o.signMap = function(signer, map_id, crt_id, callback) {
+  var that = this;
+  Step(
+    function addMapSignature() {
+      var map_sig_key = _.template(that.key_map_sig, {signer:signer, map_id:map_id});
+console.log("Adding " + crt_id + " to " + map_sig_key);
+      that._redisCmd('SADD', [ map_sig_key, crt_id ], this);
+    },
+    function addCertificateUsage(err) {
+      // Add the map to the set of maps signed by the given cert
+      if ( err ) throw err;
+      var crt_sig_key = _.template(that.key_crt_sig, {signer:signer, crt_id:crt_id});
+      that._redisCmd('SADD', [ crt_sig_key, map_id ], this);
+    },
+    function finish(err) {
+      callback(err);
+    }
+  );
+};
+
+// Sign a map with a full certificate
+//
+// @param signer a signer name (cartodb username)
+// @param map_id a layergroup_id
+// @param cert_id signature certificate identifier
+//
+// @param callback function(Error, String) return certificate id
+//
+o.addSignature = function(signer, map_id, cert, callback) {
+  var that = this;
+  var certificate_id;
+  Step(
+    function addCertificate() {
+      that.addCertificate(signer, cert, this);
+    },
+    function signMap(err, cert_id) {
+      if ( err ) throw err;
+      if ( ! cert_id ) throw new Error("addCertificate returned no certificate id");
+      certificate_id = cert_id;
+      that.signMap(signer, map_id, cert_id, this);
+    },
+    function finish(err) {
+      callback(err, certificate_id);
+    }
+  );
+};
+
+module.exports = SignedMaps;

lib/cartodb/template_maps.js

@@ -0,0 +1,586 @@
+var crypto    = require('crypto');
+var Step      = require('step');
+var _         = require('underscore');
+
+// Templates in this hash (keyed as <username>@<template_name>)
+// are being worked on.
+var user_template_locks = {};
+
+// Class handling map templates
+//
+// See http://github.com/CartoDB/Windshaft-cartodb/wiki/Template-maps
+//
+// @param redis_pool an instance of a "redis-mpool"
+//        See https://github.com/CartoDB/node-redis-mpool
+//        Needs version 0.x.x of the API.
+//
+// @param signed_maps an instance of a "signed_maps" class,
+//        See signed_maps.js
+// 
+function TemplateMaps(redis_pool, signed_maps) {
+  this.redis_pool = redis_pool;
+  this.signed_maps = signed_maps;
+
+  // Database containing templates
+  // TODO: allow configuring ?
+  // NOTE: currently it is the same as
+  //       the one containing layergroups
+  this.db_signatures = 0;
+
+  //
+  // Map templates are owned by a user that specifies access permissions
+  // for their instances.
+  // 
+  // We have the following datastores:
+  //
+  //  1. User teplates: set of per-user map templates
+  //     NOTE: each template would have an associated auth
+  //           reference, see signed_maps.js
+
+  // User templates (HASH:tpl_id->tpl_val)
+  this.key_usr_tpl = "map_tpl|<%= owner %>";
+
+  // User template locks (HASH:tpl_id->ctime)
+  this.key_usr_tpl_lck = "map_tpl|<%= owner %>|locks";
+
+};
+
+var o = TemplateMaps.prototype;
+
+//--------------- PRIVATE METHODS --------------------------------
+
+o._acquireRedis = function(callback) {
+  this.redis_pool.acquire(this.db_signatures, callback);
+};
+
+o._releaseRedis = function(client) {
+  this.redis_pool.release(this.db_signatures, client);
+};
+
+/**
+ * Internal function to communicate with redis
+ *
+ * @param redisFunc - the redis function to execute
+ * @param redisArgs - the arguments for the redis function in an array
+ * @param callback - function to pass results too.
+ */
+o._redisCmd = function(redisFunc, redisArgs, callback) {
+  var redisClient;
+  var that = this;
+  var db = that.db_signatures;
+
+  Step(
+    function getRedisClient() {
+      that.redis_pool.acquire(db, this);
+    },
+    function executeQuery(err, data) {
+      if ( err ) throw err;
+      redisClient = data;
+      redisArgs.push(this);
+      redisClient[redisFunc.toUpperCase()].apply(redisClient, redisArgs);
+    },
+    function releaseRedisClient(err, data) {
+      if ( ! _.isUndefined(redisClient) ) that.redis_pool.release(db, redisClient);
+      callback(err, data);
+    }
+  );
+};
+
+// @param callback function(err, obtained)
+o._obtainTemplateLock = function(owner, tpl_id, callback) {
+  var usr_tpl_lck_key = _.template(this.key_usr_tpl_lck, {owner:owner});
+  var that = this;
+  var gotLock = false;
+  Step (
+    function obtainLock() {
+      var ctime = Date.now();
+      that._redisCmd('HSETNX', [usr_tpl_lck_key, tpl_id, ctime], this);
+    },
+    function checkLock(err, locked) {
+      if ( err ) throw err;
+      if ( ! locked ) {
+        // Already locked
+        // TODO: unlock if expired ?
+        throw new Error("Template '" + tpl_id + "' of user '" + owner + "' is locked");
+      }
+      return gotLock = true;
+    },
+    function finish(err) {
+      callback(err, gotLock);
+    }
+  );
+};
+
+// @param callback function(err, deleted)
+o._releaseTemplateLock = function(owner, tpl_id, callback) {
+  var usr_tpl_lck_key = _.template(this.key_usr_tpl_lck, {owner:owner});
+  this._redisCmd('HDEL', [usr_tpl_lck_key, tpl_id], callback);
+};
+
+o._reValidIdentifier = /^[a-zA-Z][0-9a-zA-Z_]*$/;
+o._checkInvalidTemplate = function(template) {
+  if ( template.version != '0.0.1' ) {
+    return new Error("Unsupported template version " + template.version);
+  }
+  var tplname = template.name;
+  if ( ! tplname ) {
+    return new Error("Missing template name");
+  }
+  if ( ! tplname.match(this._reValidIdentifier) ) {
+    return new Error("Invalid characters in template name '" + tplname + "'");
+  }
+
+  var phold = template.placeholders;
+  for (var k in phold) {
+    if ( ! k.match(this._reValidIdentifier) ) {
+      return new Error("Invalid characters in placeholder name '" + k + "'");
+    }
+    if ( ! phold[k].hasOwnProperty('default') ) {
+      return new Error("Missing default for placeholder '" + k + "'");
+    }
+    if ( ! phold[k].hasOwnProperty('type') ) {
+      return new Error("Missing type for placeholder '" + k + "'");
+    }
+  };
+
+  // TODO: run more checks over template format ?
+};
+
+//--------------- PUBLIC API -------------------------------------
+
+// Extract a signature certificate from a template
+//
+// The certificate will be ready to be passed to
+// SignedMaps.addCertificate or SignedMaps.authorizedByCert
+//
+o.getTemplateCertificate = function(template) {
+  var cert = {
+    version: '0.0.1',
+    template_id: template.name,
+    auth: template.auth
+  };
+  return cert;
+};
+
+// Add a template
+//
+// NOTE: locks user+template_name or fails
+//
+// @param owner cartodb username of the template owner
+//
+// @param template layergroup template, see
+//        http://github.com/CartoDB/Windshaft-cartodb/wiki/Template-maps#template-format
+//
+// @param callback function(err, tpl_id) 
+//        Return template identifier (only valid for given user)
+//
+o.addTemplate = function(owner, template, callback) {
+  var invalidError = this._checkInvalidTemplate(template);
+  if ( invalidError ) {
+    callback(invalidError);
+    return;
+  }
+  var tplname = template.name;
+
+  // Procedure:
+  //
+  // 0. Obtain a lock for user+template_name, fail if impossible
+  // 1. Check no other template exists with the same name
+  // 2. Install certificate extracted from template, extending
+  //    it to contain a name to properly salt things out.
+  // 3. Modify the template object to reference certificate by id
+  // 4. Install template
+  // 5. Release lock
+  //
+  //
+
+  var usr_tpl_key = _.template(this.key_usr_tpl, {owner:owner});
+  var gotLock = false;
+  var that = this;
+  Step(
+    // try to obtain a lock
+    function obtainLock() {
+      that._obtainTemplateLock(owner, tplname, this);
+    },
+    function getExistingTemplate(err, locked) {
+      if ( err ) throw err;
+      if ( ! locked ) {
+        // Already locked
+        throw new Error("Template '" + tplname + "' of user '" + owner + "' is locked");
+      }
+      gotLock = true;
+      that._redisCmd('HEXISTS', [ usr_tpl_key, tplname ], this);
+    },
+    function installCertificate(err, exists) {
+      if ( err ) throw err;
+      if ( exists ) {
+        throw new Error("Template '" + tplname + "' of user '" + owner + "' already exists"); 
+      }
+      var cert = that.getTemplateCertificate(template);
+      that.signed_maps.addCertificate(owner, cert, this);
+    },
+    function installTemplate(err, crt_id) {
+      if ( err ) throw err;
+      template.auth_id = crt_id;
+      var tpl_val = JSON.stringify(template);
+      that._redisCmd('HSET', [ usr_tpl_key, tplname, tpl_val ], this);
+    },
+    function releaseLock(err, newfield) {
+      if ( ! err && ! newfield ) {
+        console.log("ERROR: addTemplate overridden existing template '"
+          + tplname + "' of '" + owner
+          + "' -- HSET returned " + overridden + ": someone added it without locking ?");
+        // TODO: how to recover this ?!
+      }
+
+      if ( ! gotLock ) {
+        if ( err ) throw err;
+        return null;
+      }
+
+      // release the lock
+      var next = this;
+      that._releaseTemplateLock(owner, tplname, function(e, d) {
+        if ( e ) {
+          console.log("Error removing lock on template '" + tplname
+            + "' of user '" + owner + "': " + e);
+        } else if ( ! d ) {
+          console.log("ERROR: lock on template '" + tplname
+            + "' of user '" + owner + "' externally removed during insert!");
+        }
+        next(err);
+      });
+    },
+    function finish(err) {
+      callback(err, tplname);
+    }
+  );
+};
+
+// Delete a template
+//
+// NOTE: locks user+template_name or fails
+//
+// Also deletes associated authentication certificate, which
+// in turn deletes all instance signatures
+//
+// @param owner cartodb username of the template owner
+//
+// @param tpl_id template identifier as returned
+//        by addTemplate or listTemplates
+//
+// @param callback function(err)
+//
+o.delTemplate = function(owner, tpl_id, callback) {
+  var usr_tpl_key = _.template(this.key_usr_tpl, {owner:owner});
+  var gotLock = false;
+  var that = this;
+  Step(
+    // try to obtain a lock
+    function obtainLock() {
+      that._obtainTemplateLock(owner, tpl_id, this);
+    },
+    function getExistingTemplate(err, locked) {
+      if ( err ) throw err;
+      if ( ! locked ) {
+        // Already locked
+        throw new Error("Template '" + tpl_id + "' of user '" + owner + "' is locked");
+      }
+      gotLock = true;
+      that._redisCmd('HGET', [ usr_tpl_key, tpl_id ], this);
+    },
+    function delCertificate(err, tplval) {
+      if ( err ) throw err;
+      if ( ! tplval ) {
+        throw new Error("Template '" + tpl_id + "' of user '" + owner + "' does not exist"); 
+      }
+      var tpl = JSON.parse(tplval);
+      if ( ! tpl.auth_id ) {
+        // not sure this is an error, in case we'll ever
+        // allow unsigned templates...
+        console.log("ERROR: installed template '" + tpl_id
+            + "' of user '" + owner + "' has no auth_id reference: "); console.dir(tpl);
+        return null;
+      }
+      var next = this;
+      that.signed_maps.delCertificate(owner, tpl.auth_id, function(err) {
+        if ( err ) {
+          var msg = "ERROR: could not delete certificate '"
+                  + tpl.auth_id + "' associated with template '"
+                  + tpl_id + "' of user '" + owner + "': " + err;
+          // I'm actually not sure we want this event to be fatal
+          // (avoiding a deletion of the template itself) 
+          next(new Error(msg));
+        } else {
+          next();
+        }
+      });
+    },
+    function delTemplate(err) {
+      if ( err ) throw err;
+      that._redisCmd('HDEL', [ usr_tpl_key, tpl_id ], this);
+    },
+    function releaseLock(err, deleted) {
+      if ( ! err && ! deleted ) {
+          console.log("ERROR: template '" + tpl_id
+            + "' of user '" + owner + "' externally removed during delete!");
+      }
+
+      if ( ! gotLock ) {
+        if ( err ) throw err;
+        return null;
+      }
+
+      // release the lock
+      var next = this;
+      that._releaseTemplateLock(owner, tpl_id, function(e, d) {
+        if ( e ) {
+          console.log("Error removing lock on template '" + tpl_id
+            + "' of user '" + owner + "': " + e);
+        } else if ( ! d ) {
+          console.log("ERROR: lock on template '" + tpl_id
+            + "' of user '" + owner + "' externally removed during delete!");
+        }
+        next(err);
+      });
+    },
+    function finish(err) {
+      callback(err);
+    }
+  );
+};
+
+// Update a template
+//
+// NOTE: locks user+template_name or fails
+//
+// Also deletes and re-creates associated authentication certificate,
+// which in turn deletes all instance signatures
+//
+// @param owner cartodb username of the template owner
+//
+// @param tpl_id template identifier as returned by addTemplate
+//
+// @param template layergroup template, see
+//        http://github.com/CartoDB/Windshaft-cartodb/wiki/Template-maps#template-format
+//
+// @param callback function(err)
+//        
+o.updTemplate = function(owner, tpl_id, template, callback) {
+
+  var invalidError = this._checkInvalidTemplate(template);
+  if ( invalidError ) {
+    callback(invalidError);
+    return;
+  }
+
+  var tplname = template.name;
+
+  if ( tpl_id != tplname ) {
+    callback(new Error("Cannot update name of a map template ('" + tpl_id + "' != '" + tplname + "')"));
+    return;
+  }
+
+  var usr_tpl_key = _.template(this.key_usr_tpl, {owner:owner});
+  var gotLock = false;
+  var that = this;
+  Step(
+    // try to obtain a lock
+    function obtainLock() {
+      that._obtainTemplateLock(owner, tpl_id, this);
+    },
+    function getExistingTemplate(err, locked) {
+      if ( err ) throw err;
+      if ( ! locked ) {
+        // Already locked
+        throw new Error("Template '" + tpl_id + "' of user '" + owner + "' is locked");
+      }
+      gotLock = true;
+      that._redisCmd('HGET', [ usr_tpl_key, tpl_id ], this);
+    },
+    function delOldCertificate(err, tplval) {
+      if ( err ) throw err;
+      if ( ! tplval ) {
+        throw new Error("Template '" + tpl_id + "' of user '"
+                         + owner +"' does not exist");
+      }
+      var tpl = JSON.parse(tplval);
+      if ( ! tpl.auth_id ) {
+        // not sure this is an error, in case we'll ever
+        // allow unsigned templates...
+        console.log("ERROR: installed template '" + tpl_id
+            + "' of user '" + owner + "' has no auth_id reference: "); console.dir(tpl);
+        return null;
+      }
+      var next = this;
+      that.signed_maps.delCertificate(owner, tpl.auth_id, function(err) {
+        if ( err ) {
+          var msg = "ERROR: could not delete certificate '"
+                  + tpl.auth_id + "' associated with template '"
+                  + tpl_id + "' of user '" + owner + "': " + err;
+          // I'm actually not sure we want this event to be fatal
+          // (avoiding a deletion of the template itself) 
+          next(new Error(msg));
+        } else {
+          next();
+        }
+      });
+    },
+    function installNewCertificate(err) {
+      if ( err ) throw err;
+      var cert = that.getTemplateCertificate(template);
+      that.signed_maps.addCertificate(owner, cert, this);
+    },
+    function updTemplate(err, crt_id) {
+      if ( err ) throw err;
+      template.auth_id = crt_id;
+      var tpl_val = JSON.stringify(template);
+      that._redisCmd('HSET', [ usr_tpl_key, tplname, tpl_val ], this);
+    },
+    function releaseLock(err, newfield) {
+      if ( ! err && newfield ) {
+          console.log("ERROR: template '" + tpl_id
+            + "' of user '" + owner + "' externally removed during update!");
+      }
+
+      if ( ! gotLock ) {
+        if ( err ) throw err;
+        return null;
+      }
+
+      // release the lock
+      var next = this;
+      that._releaseTemplateLock(owner, tpl_id, function(e, d) {
+        if ( e ) {
+          console.log("Error removing lock on template '" + tpl_id
+            + "' of user '" + owner + "': " + e);
+        } else if ( ! d ) {
+          console.log("ERROR: lock on template '" + tpl_id
+            + "' of user '" + owner + "' externally removed during update!");
+        }
+        next(err);
+      });
+    },
+    function finish(err) {
+      callback(err);
+    }
+  );
+};
+
+// List user templates
+//
+// @param owner cartodb username of the templates owner
+//
+// @param callback function(err, tpl_id_list)
+//        Returns a list of template identifiers
+// 
+o.listTemplates = function(owner, callback) {
+  var usr_tpl_key = _.template(this.key_usr_tpl, {owner:owner});
+  this._redisCmd('HKEYS', [ usr_tpl_key ], callback);
+};
+
+// Get a templates
+//
+// @param owner cartodb username of the template owner
+//
+// @param tpl_id template identifier as returned
+//        by addTemplate or listTemplates
+//
+// @param callback function(err, template)
+//        Return full template definition
+//
+o.getTemplate = function(owner, tpl_id, callback) {
+  var usr_tpl_key = _.template(this.key_usr_tpl, {owner:owner});
+  var that = this;
+  Step(
+    function getTemplate() {
+      that._redisCmd('HGET', [ usr_tpl_key, tpl_id ], this);
+    },
+    function parseTemplate(err, tpl_val) {
+      if ( err ) throw err;
+      var tpl = JSON.parse(tpl_val);
+      // Should we strip auth_id ?
+      return tpl;
+    },
+    function finish(err, tpl) {
+      callback(err, tpl);
+    }
+  );
+};
+
+// Perform placeholder substitutions on a template
+//
+// @param template a template object (will not be modified)
+//
+// @param params an object containing named subsitution parameters
+//        Only the ones found in the template's placeholders object
+//        will be used, with missing ones taking default values.
+//
+// @returns a layergroup configuration 
+//
+// @throws Error on malformed template or parameter
+//
+o._reNumber = /^([-+]?[\d\.]?\d+([eE][+-]?\d+)?)$/;
+o._reCSSColorName = /^[a-zA-Z]+$/;
+o._reCSSColorVal = /^#[0-9a-fA-F]{3,6}$/;
+o._replaceVars = function(str, params) {
+  //return _.template(str, params); // lazy way, possibly dangerous 
+  // Construct regular expressions for each param
+  if ( ! params._re ) {
+    params._re = {};
+    for (var k in params) {
+      params._re[k] = RegExp("<%= " + k + " %>", "g");
+    }
+  }
+  for (var k in params) str = str.replace(params._re[k], params[k]);
+  return str;
+};
+o.instance = function(template, params) {
+  var all_params = {};
+  var phold = template.placeholders;
+  for (var k in phold) {
+    var val = params.hasOwnProperty(k) ? params[k] : phold[k].default;
+    var type = phold[k].type;
+    // properly escape
+    if ( type === 'sql_literal' ) {
+      // duplicate any single-quote
+      val = val.replace(/'/g, "''");
+    }
+    else if ( type === 'sql_ident' ) {
+      // duplicate any double-quote
+      val = val.replace(/"/g, '""');
+    }
+    else if ( type === 'number' ) {
+      // check it's a number
+      if ( ! val.match(this._reNumber) ) {
+        throw new Error("Invalid number value for template parameter '"
+          + k + "': " + val);
+      }
+    }
+    else if ( type === 'css_color' ) {
+      // check it only contains letters or
+      // starts with # and only contains hexdigits
+      if ( ! val.match(this._reCSSColorName) && ! val.match(this._reCSSColorVal) ) {
+        throw new Error("Invalid css_color value for template parameter '"
+          + k + "': " + val);
+      }
+    }
+    else {
+      // NOTE: should be checked at template create/update time
+      throw new Error("Invalid placeholder type '" + type + "'");
+    }
+    all_params[k] = val;
+  }
+
+  // NOTE: we're deep-cloning the layergroup here
+  var layergroup = JSON.parse(JSON.stringify(template.layergroup));
+  for (var i=0; i<layergroup.layers.length; ++i) {
+    var lyropt = layergroup.layers[i].options;
+    lyropt.cartocss = this._replaceVars(lyropt.cartocss, all_params);
+    lyropt.sql = this._replaceVars(lyropt.sql, all_params);
+    // Anything else ?
+  }
+  return layergroup;
+};
+
+module.exports = TemplateMaps;

npm-shrinkwrap.json

@@ -1,261 +1,204 @@
 {
   "name": "windshaft-cartodb",
-  "version": "0.2.0-dev",
+  "version": "1.6.0",
   "dependencies": {
-    "cluster2": {
-      "version": "0.3.5",
-      "dependencies": {
-        "express": {
-          "version": "2.5.11",
-          "dependencies": {
-            "connect": {
-              "version": "1.9.2",
-              "dependencies": {
-                "formidable": {
-                  "version": "1.0.11"
-                }
-              }
-            },
-            "mime": {
-              "version": "1.2.4"
-            },
-            "qs": {
-              "version": "0.4.2"
-            },
-            "mkdirp": {
-              "version": "0.3.0"
-            }
-          }
-        },
-        "ejs": {
-          "version": "0.8.3"
-        },
-        "npm": {
-          "version": "1.1.62",
-          "dependencies": {
-            "semver": {
-              "version": "1.0.14"
-            },
-            "ini": {
-              "version": "1.0.4"
-            },
-            "slide": {
-              "version": "1.1.3"
-            },
-            "abbrev": {
-              "version": "1.0.3"
-            },
-            "graceful-fs": {
-              "version": "1.1.14"
-            },
-            "minimatch": {
-              "version": "0.2.6"
-            },
-            "nopt": {
-              "version": "2.0.0"
-            },
-            "rimraf": {
-              "version": "2.0.2"
-            },
-            "request": {
-              "version": "2.9.203",
-              "from": "git://github.com/isaacs/request"
-            },
-            "which": {
-              "version": "1.0.5"
-            },
-            "tar": {
-              "version": "0.1.13"
-            },
-            "fstream": {
-              "version": "0.1.19"
-            },
-            "block-stream": {
-              "version": "0.0.6"
-            },
-            "inherits": {
-              "version": "1.0.0",
-              "from": "git://github.com/isaacs/inherits"
-            },
-            "mkdirp": {
-              "version": "0.3.4"
-            },
-            "read": {
-              "version": "1.0.4",
-              "dependencies": {
-                "mute-stream": {
-                  "version": "0.0.3"
-                }
-              }
-            },
-            "lru-cache": {
-              "version": "2.0.4"
-            },
-            "node-gyp": {
-              "version": "0.6.11"
-            },
-            "fstream-npm": {
-              "version": "0.1.2",
-              "dependencies": {
-                "fstream-ignore": {
-                  "version": "0.0.5"
-                }
-              }
-            },
-            "uid-number": {
-              "version": "0.0.3"
-            },
-            "archy": {
-              "version": "0.0.2"
-            },
-            "chownr": {
-              "version": "0.0.1"
-            },
-            "npmlog": {
-              "version": "0.0.2"
-            },
-            "ansi": {
-              "version": "0.1.2"
-            },
-            "npm-registry-client": {
-              "version": "0.2.7"
-            },
-            "read-package-json": {
-              "version": "0.1.5"
-            },
-            "read-installed": {
-              "version": "0.0.2"
-            },
-            "glob": {
-              "version": "3.1.12"
-            },
-            "init-package-json": {
-              "version": "0.0.5",
-              "dependencies": {
-                "promzard": {
-                  "version": "0.2.0"
-                }
-              }
-            },
-            "osenv": {
-              "version": "0.0.3"
-            },
-            "lockfile": {
-              "version": "0.2.1"
-            },
-            "retry": {
-              "version": "0.6.0"
-            },
-            "couch-login": {
-              "version": "0.1.12"
-            },
-            "once": {
-              "version": "1.1.1"
-            },
-            "npmconf": {
-              "version": "0.0.16",
-              "dependencies": {
-                "config-chain": {
-                  "version": "1.1.2",
-                  "dependencies": {
-                    "proto-list": {
-                      "version": "1.2.2"
-                    }
-                  }
-                }
-              }
-            },
-            "opener": {
-              "version": "1.3.0"
-            }
-          }
-        }
-      }
-    },
     "node-varnish": {
       "version": "0.1.1"
     },
     "underscore": {
-      "version": "1.1.7"
+      "version": "1.3.3"
     },
-    "grainstore": {
-      "version": "0.6.4",
+    "windshaft": {
+      "version": "0.14.5",
       "dependencies": {
-        "carto": {
-          "version": "0.8.2-cdb-dev-3",
-          "from": "git://github.com/CartoDB/carto.git#cdb-0.8",
+        "grainstore": {
+          "version": "0.15.2",
           "dependencies": {
-            "underscore": {
-              "version": "1.3.3"
+            "carto": {
+              "version": "0.9.5-cdb2",
+              "from": "git://github.com/CartoDB/carto.git#0.9.5-cdb2",
+              "dependencies": {
+                "underscore": {
+                  "version": "1.4.4"
+                },
+                "xml2js": {
+                  "version": "0.2.8",
+                  "dependencies": {
+                    "sax": {
+                      "version": "0.5.5"
+                    }
+                  }
+                },
+                "optimist": {
+                  "version": "0.6.0",
+                  "dependencies": {
+                    "wordwrap": {
+                      "version": "0.0.2"
+                    },
+                    "minimist": {
+                      "version": "0.0.5"
+                    }
+                  }
+                }
+              }
             },
             "mapnik-reference": {
-              "version": "4.0.5"
+              "version": "5.0.7"
             },
-            "xml2js": {
-              "version": "0.1.14",
+            "millstone": {
+              "version": "0.6.8",
               "dependencies": {
-                "sax": {
+                "underscore": {
+                  "version": "1.5.2"
+                },
+                "request": {
+                  "version": "2.26.0",
+                  "dependencies": {
+                    "qs": {
+                      "version": "0.6.6"
+                    },
+                    "json-stringify-safe": {
+                      "version": "5.0.0"
+                    },
+                    "forever-agent": {
+                      "version": "0.5.0"
+                    },
+                    "tunnel-agent": {
+                      "version": "0.3.0"
+                    },
+                    "http-signature": {
+                      "version": "0.10.0",
+                      "dependencies": {
+                        "assert-plus": {
+                          "version": "0.1.2"
+                        },
+                        "asn1": {
+                          "version": "0.1.11"
+                        },
+                        "ctype": {
+                          "version": "0.5.2"
+                        }
+                      }
+                    },
+                    "hawk": {
+                      "version": "1.0.0",
+                      "dependencies": {
+                        "hoek": {
+                          "version": "0.9.1"
+                        },
+                        "boom": {
+                          "version": "0.4.2"
+                        },
+                        "cryptiles": {
+                          "version": "0.2.2"
+                        },
+                        "sntp": {
+                          "version": "0.2.4"
+                        }
+                      }
+                    },
+                    "aws-sign": {
+                      "version": "0.3.0"
+                    },
+                    "oauth-sign": {
+                      "version": "0.3.0"
+                    },
+                    "cookie-jar": {
+                      "version": "0.3.0"
+                    },
+                    "node-uuid": {
+                      "version": "1.4.1"
+                    },
+                    "form-data": {
+                      "version": "0.1.2",
+                      "dependencies": {
+                        "combined-stream": {
+                          "version": "0.0.4",
+                          "dependencies": {
+                            "delayed-stream": {
+                              "version": "0.0.5"
+                            }
+                          }
+                        },
+                        "async": {
+                          "version": "0.2.9"
+                        }
+                      }
+                    }
+                  }
+                },
+                "srs": {
+                  "version": "0.3.8"
+                },
+                "zipfile": {
                   "version": "0.4.2"
+                },
+                "sqlite3": {
+                  "version": "2.1.19",
+                  "dependencies": {
+                    "tar.gz": {
+                      "version": "0.1.1",
+                      "dependencies": {
+                        "fstream": {
+                          "version": "0.1.25",
+                          "dependencies": {
+                            "rimraf": {
+                              "version": "2.2.4"
+                            },
+                            "graceful-fs": {
+                              "version": "2.0.1"
+                            },
+                            "inherits": {
+                              "version": "2.0.1"
+                            }
+                          }
+                        },
+                        "tar": {
+                          "version": "0.1.18",
+                          "dependencies": {
+                            "inherits": {
+                              "version": "2.0.1"
+                            },
+                            "block-stream": {
+                              "version": "0.0.7"
+                            }
+                          }
+                        },
+                        "commander": {
+                          "version": "1.1.1",
+                          "dependencies": {
+                            "keypress": {
+                              "version": "0.1.0"
+                            }
+                          }
+                        }
+                      }
+                    }
+                  }
+                },
+                "mime": {
+                  "version": "1.2.11"
+                },
+                "mkdirp": {
+                  "version": "0.3.5"
+                },
+                "optimist": {
+                  "version": "0.6.0",
+                  "dependencies": {
+                    "wordwrap": {
+                      "version": "0.0.2"
+                    },
+                    "minimist": {
+                      "version": "0.0.5"
+                    }
+                  }
                 }
               }
             }
           }
         },
-        "millstone": {
-          "version": "0.5.10-cdb-01",
-          "from": "git://github.com/CartoDB/millstone.git#cdb-node04-devel",
-          "dependencies": {
-            "underscore": {
-              "version": "1.3.3"
-            },
-            "request": {
-              "version": "2.11.4",
-              "dependencies": {
-                "form-data": {
-                  "version": "0.0.3",
-                  "dependencies": {
-                    "combined-stream": {
-                      "version": "0.0.3",
-                      "dependencies": {
-                        "delayed-stream": {
-                          "version": "0.0.5"
-                        }
-                      }
-                    },
-                    "async": {
-                      "version": "0.1.9"
-                    }
-                  }
-                },
-                "mime": {
-                  "version": "1.2.7"
-                }
-              }
-            },
-            "srs": {
-              "version": "0.2.16"
-            },
-            "zipfile": {
-              "version": "0.3.2"
-            },
-            "sqlite3": {
-              "version": "2.1.5"
-            },
-            "mime": {
-              "version": "1.2.7"
-            },
-            "mkdirp": {
-              "version": "0.3.4"
-            }
-          }
-        }
-      }
-    },
-    "windshaft": {
-      "version": "0.5.8",
-      "dependencies": {
-        "underscore": {
-          "version": "1.3.3"
+        "generic-pool": {
+          "version": "2.0.4"
         },
         "express": {
           "version": "2.5.11",
@@ -264,7 +207,7 @@
               "version": "1.9.2",
               "dependencies": {
                 "formidable": {
-                  "version": "1.0.11"
+                  "version": "1.0.14"
                 }
               }
             },
@@ -280,10 +223,10 @@
           }
         },
         "tilelive": {
-          "version": "4.3.1",
+          "version": "4.4.3",
           "dependencies": {
             "optimist": {
-              "version": "0.3.4",
+              "version": "0.3.7",
               "dependencies": {
                 "wordwrap": {
                   "version": "0.0.2"
@@ -296,59 +239,122 @@
           }
         },
         "tilelive-mapnik": {
-          "version": "0.3.3-dev",
-          "from": "git://github.com/Vizzuality/tilelive-mapnik.git#7df70554",
+          "version": "0.6.4",
           "dependencies": {
-            "mapnik": {
-              "version": "0.7.14"
-            },
             "eio": {
-              "version": "0.1.0"
+              "version": "0.2.2"
+            },
+            "mime": {
+              "version": "1.2.11"
             },
             "sphericalmercator": {
               "version": "1.0.2"
             }
           }
+        },
+        "lru-cache": {
+          "version": "2.3.1"
         }
       }
     },
     "step": {
       "version": "0.0.5"
     },
-    "generic-pool": {
-      "version": "1.0.12"
-    },
-    "redis": {
-      "version": "0.7.2"
-    },
-    "hiredis": {
-      "version": "0.1.14"
-    },
     "request": {
       "version": "2.9.202"
     },
+    "cartodb-redis": {
+      "version": "0.3.0"
+    },
+    "redis-mpool": {
+      "version": "0.0.2",
+      "dependencies": {
+        "generic-pool": {
+          "version": "2.0.4"
+        }
+      }
+    },
+    "mapnik": {
+      "version": "0.7.25"
+    },
+    "lzma": {
+      "version": "1.2.3"
+    },
+    "strftime": {
+      "version": "0.6.2"
+    },
+    "semver": {
+      "version": "1.1.4"
+    },
+    "redis": {
+      "version": "0.8.6"
+    },
+    "redis-mpool": {
+      "version": "0.0.2",
+      "dependencies": {
+        "generic-pool": {
+          "version": "2.0.4"
+        }
+      }
+    },
+    "hiredis": {
+      "version": "0.1.15",
+      "dependencies": {
+        "bindings": {
+          "version": "1.1.0"
+        }
+      }
+    },
     "mocha": {
-      "version": "1.2.1",
+      "version": "1.14.0",
       "dependencies": {
         "commander": {
-          "version": "0.6.1"
+          "version": "2.0.0"
         },
         "growl": {
-          "version": "1.5.1"
+          "version": "1.7.0"
         },
         "jade": {
           "version": "0.26.3",
           "dependencies": {
+            "commander": {
+              "version": "0.6.1"
+            },
             "mkdirp": {
               "version": "0.3.0"
             }
           }
         },
         "diff": {
-          "version": "1.0.2"
+          "version": "1.0.7"
         },
         "debug": {
-          "version": "0.7.0"
+          "version": "0.7.4"
+        },
+        "mkdirp": {
+          "version": "0.3.5"
+        },
+        "glob": {
+          "version": "3.2.3",
+          "dependencies": {
+            "minimatch": {
+              "version": "0.2.12",
+              "dependencies": {
+                "lru-cache": {
+                  "version": "2.5.0"
+                },
+                "sigmund": {
+                  "version": "1.0.0"
+                }
+              }
+            },
+            "graceful-fs": {
+              "version": "2.0.1"
+            },
+            "inherits": {
+              "version": "2.0.1"
+            }
+          }
         }
       }
     }

package.json

@@ -1,36 +1,42 @@
 {
     "private": true,
     "name": "windshaft-cartodb",
-    "version": "1.0.1",
+    "version": "1.6.0",
     "description": "A map tile server for CartoDB",
-    "url": "https://github.com/Vizzuality/Windshaft-cartodb",
+    "keywords": [
+      "cartodb"
+    ],
+    "url": "https://github.com/CartoDB/Windshaft-cartodb",
     "licenses": [{
         "type": "BSD",
-        "url": "https://github.com/Vizzuality/Windshaft-cartodb/blob/master/LICENCE"
+        "url": "https://github.com/CartoDB/Windshaft-cartodb/blob/master/LICENCE"
     }],
-    "repositories": [{
+    "repository": {
         "type": "git",
-        "url": "git://github.com/Vizzuality/Windshaft-cartodb.git"
-    }],
-    "author": {
-      "name": "Simon Tokumine, Javi Santana, Vizzuality",
-      "url": "http://vizzuality.com",
-      "email": "simon@vizzuality.com"
+        "url": "git://github.com/CartoDB/Windshaft-cartodb.git"
     },
+    "author": "Vizzuality <contact@vizzuality.com> (http://vizzuality.com)",
+    "contributors": [
+      "Simon Tokumine <simon@vizzuality.com>",
+      "Javi Santana <jsantana@vizzuality.com>",
+      "Sandro Santilli <strk@vizzuality.com>"
+    ],
     "dependencies": {
-        "cluster2": "git://github.com/CartoDB/cluster2.git#28cde11",
         "node-varnish": "0.1.1",
-        "underscore"  : "1.1.x",
-        "grainstore"  : "~0.6.2",
-        "windshaft"  : "~0.5.8",
+        "underscore"  : "~1.3.3",
+        "windshaft"  : "~0.14.5",
         "step": "0.0.x",
-        "generic-pool": "1.0.x",
-        "redis": "0.7.2",
-        "hiredis": "~0.1.14",
-        "request": "2.9.202"
+        "request": "2.9.202",
+        "cartodb-redis": "~0.3.0",
+        "redis-mpool": "~0.0.2",
+        "mapnik": "~0.7.22",
+        "lzma": "~1.2.3"
     },
     "devDependencies": {
-        "mocha": "1.2.1"
+        "mocha": "1.14.0",
+        "redis": "~0.8.3",
+        "strftime": "~0.6.0",
+        "semver": "~1.1.0"
     },
     "scripts": {
         "test": "make check"

run_tests.sh

@@ -1,11 +1,33 @@
 #!/bin/sh
 
-# Must match config.redis_pool.port in test/support/config.js
-REDIS_PORT=6333
+OPT_CREATE_REDIS=yes # create the redis test environment
+OPT_CREATE_PGSQL=yes # create the PostgreSQL test environment
+OPT_DROP_REDIS=yes   # drop the redis test environment
+OPT_DROP_PGSQL=yes   # drop the PostgreSQL test environment
+
+cd $(dirname $0)
+BASEDIR=$(pwd)
+cd -
+
+REDIS_PORT=`node -e "console.log(require('${BASEDIR}/config/environments/test.js').redis.port)"`
+export REDIS_PORT
 
 cleanup() {
-	echo "Cleaning up"
-	kill ${PID_REDIS}
+  if test x"$OPT_DROP_REDIS" = xyes; then
+    if test x"$PID_REDIS" = x; then
+      PID_REDIS=$(cat ${BASEDIR}/redis.pid)
+      if test x"$PID_REDIS" = x; then
+        echo "Could not find a test redis pid to kill it"
+        return;
+      fi
+    fi
+    echo "Killing test redis pid ${PID_REDIS}"
+    kill ${PID_REDIS}
+  fi
+  if test x"$OPT_DROP_PGSQL" = xyes; then
+    # TODO: drop postgresql ?
+    echo "Dropping PostgreSQL test database isn't implemented yet"
+  fi
 }
 
 cleanup_and_exit() {
@@ -22,21 +44,76 @@ die() {
 
 trap 'cleanup_and_exit' 1 2 3 5 9 13
 
-echo "Starting redis on port ${REDIS_PORT}"
-echo "port ${REDIS_PORT}" | redis-server - > test.log &
-PID_REDIS=$!
+while [ -n "$1" ]; do
+        # This is kept for backward compatibility
+        if test "$1" = "--nodrop"; then
+                OPT_DROP_REDIS=no
+                OPT_DROP_PGSQL=no
+                shift
+                continue
+        elif test "$1" = "--nodrop-pg"; then
+                OPT_DROP_PGSQL=no
+                shift
+                continue
+        elif test "$1" = "--nodrop-redis"; then
+                OPT_DROP_REDIS=no
+                shift
+                continue
+        elif test "$1" = "--nocreate-pg"; then
+                OPT_CREATE_PGSQL=no
+                shift
+                continue
+        elif test "$1" = "--nocreate-redis"; then
+                OPT_CREATE_REDIS=no
+                shift
+                continue
+        # This is kept for backward compatibility
+        elif test "$1" = "--nocreate"; then
+                OPT_CREATE_REDIS=no
+                OPT_CREATE_PGSQL=no
+                shift
+                continue
+        else
+                break
+        fi
+done
 
-echo "Preparing the database"
-cd test/support; sh prepare_db.sh >> test.log || die "database preparation failure (see test.log)"; cd -;
+if [ -z "$1" ]; then
+        echo "Usage: $0 [<options>] <test> [<test>]" >&2
+        echo "Options:" >&2
+        echo " --nocreate   do not create the test environment on start" >&2
+        echo " --nodrop     do not drop the test environment on exit" >&2
+        exit 1
+fi
+
+TESTS=$@
+
+if test x"$OPT_CREATE_REDIS" = xyes; then
+  echo "Starting redis on port ${REDIS_PORT}"
+  echo "port ${REDIS_PORT}" | redis-server - > ${BASEDIR}/test.log &
+  PID_REDIS=$!
+  echo ${PID_REDIS} > ${BASEDIR}/redis.pid
+fi
+
+PREPARE_DB_OPTS=
+if test x"$OPT_CREATE_PGSQL" != xyes; then
+  PREPARE_DB_OPTS="$PREPARE_DB_OPTS --skip-pg"
+fi
+if test x"$OPT_CREATE_REDIS" != xyes; then
+  PREPARE_DB_OPTS="$PREPARE_DB_OPTS --skip-redis"
+fi
+
+echo "Preparing the environment"
+cd ${BASEDIR}/test/support
+sh prepare_db.sh ${PREPARE_DB_OPTS} || die "database preparation failure"
+cd -
 
 PATH=node_modules/.bin/:$PATH
 
 echo "Running tests"
-mocha -u tdd \
-  test/unit/cartodb/redis_pool.test.js \
-  test/unit/cartodb/req2params.test.js \
-  test/acceptance/cache_validator.js \
-  test/acceptance/server.js
-
+mocha -t 10000 -u tdd ${MOCHA_OPTS} ${TESTS}
+ret=$?
 
 cleanup
+
+exit $ret

test/acceptance/multilayer.js

@@ -0,0 +1,885 @@
+var assert      = require('../support/assert');
+var tests       = module.exports = {};
+var _           = require('underscore');
+var redis       = require('redis');
+var querystring = require('querystring');
+var semver      = require('semver');
+var mapnik      = require('mapnik');
+var Step        = require('step');
+var strftime    = require('strftime');
+var SQLAPIEmu   = require(__dirname + '/../support/SQLAPIEmu.js');
+var redis_stats_db = 5;
+
+require(__dirname + '/../support/test_helper');
+
+var windshaft_fixtures = __dirname + '/../../node_modules/windshaft/test/fixtures';
+
+var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft');
+var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
+var server = new CartodbWindshaft(serverOptions);
+server.setMaxListeners(0);
+
+suite('multilayer', function() {
+
+    var redis_client = redis.createClient(global.environment.redis.port);
+    var sqlapi_server;
+    var expected_last_updated_epoch = 1234567890123; // this is hard-coded into SQLAPIEmu
+    var expected_last_updated = new Date(expected_last_updated_epoch).toISOString();
+
+    suiteSetup(function(done){
+      sqlapi_server = new SQLAPIEmu(global.environment.sqlapi.port, done);
+    });
+
+    test("layergroup with 2 layers, each with its style", function(done) {
+
+      var layergroup =  {
+        version: '1.0.0',
+        layers: [
+           { options: {
+               sql: 'select cartodb_id, ST_Translate(the_geom_webmercator, 5e6, 0) as the_geom_webmercator from test_table limit 2',
+               cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', 
+               cartocss_version: '2.0.1',
+               interactivity: 'cartodb_id'
+             } },
+           { options: {
+               sql: 'select cartodb_id, ST_Translate(the_geom_webmercator, -5e6, 0) as the_geom_webmercator from test_table limit 2 offset 2',
+               cartocss: '#layer { marker-fill:blue; marker-allow-overlap:true; }', 
+               cartocss_version: '2.0.2',
+               interactivity: 'cartodb_id'
+             } }
+        ]
+      };
+
+      var expected_token = "e34dd7e235138a062f8ba7ad051aa3a7";
+      Step(
+        function do_post()
+        {
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup',
+              method: 'POST',
+              headers: {host: 'localhost', 'Content-Type': 'application/json' },
+              data: JSON.stringify(layergroup)
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 200, res.body);
+              var parsedBody = JSON.parse(res.body);
+              var expectedBody = { layergroupid: expected_token };
+              // check last modified
+              var qTables = JSON.stringify({
+                'q': 'SELECT CDB_QueryTables($windshaft$'
+                    + layergroup.layers[0].options.sql + ';'
+                    + layergroup.layers[1].options.sql 
+                    + '$windshaft$)'
+              });
+              assert.equal(parsedBody.last_updated, expected_last_updated);
+              if ( expected_token ) {
+                assert.equal(parsedBody.layergroupid, expected_token + ':' + expected_last_updated_epoch);
+              }
+              else expected_token = parsedBody.layergroupid;
+              next(null, res);
+          });
+        },
+        function do_get_tile(err)
+        {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup/' + expected_token + ':cb0/0/0/0.png',
+              method: 'GET',
+              headers: {host: 'localhost' },
+              encoding: 'binary'
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 200, res.body);
+              assert.equal(res.headers['content-type'], "image/png");
+
+              // Check Cache-Control
+              var cc = res.headers['cache-control'];
+              assert.equal(cc, 'public,max-age=31536000');  // 1 year
+
+              // Check X-Cache-Channel
+              cc = res.headers['x-cache-channel'];
+              assert.ok(cc); 
+              var dbname = 'test_cartodb_user_1_db'
+              assert.equal(cc.substring(0, dbname.length), dbname);
+              var jsonquery = cc.substring(dbname.length+1);
+              var sentquery = JSON.parse(jsonquery);
+              assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$'
+                + layergroup.layers[0].options.sql + ';'
+                + layergroup.layers[1].options.sql 
+                + '$windshaft$)');
+
+              assert.imageEqualsFile(res.body, 'test/fixtures/test_table_0_0_0_multilayer1.png', 2,
+                function(err, similarity) {
+                  next(err);
+              });
+          });
+        },
+        function do_get_grid_layer0(err)
+        {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup/' + expected_token
+                 + '/0/0/0/0.grid.json',
+              headers: {host: 'localhost' },
+              method: 'GET'
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 200, res.body);
+              assert.equal(res.headers['content-type'], "text/javascript; charset=utf-8; charset=utf-8");
+              assert.utfgridEqualsFile(res.body, 'test/fixtures/test_table_0_0_0_multilayer1.layer0.grid.json', 2,
+                function(err, similarity) {
+                  next(err);
+              });
+          });
+        },
+        function do_get_grid_layer1(err)
+        {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup/' + expected_token
+                 + '/1/0/0/0.grid.json',
+              headers: {host: 'localhost' },
+              method: 'GET'
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 200, res.body);
+              assert.equal(res.headers['content-type'], "text/javascript; charset=utf-8; charset=utf-8");
+              assert.utfgridEqualsFile(res.body, 'test/fixtures/test_table_0_0_0_multilayer1.layer1.grid.json', 2,
+                function(err, similarity) {
+                  next(err);
+              });
+          });
+        },
+        function finish(err) {
+          var errors = [];
+          if ( err ) {
+            errors.push(err.message);
+            console.log("Error: " + err);
+          }
+          redis_client.keys("map_style|test_cartodb_user_1_db|~" + expected_token, function(err, matches) {
+              if ( err ) errors.push(err.message);
+              assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches);
+              redis_client.del(matches, function(err) {
+                if ( err ) errors.push(err.message);
+                if ( errors.length ) done(new Error(errors));
+                else done(null);
+              });
+          });
+        }
+      );
+    });
+
+
+    test("layergroup can hold substitution tokens", function(done) {
+
+      var layergroup =  {
+        version: '1.0.0',
+        layers: [
+           { options: {
+               sql: 'select 1 as cartodb_id, '
+                  + 'ST_Buffer(!bbox!, -32*greatest(!pixel_width!,!pixel_height!)) as the_geom_webmercator',
+               cartocss: '#layer { polygon-fill:red; }', 
+               cartocss_version: '2.0.1',
+               interactivity: 'cartodb_id'
+             } }
+        ]
+      };
+
+      var expected_token  = "6d8e4ad5458e2d25cf0eef38e38717a6";
+      Step(
+        function do_post()
+        {
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup',
+              method: 'POST',
+              headers: {host: 'localhost', 'Content-Type': 'application/json' },
+              data: JSON.stringify(layergroup)
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 200, res.body);
+              var parsedBody = JSON.parse(res.body);
+              var expectedBody = { layergroupid: expected_token };
+              // check last modified
+              var qTables = JSON.stringify({
+                'q': 'SELECT CDB_QueryTables($windshaft$'
+                    + layergroup.layers[0].options.sql
+                    + '$windshaft$)'
+              });
+              assert.equal(parsedBody.last_updated, expected_last_updated);
+              if ( expected_token ) {
+                assert.equal(parsedBody.layergroupid, expected_token + ':' + expected_last_updated_epoch);
+              }
+              else expected_token = parsedBody.layergroupid;
+              next(null, res);
+          });
+        },
+        function do_get_tile1(err)
+        {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup/' + expected_token + ':cb10/1/0/0.png',
+              method: 'GET',
+              headers: {host: 'localhost' },
+              encoding: 'binary'
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 200, res.body);
+              assert.equal(res.headers['content-type'], "image/png");
+
+              // Check X-Cache-Channel
+              var cc = res.headers['x-cache-channel'];
+              assert.ok(cc); 
+              var dbname = 'test_cartodb_user_1_db'
+              assert.equal(cc.substring(0, dbname.length), dbname);
+              var jsonquery = cc.substring(dbname.length+1);
+              var sentquery = JSON.parse(jsonquery);
+              assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$'
+                + layergroup.layers[0].options.sql
+                    .replace(RegExp('!bbox!', 'g'), 'ST_MakeEnvelope(0,0,0,0)')
+                    .replace(RegExp('!pixel_width!', 'g'), '1')
+                    .replace(RegExp('!pixel_height!', 'g'), '1')
+                + '$windshaft$)');
+
+              assert.imageEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.png', 2,
+                function(err, similarity) {
+                  next(err);
+              });
+          });
+        },
+        function do_get_tile4(err)
+        {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup/' + expected_token + ':cb11/4/0/0.png',
+              method: 'GET',
+              headers: {host: 'localhost' },
+              encoding: 'binary'
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 200, res.body);
+              assert.equal(res.headers['content-type'], "image/png");
+
+              // Check X-Cache-Channel
+              var cc = res.headers['x-cache-channel'];
+              assert.ok(cc); 
+              var dbname = 'test_cartodb_user_1_db'
+              assert.equal(cc.substring(0, dbname.length), dbname);
+              var jsonquery = cc.substring(dbname.length+1);
+              var sentquery = JSON.parse(jsonquery);
+              assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$'
+                + layergroup.layers[0].options.sql
+                    .replace('!bbox!', 'ST_MakeEnvelope(0,0,0,0)')
+                    .replace('!pixel_width!', '1')
+                    .replace('!pixel_height!', '1')
+                + '$windshaft$)');
+
+              assert.imageEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.png', 2,
+                function(err, similarity) {
+                  next(err);
+              });
+          });
+        },
+        function do_get_grid1(err)
+        {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup/' + expected_token
+                 + '/0/1/0/0.grid.json',
+              headers: {host: 'localhost' },
+              method: 'GET'
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 200, res.body);
+              assert.equal(res.headers['content-type'], "text/javascript; charset=utf-8; charset=utf-8");
+              assert.utfgridEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.grid.json', 2,
+                function(err, similarity) {
+                  next(err);
+              });
+          });
+        },
+        function do_get_grid4(err)
+        {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup/' + expected_token
+                 + '/0/4/0/0.grid.json',
+              headers: {host: 'localhost' },
+              method: 'GET'
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 200, res.body);
+              assert.equal(res.headers['content-type'], "text/javascript; charset=utf-8; charset=utf-8");
+              assert.utfgridEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.grid.json', 2,
+                function(err, similarity) {
+                  next(err);
+              });
+          });
+        },
+        function finish(err) {
+          var errors = [];
+          if ( err ) {
+            errors.push(err.message);
+            console.log("Error: " + err);
+          }
+          redis_client.keys("map_style|test_cartodb_user_1_db|~" + expected_token, function(err, matches) {
+              if ( err ) errors.push(err.message);
+              assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches);
+              redis_client.del(matches, function(err) {
+                if ( err ) errors.push(err.message);
+                if ( errors.length ) done(new Error(errors));
+                else done(null);
+              });
+          });
+        }
+      );
+    });
+
+    test("layergroup creation raises mapviews counter", function(done) {
+      var layergroup =  {
+        stat_tag: 'random_tag',
+        version: '1.0.0',
+        layers: [
+           { options: {
+               sql: 'select 1 as cartodb_id, !pixel_height! as h, '
+                  + 'ST_Buffer(!bbox!, -32*greatest(!pixel_width!,!pixel_height!)) as the_geom_webmercator',
+               cartocss: '#layer { polygon-fill:red; }', 
+               cartocss_version: '2.0.1' 
+             } }
+        ]
+      };
+      var statskey = "user:localhost:mapviews";
+      var redis_stats_client = redis.createClient(global.environment.redis.port);
+      var expected_token; // will be set on first post and checked on second
+      var now = strftime("%Y%m%d", new Date());
+      var errors = [];
+      Step(
+        function clean_stats()
+        {
+          var next = this;
+          redis_stats_client.select(redis_stats_db, function(err) {
+            if ( err ) next(err);
+            else redis_stats_client.del(statskey+':global', next);
+          });
+        },
+        function do_post_1(err)
+        {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup',
+              method: 'POST',
+              headers: {host: 'localhost', 'Content-Type': 'application/json' },
+              data: JSON.stringify(layergroup)
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 200, res.body);
+              expected_token = JSON.parse(res.body).layergroupid;
+              redis_stats_client.zscore(statskey + ":global", now, next);
+          });
+        },
+        function check_global_stats_1(err, val) {
+          if ( err ) throw err;
+          assert.equal(val, 1, "Expected score of " + now + " in "
+              +  statskey + ":global to be 1, got " + val);
+          redis_stats_client.zscore(statskey+':stat_tag:random_tag', now, this);
+        },
+        function check_tag_stats_1_do_post_2(err, val) {
+          if ( err ) throw err;
+          assert.equal(val, 1, "Expected score of " + now + " in "
+              +  statskey + ":stat_tag:" + layergroup.stat_tag + " to be 1, got " + val);
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup',
+              method: 'POST',
+              headers: {host: 'localhost', 'Content-Type': 'application/json' },
+              data: JSON.stringify(layergroup)
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 200, res.body);
+              assert.equal(JSON.parse(res.body).layergroupid, expected_token);
+              redis_stats_client.zscore(statskey+':global', now, next);
+          });
+        },
+        function check_global_stats_2(err, val)
+        {
+          if ( err ) throw err;
+          assert.equal(val, 2, "Expected score of " + now + " in "
+              +  statskey + ":global to be 2, got " + val);
+          redis_stats_client.zscore(statskey+':stat_tag:' + layergroup.stat_tag, now, this);
+        },
+        function check_tag_stats_2(err, val)
+        {
+          if ( err ) throw err;
+          assert.equal(val, 2, "Expected score of " + now + " in "
+              +  statskey + ":stat_tag:" + layergroup.stat_tag + " to be 2, got " + val);
+          return 1;
+        },
+        function cleanup_map_style(err) {
+          if ( err ) errors.push('' + err);
+          var next = this;
+          // trip epoch
+          expected_token = expected_token.split(':')[0];
+          redis_client.keys("map_style|test_cartodb_user_1_db|~" + expected_token, function(err, matches) {
+              redis_client.del(matches, next);
+          });
+        },
+        function cleanup_stats(err) {
+          if ( err ) errors.push('' + err);
+          redis_client.del([statskey+':global', statskey+':stat_tag:'+layergroup.stat_tag], this);
+        },
+        function finish(err) {
+          if ( err ) errors.push('' + err);
+          if ( errors.length ) done(new Error(errors.join(',')));
+          else done(null);
+        }
+      );
+    });
+
+    test("layergroup creation fails if CartoCSS is bogus", function(done) {
+      var layergroup =  {
+        stat_tag: 'random_tag',
+        version: '1.0.0',
+        layers: [
+           { options: {
+               sql: 'select 1 as cartodb_id, !pixel_height! as h'
+                  + 'ST_Buffer(!bbox!, -32*greatest(!pixel_width!,!pixel_height!)) as the_geom_webmercator',
+               cartocss: '#layer { polygon-fit:red; }', 
+               cartocss_version: '2.0.1' 
+             } }
+        ]
+      };
+      assert.response(server, {
+          url: '/tiles/layergroup',
+          method: 'POST',
+          headers: {host: 'localhost', 'Content-Type': 'application/json' },
+          data: JSON.stringify(layergroup)
+      }, {}, function(res) {
+          assert.equal(res.statusCode, 400, res.body);
+          var parsed = JSON.parse(res.body);
+          assert.ok(parsed.errors[0].match(/^style0/));
+          assert.ok(parsed.errors[0].match(/Unrecognized rule: polygon-fit/));
+          done();
+      });
+    });
+
+    test("layergroup with 2 private-table layers", function(done) {
+
+      var layergroup =  {
+        version: '1.0.0',
+        layers: [
+           { options: {
+               sql: 'select * from test_table_private_1 where cartodb_id=1',
+               cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', 
+               cartocss_version: '2.1.0',
+               interactivity: 'cartodb_id'
+             } },
+           { options: {
+               sql: 'select * from test_table_private_1 where cartodb_id=2',
+               cartocss: '#layer { marker-fill:blue; marker-allow-overlap:true; }', 
+               cartocss_version: '2.1.0',
+               interactivity: 'cartodb_id'
+             } }
+        ]
+      };
+
+      var expected_token = "b4ed64d93a411a59f330ab3d798e4009";
+      Step(
+        function do_post()
+        {
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup?map_key=1234',
+              method: 'POST',
+              headers: {host: 'localhost', 'Content-Type': 'application/json' },
+              data: JSON.stringify(layergroup)
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 200, res.body);
+              var parsedBody = JSON.parse(res.body);
+              var expectedBody = { layergroupid: expected_token };
+              // check last modified
+              var qTables = JSON.stringify({
+                'q': 'SELECT CDB_QueryTables($windshaft$'
+                    + layergroup.layers[0].options.sql + ';'
+                    + layergroup.layers[1].options.sql 
+                    + '$windshaft$)'
+              });
+              assert.equal(parsedBody.last_updated, expected_last_updated);
+              if ( expected_token ) {
+                assert.equal(parsedBody.layergroupid, expected_token + ':' + expected_last_updated_epoch);
+              }
+              else expected_token = parsedBody.layergroupid;
+              next(null, res);
+          });
+        },
+        function do_get_tile(err)
+        {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup/' + expected_token + ':cb0/0/0/0.png?map_key=1234',
+              method: 'GET',
+              headers: {host: 'localhost' },
+              encoding: 'binary'
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 200, res.body);
+              assert.equal(res.headers['content-type'], "image/png");
+
+              // Check X-Cache-Channel
+              var cc = res.headers['x-cache-channel'];
+              assert.ok(cc); 
+              var dbname = 'test_cartodb_user_1_db'
+              assert.equal(cc.substring(0, dbname.length), dbname);
+              next(err);
+          });
+        },
+        function do_get_grid_layer0(err)
+        {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup/' + expected_token
+                 + '/0/0/0/0.grid.json?map_key=1234',
+              headers: {host: 'localhost' },
+              method: 'GET'
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 200, res.body);
+              next(err);
+          });
+        },
+        function do_get_grid_layer1(err)
+        {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup/' + expected_token
+                 + '/1/0/0/0.grid.json?map_key=1234',
+              headers: {host: 'localhost' },
+              method: 'GET'
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 200, res.body);
+              assert.equal(res.headers['content-type'], "text/javascript; charset=utf-8; charset=utf-8");
+              next(err);
+          });
+        },
+        function do_get_tile_unauth(err)
+        {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup/' + expected_token + ':cb0/0/0/0.png',
+              method: 'GET',
+              headers: {host: 'localhost' },
+              encoding: 'binary'
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 401);
+              var re = RegExp('permission denied');
+              assert.ok(res.body.match(re), 'No "permission denied" error: ' + res.body);
+              next(err);
+          });
+        },
+        function do_get_grid_layer0_unauth(err)
+        {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup/' + expected_token
+                 + '/0/0/0/0.grid.json',
+              headers: {host: 'localhost' },
+              method: 'GET'
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 401);
+              var re = RegExp('permission denied');
+              assert.ok(res.body.match(re), 'No "permission denied" error: ' + res.body);
+              next(err);
+          });
+        },
+        function do_get_grid_layer1_unauth(err)
+        {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup/' + expected_token
+                 + '/1/0/0/0.grid.json',
+              headers: {host: 'localhost' },
+              method: 'GET'
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 401);
+              var re = RegExp('permission denied');
+              assert.ok(res.body.match(re), 'No "permission denied" error: ' + res.body);
+              next(err);
+          });
+        },
+        function finish(err) {
+          var errors = [];
+          if ( err ) {
+            errors.push(err.message);
+            console.log("Error: " + err);
+          }
+          redis_client.keys("map_style|test_cartodb_user_1_db|~" + expected_token, function(err, matches) {
+              if ( err ) errors.push(err.message);
+              assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches);
+              redis_client.del(matches, function(err) {
+                if ( err ) errors.push(err.message);
+                if ( errors.length ) done(new Error(errors));
+                else done(null);
+              });
+          });
+        }
+      );
+    });
+
+    // https://github.com/cartodb/Windshaft-cartodb/issues/81
+    test("invalid text-name in CartoCSS", function(done) {
+
+      var layergroup =  {
+        version: '1.0.1',
+        layers: [
+           { options: {
+               sql: "select 1 as cartodb_id, 'SRID=3857;POINT(0 0)'::geometry as the_geom_webmercator",
+               cartocss: '#sample { text-name: cartodb_id; text-face-name: "Dejagnu"; }',
+               cartocss_version: '2.1.0',
+             } }
+        ]
+      };
+
+      assert.response(server, {
+          url: '/tiles/layergroup?',
+          method: 'POST',
+          headers: {host: 'localhost', 'Content-Type': 'application/json' },
+          data: JSON.stringify(layergroup)
+      }, {}, function(res) {
+          assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body);
+          var parsed = JSON.parse(res.body);
+          assert.equal(parsed.errors.length, 1);
+          var errmsg = parsed.errors[0];
+          assert.ok(errmsg.match(/text-face-name.*Dejagnu/), parsed.errors.toString());
+          done();
+      });
+    });
+
+    test("quotes CartoCSS", function(done) {
+
+      var layergroup =  {
+        version: '1.0.1',
+        layers: [
+           { options: {
+               sql: "select 'single''quote' as n, 'SRID=3857;POINT(0 0)'::geometry as the_geom_webmercator",
+               cartocss: '#s [n="single\'quote" ] { marker-fill:red; }',
+               cartocss_version: '2.1.0',
+             } },
+           { options: {
+               sql: "select 'double\"quote' as n, 'SRID=3857;POINT(2 0)'::geometry as the_geom_webmercator",
+               cartocss: '#s [n="double\\"quote" ] { marker-fill:red; }',
+               cartocss_version: '2.1.0',
+             } }
+        ]
+      };
+
+      assert.response(server, {
+          url: '/tiles/layergroup?',
+          method: 'POST',
+          headers: {host: 'localhost', 'Content-Type': 'application/json' },
+          data: JSON.stringify(layergroup)
+      }, {}, function(res) {
+          assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+          done();
+      });
+    });
+
+    // See https://github.com/CartoDB/Windshaft-cartodb/issues/87
+    test("exponential notation in CartoCSS filter values", function(done) {
+      var layergroup =  {
+        version: '1.0.1',
+        layers: [
+           { options: {
+               sql: "select .4 as n, 'SRID=3857;POINT(0 0)'::geometry as the_geom_webmercator",
+               cartocss: '#s [n<=.2e-2] { marker-fill:red; }',
+               cartocss_version: '2.1.0',
+             } }
+        ]
+      };
+      assert.response(server, {
+          url: '/tiles/layergroup?',
+          method: 'POST',
+          headers: {host: 'localhost', 'Content-Type': 'application/json' },
+          data: JSON.stringify(layergroup)
+      }, {}, function(res) {
+          assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+          done();
+      });
+    });
+
+    // See https://github.com/CartoDB/Windshaft-cartodb/issues/93
+    test("accepts unused directives", function(done) {
+      var layergroup =  {
+        version: '1.0.0',
+        layers: [
+           { options: {
+               sql: "select 'SRID=3857;POINT(0 0)'::geometry as the_geom_webmercator",
+               cartocss: '#layer { point-transform:"scale(20)"; }', 
+               cartocss_version: '2.0.1'
+             } }
+        ]
+      };
+      var expected_token; // = "e34dd7e235138a062f8ba7ad051aa3a7";
+      Step(
+        function do_post()
+        {
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup',
+              method: 'POST',
+              headers: {host: 'localhost', 'Content-Type': 'application/json' },
+              data: JSON.stringify(layergroup)
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 200, res.body);
+              var parsedBody = JSON.parse(res.body);
+              var expectedBody = { layergroupid: expected_token };
+              if ( expected_token ) {
+                assert.equal(parsedBody.layergroupid, expected_token + ':' + expected_last_updated_epoch);
+              }
+              else {
+                var token_components = parsedBody.layergroupid.split(':');
+                expected_token = token_components[0];
+                expected_last_updated_epoch = token_components[1];
+              }
+              next(null, res);
+          });
+        },
+        function do_get_tile(err)
+        {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup/' + expected_token + ':cb0/0/0/0.png',
+              method: 'GET',
+              headers: {host: 'localhost' },
+              encoding: 'binary'
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 200, res.body);
+              assert.equal(res.headers['content-type'], "image/png");
+              assert.imageEqualsFile(res.body, windshaft_fixtures + '/test_default_mapnik_point.png', 2,
+                function(err, similarity) {
+                  next(err);
+              });
+          });
+        },
+        function finish(err) {
+          var errors = [];
+          if ( err ) {
+            errors.push(err.message);
+            console.log("Error: " + err);
+          }
+          redis_client.keys("map_style|test_cartodb_user_1_db|~" + expected_token, function(err, matches) {
+              if ( err ) errors.push(err.message);
+              assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches);
+              redis_client.del(matches, function(err) {
+                if ( err ) errors.push(err.message);
+                if ( errors.length ) done(new Error(errors));
+                else done(null);
+              });
+          });
+        }
+      );
+    });
+
+    // See https://github.com/CartoDB/Windshaft-cartodb/issues/91
+    // and https://github.com/CartoDB/Windshaft-cartodb/issues/38
+    test("tiles for private tables can be fetched with api_key", function(done) {
+      var errors = [];
+      var layergroup =  {
+        version: '1.0.0',
+        layers: [
+           { options: {
+               sql: "select * from test_table_private_1 LIMIT 0",
+               cartocss: '#layer { marker-fill:red; }', 
+               cartocss_version: '2.0.1'
+             } }
+        ]
+      };
+      var expected_token; // = "e34dd7e235138a062f8ba7ad051aa3a7";
+      Step(
+        function do_post()
+        {
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup?api_key=1234',
+              method: 'POST',
+              headers: {host: 'localhost', 'Content-Type': 'application/json' },
+              data: JSON.stringify(layergroup)
+          }, {}, function(res) { next(null, res); });
+        },
+        function check_result(err, res) {
+          if ( err ) throw err;
+          var next = this;
+          assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+          var parsedBody = JSON.parse(res.body);
+          if ( expected_token ) {
+            assert.equal(parsedBody.layergroupid, expected_token + ':' + expected_last_updated_epoch);
+          }
+          else {
+            var token_components = parsedBody.layergroupid.split(':');
+            expected_token = token_components[0];
+            expected_last_updated_epoch = token_components[1];
+          }
+          next(null, res);
+        },
+        function do_get_tile(err)
+        {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup/' + expected_token + ':cb0/0/0/0.png?api_key=1234',
+              method: 'GET',
+              headers: {host: 'localhost' },
+              encoding: 'binary'
+          }, {}, function(res) { next(null, res); });
+        },
+        function check_get_tile(err, res) {
+          if ( err ) throw err;
+          var next = this;
+          assert.equal(res.statusCode, 200, res.body);
+          return null;
+        },
+        function cleanup(err) {
+          if ( err ) errors.push(err.message);
+          if ( ! expected_token ) return null;
+          var next = this;
+          redis_client.keys("map_style|test_cartodb_user_1_db|~" + expected_token, function(err, matches) {
+              if ( err ) errors.push(err.message);
+              assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches);
+              redis_client.del(matches, function(err) {
+                if ( err ) errors.push(err.message);
+                next();
+              });
+          });
+        },
+        function finish(err) {
+          if ( err ) {
+            errors.push(err.message);
+            console.log("Error: " + err);
+          }
+          if ( errors.length ) done(new Error(errors));
+          else done(null);
+        }
+      );
+    });
+
+    suiteTeardown(function(done) {
+
+        // This test will add map_style records, like
+        // 'map_style|null|publicuser|my_table',
+        redis_client.keys("map_style|*", function(err, matches) {
+            redis_client.del(matches, function(err) {
+              redis_client.select(5, function(err, matches) {
+                redis_client.keys("user:localhost:mapviews*", function(err, matches) {
+                  redis_client.del(matches, function(err) {
+                    sqlapi_server.close(done);
+                  });
+                });
+              });
+            });
+        });
+
+    });
+    
+});
+

test/acceptance/server.js

@@ -3,7 +3,13 @@ var tests       = module.exports = {};
 var _           = require('underscore');
 var redis       = require('redis');
 var querystring = require('querystring');
-require(__dirname + '/../support/test_helper');
+var semver      = require('semver');
+var mapnik      = require('mapnik');
+var Step        = require('step');
+var http        = require('http');
+var SQLAPIEmu   = require(__dirname + '/../support/SQLAPIEmu.js');
+
+var helper = require(__dirname + '/../support/test_helper');
 
 var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft');
 var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
@@ -13,8 +19,30 @@ server.setMaxListeners(0);
 suite('server', function() {
 
     var redis_client = redis.createClient(global.environment.redis.port);
+    var sqlapi_server;
+
+    var mapnik_version = global.environment.mapnik_version || mapnik.versions.mapnik;
+    var test_database = 'test_cartodb_user_1_db';
+    var default_style;
+    if ( semver.satisfies(mapnik_version, '<2.1.0') ) {
+      // 2.0.0 default
+      default_style = '#<%= table %>{marker-fill: #FF6600;marker-opacity: 1;marker-width: 8;marker-line-color: white;marker-line-width: 3;marker-line-opacity: 0.9;marker-placement: point;marker-type: ellipse;marker-allow-overlap: true;}';
+    }
+    else if ( semver.satisfies(mapnik_version, '<2.2.0') ) {
+      // 2.1.0 default
+      default_style = '#<%= table %>[mapnik-geometry-type=1] {marker-fill: #FF6600;marker-opacity: 1;marker-width: 16;marker-line-color: white;marker-line-width: 3;marker-line-opacity: 0.9;marker-placement: point;marker-type: ellipse;marker-allow-overlap: true;}#<%= table %>[mapnik-geometry-type=2] {line-color:#FF6600; line-width:1; line-opacity: 0.7;}#<%= table %>[mapnik-geometry-type=3] {polygon-fill:#FF6600; polygon-opacity: 0.7; line-opacity:1; line-color: #FFFFFF;}';
+    }
+    else {
+      // 2.2.0+ default
+      default_style = '#<%= table %>["mapnik::geometry_type"=1] {marker-fill: #FF6600;marker-opacity: 1;marker-width: 16;marker-line-color: white;marker-line-width: 3;marker-line-opacity: 0.9;marker-placement: point;marker-type: ellipse;marker-allow-overlap: true;}#<%= table %>["mapnik::geometry_type"=2] {line-color:#FF6600; line-width:1; line-opacity: 0.7;}#<%= table %>["mapnik::geometry_type"=3] {polygon-fill:#FF6600; polygon-opacity: 0.7; line-opacity:1; line-color: #FFFFFF;}';
+    }
+
+    // A couple of styles to use during testing
+    var test_style_black_200 = "#test_table{marker-fill:black;marker-line-color:red;marker-width:10}";
+    var test_style_black_210 = "#test_table{marker-fill:black;marker-line-color:red;marker-width:20}";
     
-    suiteSetup(function(){
+    suiteSetup(function(done){
+      sqlapi_server = new SQLAPIEmu(global.environment.sqlapi.port, done);
     });
 
     /////////////////////////////////////////////////////////////////////////////////
@@ -33,6 +61,35 @@ suite('server', function() {
         }, function() { done(); });
     });
 
+    /////////////////////////////////////////////////////////////////////////////////
+    //
+    // GET VERSION
+    //
+    /////////////////////////////////////////////////////////////////////////////////
+
+    test("get call to server returns 200", function(done){
+        assert.response(server, {
+            url: '/version',
+            method: 'GET'
+        },{
+          status: 200
+        }, function(res) {
+          var parsed = JSON.parse(res.body);
+          assert.ok(parsed.hasOwnProperty('windshaft_cartodb'), "No 'windshaft_cartodb' version in " + parsed);
+          console.log("Windshaft-cartodb: " + parsed.windshaft_cartodb);
+          assert.ok(parsed.hasOwnProperty('windshaft'), "No 'windshaft' version in " + parsed);
+          console.log("Windshaft: " + parsed.windshaft);
+          assert.ok(parsed.hasOwnProperty('grainstore'), "No 'grainstore' version in " + parsed);
+          console.log("Grainstore: " + parsed.grainstore);
+          assert.ok(parsed.hasOwnProperty('node_mapnik'), "No 'node_mapnik' version in " + parsed);
+          console.log("Node-mapnik: " + parsed.node_mapnik);
+          assert.ok(parsed.hasOwnProperty('mapnik'), "No 'mapnik' version in " + parsed);
+          console.log("Mapnik: " + parsed.mapnik);
+          // TODO: check actual versions ?
+          done();
+        });
+    });
+
     /////////////////////////////////////////////////////////////////////////////////
     //
     // GET STYLE
@@ -46,9 +103,13 @@ suite('server', function() {
             method: 'GET'
         },{
             status: 200,
-            headers: { 'X-Cache-Channel': 'cartodb_test_user_1_db:my_table' },
-            body: '{"style":"#my_table {marker-fill: #FF6600;marker-opacity: 1;marker-width: 8;marker-line-color: white;marker-line-width: 3;marker-line-opacity: 0.9;marker-placement: point;marker-type: ellipse;marker-allow-overlap: true;}"}'
-        }, function() { done(); });
+            headers: { 'X-Cache-Channel': test_database+':my_table' },
+        }, function(res) {
+            var parsed = JSON.parse(res.body);
+            assert.equal(parsed.style, _.template(default_style, {table: 'my_table'}));
+            assert.equal(parsed.style_version, mapnik.versions.mapnik);
+            done();
+        });
     });
 
     // See https://github.com/Vizzuality/Windshaft-cartodb/issues/43
@@ -61,13 +122,30 @@ suite('server', function() {
         },{
         }, function(res) {
           // FIXME: should be 401 Unauthorized
-          assert.equal(res.statusCode, 500, res.body);
+          assert.equal(res.statusCode, 400, res.body);
           assert.deepEqual(JSON.parse(res.body),
             {error: 'Sorry, you are unauthorized (permission denied)'});
           done();
         });
     });
 
+    // See http://github.com/Vizzuality/Windshaft-cartodb/issues/55
+    test("get'ing style of private table should fail on unknown username",
+    function(done) {
+        assert.response(server, {
+            headers: {host: 'unknown_user'},
+            url: '/tiles/test_table_private_1/style',
+            method: 'GET'
+        },{
+        }, function(res) {
+          // FIXME: should be 401 Unauthorized
+          assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body);
+          assert.deepEqual(JSON.parse(res.body),
+            {error:"missing unknown_user's database_name in redis (try CARTODB/script/restore_redis)"});
+          done();
+        });
+    });
+
     test("get'ing style of private table should succeed when authenticated",
     function(done) {
         assert.response(server, {
@@ -77,11 +155,51 @@ suite('server', function() {
         },{
         }, function(res) {
           assert.equal(res.statusCode, 200, res.body);
-          assert.deepEqual(res.body, '{"style":"#test_table_private_1 {marker-fill: #FF6600;marker-opacity: 1;marker-width: 8;marker-line-color: white;marker-line-width: 3;marker-line-opacity: 0.9;marker-placement: point;marker-type: ellipse;marker-allow-overlap: true;}"}');
+          var parsed = JSON.parse(res.body);
+          var style = _.template(default_style, {table: 'test_table_private_1'});
+          assert.equal(parsed.style, style);
+          assert.equal(parsed.style_version, mapnik.versions.mapnik); 
           done();
         });
     });
 
+    // See https://github.com/CartoDB/Windshaft-cartodb/issues/94
+    test("get'ing unrenderable style", function(done) {
+      var base_key = 'map_style|'+test_database+'|issue94';
+      var style = '#s{bogus}';
+      Step(
+        function checkRedis() {
+          redis_client.keys(base_key+'*', this);
+        },
+        function setupRedisBase(err, matches) {
+          if ( err ) throw err;
+          assert.equal(matches.length, 0);
+          redis_client.set(base_key, 
+            JSON.stringify({ style: style }),
+          this);
+        },
+        function getStyle(err) {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              headers: {host: 'localhost'},
+              url: '/tiles/issue94/style',
+              method: 'GET'
+              }, {}, function(res) { next(null, res); });
+        },
+        function checkStyle(err, res) {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+          var parsed = JSON.parse(res.body);
+          assert.equal(parsed.style, style);
+          return null
+        },
+        function finish(err) {
+          done(err);
+        }
+      );
+    });
+
     /////////////////////////////////////////////////////////////////////////////////
     //
     // POST STYLE
@@ -106,7 +224,7 @@ suite('server', function() {
             headers: {host: 'localhost', 'Content-Type': 'application/x-www-form-urlencoded' },
             data: querystring.stringify({style: '#my_table3{backgxxxxxround-color:#fff;}'})
         },{
-            status: 500, // FIXME: should be 400 !
+            status: 400, 
             body: /Unrecognized rule: backgxxxxxround-color/
         }, function() { done(); });
     });
@@ -117,10 +235,11 @@ suite('server', function() {
             method: 'POST',
             headers: {host: 'localhost', 'Content-Type': 'application/x-www-form-urlencoded' },
             data: querystring.stringify({style: '#my_table3{'})
-        },{
-            status: 500, // FIXME: should be 400 !
-            body: /Missing closing/
-        }, function() { done(); });
+        },{}, function(res) {
+          assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body);
+          assert.ok( RegExp(/missing closing/i).test(res.body) );
+          done();
+        });
     });
     
     test("post'ing multiple bad styles returns 400 with error array", function(done){
@@ -130,7 +249,7 @@ suite('server', function() {
             headers: {host: 'localhost', 'Content-Type': 'application/x-www-form-urlencoded' },
             data: querystring.stringify({style: '#my_table4{backgxxxxxround-color:#fff;foo:bar}'})
         },{
-            status: 500, // FIXME: should be 400 !
+            status: 400, 
         }, function(res) {
           var parsed = JSON.parse(res.body);
           assert.equal(parsed.length, 2);
@@ -145,7 +264,7 @@ suite('server', function() {
             url: '/tiles/my_table5/style?map_key=1234',
             method: 'POST',
             headers: {host: 'localhost', 'Content-Type': 'application/x-www-form-urlencoded' },
-            data: querystring.stringify({style: 'Map {background-color:#fff;}'})
+            data: querystring.stringify({style: 'Map { background-color:#fff; }'})
         },{
         }, function(res) {
             assert.equal(res.statusCode, 200, res.body);
@@ -159,7 +278,7 @@ suite('server', function() {
             url: '/tiles/my_table5/style?api_key=1234',
             method: 'POST',
             headers: {host: 'localhost', 'Content-Type': 'application/x-www-form-urlencoded' },
-            data: querystring.stringify({style: 'Map {background-color:#fff;}'})
+            data: querystring.stringify({style: 'Map { background-color:#fff; }'})
         },{}, function(res) {
             assert.equal(res.statusCode, 200, res.body);
             done();
@@ -172,7 +291,7 @@ suite('server', function() {
             url: '/tiles/my_table5/style?map_key=1234',
             method: 'POST',
             headers: {host: 'localhost', 'Content-Type': 'application/x-www-form-urlencoded' },
-            data: querystring.stringify({style: 'Map {background-color:#fff;}'})
+            data: querystring.stringify({style: 'Map { background-color:#fff; }'})
         },{
         }, function(res) {
           assert.equal(res.statusCode, 200, res.body);
@@ -180,10 +299,10 @@ suite('server', function() {
               url: '/tiles/my_table5/style',
               method: 'POST',
               headers: {host: 'localhost', 'Content-Type': 'application/x-www-form-urlencoded' },
-              data: querystring.stringify({style: 'Map {background-color:#aaa;}'})
+              data: querystring.stringify({style: 'Map { background-color:#aaa; }'})
           },{}, function(res) {
             // FIXME: should be 401 Unauthorized
-            assert.equal(res.statusCode, 500, res.body);
+            assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body);
             assert.ok(res.body.indexOf('map state cannot be changed by unauthenticated request') != -1, res.body);
 
             assert.response(server, {
@@ -192,22 +311,25 @@ suite('server', function() {
                 method: 'GET'
             },{
                 status: 200,
-                body: JSON.stringify({style: 'Map {background-color:#fff;}'})
-            }, function() { done(); });
+            }, function(res) {
+              var parsed = JSON.parse(res.body);
+              assert.equal(parsed.style, 'Map { background-color:#fff; }');
+              assert.equal(parsed.style_version, '2.0.0');
+              done();
+            });
 
           });
         });
     });
 
     test("post'ing good style returns 200 then getting returns original style", function(done){
-        var style = 'Map {background-color:#fff;}';
+        var style = 'Map { background-color:#fff; }';
         assert.response(server, {
             url: '/tiles/my_table5/style?map_key=1234',
             method: 'POST',
             headers: {host: 'localhost', 'Content-Type': 'application/x-www-form-urlencoded' },
-            data: querystring.stringify({style: style})
-        },{
-        }, function(res) { 
+            data: querystring.stringify({style: style, style_version: '2.0.2'})
+        },{}, function(res) { 
 
             assert.equal(res.statusCode, 200, res.body);
 
@@ -215,15 +337,54 @@ suite('server', function() {
                 headers: {host: 'localhost'},
                 url: '/tiles/my_table5/style',
                 method: 'GET'
-            },{
-                status: 200,
-                body: JSON.stringify({style: style})
-            }, function() { done(); });
+            },{}, function(res) {
+              assert.equal(res.statusCode, 200, res.body);
+              var parsed = JSON.parse(res.body);
+              assert.equal(parsed.style, style);
+              assert.equal(parsed.style_version, '2.0.2');
+
+              assert.response(server, {
+                  headers: {host: 'localhost'},
+                  url: '/tiles/my_table5/style?style_convert=true',
+                  method: 'GET'
+              },{}, function(res) {
+                assert.equal(res.statusCode, 200, res.body);
+                var parsed = JSON.parse(res.body);
+                assert.equal(parsed.style, style);
+                assert.equal(parsed.style_version, mapnik.versions.mapnik);
+                done();
+              });
+            });
 
         });
     
     });
 
+    test("post'ing good style with style_convert returns 200 then getting returns converted style", function(done){
+        var style = 'Map { background-color:#fff; }';
+        assert.response(server, {
+            url: '/tiles/my_table5/style?map_key=1234',
+            method: 'POST',
+            headers: {host: 'localhost', 'Content-Type': 'application/x-www-form-urlencoded' },
+            data: querystring.stringify({style: style, style_version: '2.0.2', style_convert: true})
+        },{}, function(res) { 
+
+            assert.equal(res.statusCode, 200, res.body);
+            assert.response(server, {
+                headers: {host: 'localhost'},
+                url: '/tiles/my_table5/style',
+                method: 'GET'
+            },{}, function(res) {
+              assert.equal(res.statusCode, 200, res.body);
+              var parsed = JSON.parse(res.body);
+              // NOTE: no transform expected for the specific style
+              assert.equal(parsed.style, style);
+              assert.equal(parsed.style_version, mapnik.versions.mapnik);
+              done();
+            });
+        });
+    });
+
     /////////////////////////////////////////////////////////////////////////////////
     //
     // DELETE STYLE
@@ -248,14 +409,18 @@ suite('server', function() {
               method: 'GET'
           },{
               status: 200,
-              body: JSON.stringify({style: 'Map {background-color:#fff;}'})
-          }, function() { done(); });
+          }, function(res) {
+              var parsed = JSON.parse(res.body);
+              assert.equal(parsed.style, 'Map { background-color:#fff; }');
+              //assert.equal(parsed.version, '2.0.0');
+              done();
+          });
         });
     });
 
     test("delete'ing style returns 200 then getting returns default style", function(done){
         // this is the default style
-        var style = '#my_table5 {marker-fill: #FF6600;marker-opacity: 1;marker-width: 8;marker-line-color: white;marker-line-width: 3;marker-line-opacity: 0.9;marker-placement: point;marker-type: ellipse;marker-allow-overlap: true;}'
+        var style = _.template(default_style, {table: 'my_table5'});
         assert.response(server, {
             url: '/tiles/my_table5/style?map_key=1234',
             method: 'DELETE',
@@ -313,7 +478,7 @@ suite('server', function() {
             method: 'GET'
         },{
             status: 200,
-            headers: { 'X-Cache-Channel': 'cartodb_test_user_1_db:my_tablez' },
+            headers: { 'X-Cache-Channel': test_database+':my_tablez' },
             body: '{"infowindow":null}'
         }, function() { done(); });
     });
@@ -349,7 +514,24 @@ suite('server', function() {
             method: 'GET'
         },{}, function(res) {
           // FIXME: should be 401 Unauthorized
-          assert.equal(res.statusCode, 500, res.body);
+          assert.equal(res.statusCode, 500, res.statusCode + ': ' + res.body);
+          done();
+        });
+    });
+
+    // See http://github.com/Vizzuality/Windshaft-cartodb/issues/55
+    test("get'ing infowindow of private table should fail on unknown username",
+    function(done) {
+        assert.response(server, {
+            headers: {host: 'unknown_user'},
+            url: '/tiles/test_table_private_1/infowindow',
+            method: 'GET'
+        },{
+        }, function(res) {
+          // FIXME: should be 401 Unauthorized
+          assert.equal(res.statusCode, 500, res.statusCode + ': ' + res.body);
+          assert.deepEqual(JSON.parse(res.body),
+            {error:"missing unknown_user's database_name in redis (try CARTODB/script/restore_redis)"});
           done();
         });
     });
@@ -380,7 +562,7 @@ suite('server', function() {
         },{
             status: 200,
             headers: { 'Content-Type': 'text/javascript; charset=utf-8; charset=utf-8',
-                       'X-Cache-Channel': 'cartodb_test_user_1_db:gadm4' }
+                       'X-Cache-Channel': test_database+':gadm4' }
         }, function() { done(); });
     });
     
@@ -420,6 +602,23 @@ suite('server', function() {
         });
     });
 
+    // See http://github.com/Vizzuality/Windshaft-cartodb/issues/55
+    test("get'ing grid of private table should fail on unknown username",
+    function(done) {
+        assert.response(server, {
+            headers: {host: 'unknown_user'},
+            url: '/tiles/test_table_private_1/6/31/24.grid.json',
+            method: 'GET'
+        },{
+        }, function(res) {
+          // FIXME: should be 401 Unauthorized
+          assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body);
+          assert.deepEqual(JSON.parse(res.body),
+            {error:"missing unknown_user's database_name in redis (try CARTODB/script/restore_redis)"});
+          done();
+        });
+    });
+
     test("get'ing the grid of a private table should succeed when authenticated",
     function(done) {
         assert.response(server, {
@@ -437,7 +636,40 @@ suite('server', function() {
     // GET TILE
     //
     /////////////////////////////////////////////////////////////////////////////////
-    
+
+    test("should send Cache-Control header with short expiration by default", function(done){
+        assert.response(server, {
+            headers: {host: 'localhost'},
+            url: '/tiles/gadm4/6/31/24.png',
+            method: 'GET'
+        },{
+            status: 200,
+        }, function(res) {
+          var cc = res.headers['cache-control'];
+          assert.ok(cc);
+          //assert.equal(cc, 'public,max-age=31536000');  // 1 year
+          assert.ok(cc.match('no-cache'), cc);
+          assert.ok(cc.match('must-revalidate'), cc);
+          assert.ok(cc.match('public'), cc);
+          done();
+        });
+    });
+
+    test("should send Cache-Control header with long expiration when requested", function(done){
+        assert.response(server, {
+            headers: {host: 'localhost'},
+            url: '/tiles/gadm4/6/31/24.png?cache_policy=persist',
+            method: 'GET'
+        },{
+            status: 200,
+        }, function(res) {
+          var cc = res.headers['cache-control'];
+          assert.ok(cc);
+          assert.equal(cc, 'public,max-age=31536000');  // 1 year
+          done();
+        });
+    });
+
     test("get'ing a tile with default style should return an image", function(done){
         assert.response(server, {
             headers: {host: 'localhost'},
@@ -445,7 +677,7 @@ suite('server', function() {
             method: 'GET'
         },{
             status: 200,
-            headers: { 'Content-Type': 'image/png', 'X-Cache-Channel': 'cartodb_test_user_1_db:gadm4' }
+            headers: { 'Content-Type': 'image/png', 'X-Cache-Channel': test_database+':gadm4' }
         }, function() { done(); });
     });
     
@@ -519,6 +751,25 @@ suite('server', function() {
         });
     });
 
+    test("get'ing a tile with data from private table should fail on unknown username", function(done){
+        var sql = querystring.stringify({
+          sql: "SELECT * FROM test_table_private_1",
+          cache_buster:2 // this is to avoid getting the cached response
+        });
+        assert.response(server, {
+            headers: {host: 'unknown_user'},
+            url: '/tiles/gadm4/6/31/24.png?' + sql,
+            method: 'GET'
+        },{
+        }, function(res) {
+          // FIXME: should be 401 Unauthorized
+          assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body);
+          assert.deepEqual(JSON.parse(res.body),
+            {error:"missing unknown_user's database_name in redis (try CARTODB/script/restore_redis)"});
+          done();
+        });
+    });
+
     test("get'ing a tile with data from private table should fail when unauthenticated (uses old redis key)", function(done){
         var sql = querystring.stringify({
           sql: "SELECT * FROM test_table_private_1",
@@ -539,6 +790,382 @@ suite('server', function() {
         });
     });
 
+    test("get'ing a tile with url specified 2.0.0 style should return an expected tile",  function(done){
+        var style = querystring.stringify({style: test_style_black_200, style_version: '2.0.0'});
+        assert.response(server, {
+            headers: {host: 'localhost'},
+            url: '/tiles/test_table/15/16046/12354.png?cache_buster=4&' + style, // madrid
+            method: 'GET',
+            encoding: 'binary'
+        },{}, function(res){
+          assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+          var ct = res.headers['content-type'];
+          assert.equal(ct, 'image/png');
+          assert.imageEqualsFile(res.body, './test/fixtures/test_table_15_16046_12354_styled_black.png',  2,
+            function(err, similarity) {
+              if (err) throw err;
+              done();
+          });
+        });
+    });
+
+    // See https://github.com/CartoDB/Windshaft-cartodb/issues/88
+    test("get'ing a tile from a user-specific database should return an expected tile",  function(done){
+        var style = querystring.stringify({style: test_style_black_200, style_version: '2.0.0'});
+        var backupDBHost = global.environment.postgres.host;
+        global.environment.postgres.host = '6.6.6.6';
+        Step (
+          function() {
+            var next = this;
+            assert.response(server, {
+                headers: {host: 'cartodb250user'},
+                url: '/tiles/test_table/15/16046/12354.png?cache_buster=4.10&' + style, 
+                method: 'GET',
+                encoding: 'binary'
+            },{}, function(res){
+              next(null, res);
+            });
+          },
+          function checkRes(err, res) {
+            assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+            var ct = res.headers['content-type'];
+            assert.equal(ct, 'image/png');
+            assert.imageEqualsFile(res.body,
+              './test/fixtures/test_table_15_16046_12354_styled_black.png',
+              2, this);
+          },
+          function checkImage(err, similarity) {
+              if (err) throw err;
+              return null
+          },
+          function finish(err) {
+            global.environment.postgres.host = backupDBHost;
+            done(err);
+          }
+        );
+    });
+
+    // See https://github.com/CartoDB/Windshaft-cartodb/issues/89
+    test("get'ing a tile with a user-specific database password",  function(done){
+        var style = querystring.stringify({style: test_style_black_200, style_version: '2.0.0'});
+        var backupDBPass = global.settings.postgres_auth_pass;
+        global.settings.postgres_auth_pass = '<%= user_password %>';
+        Step (
+          function() {
+            var next = this;
+            assert.response(server, {
+                headers: {host: 'cartodb250user'},
+                url: '/tiles/test_table/15/16046/12354.png?'
+                  + 'cache_buster=4.20&api_key=4321&' + style, 
+                method: 'GET',
+                encoding: 'binary'
+            },{}, function(res){
+              next(null, res);
+            });
+          },
+          function checkRes(err, res) {
+            assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+            var ct = res.headers['content-type'];
+            assert.equal(ct, 'image/png');
+            assert.imageEqualsFile(res.body,
+              './test/fixtures/test_table_15_16046_12354_styled_black.png',
+              2, this);
+          },
+          function checkImage(err, similarity) {
+              if (err) throw err;
+              return null
+          },
+          function finish(err) {
+            global.settings.postgres_auth_pass = backupDBPass;
+            done(err);
+          }
+        );
+    });
+
+    test("get'ing a tile with url specified 2.1.0 style should return an expected tile",  function(done){
+        var style = querystring.stringify({style: test_style_black_210, style_version: '2.1.0'});
+        assert.response(server, {
+            headers: {host: 'localhost'},
+            url: '/tiles/test_table/15/16046/12354.png?cache_buster=4&' + style, // madrid
+            method: 'GET',
+            encoding: 'binary'
+        },{}, function(res){
+          assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+          var ct = res.headers['content-type'];
+          assert.equal(ct, 'image/png');
+          assert.imageEqualsFile(res.body, './test/fixtures/test_table_15_16046_12354_styled_black.png',  2,
+            function(err, similarity) {
+              if (err) throw err;
+              done();
+          });
+        });
+    });
+
+    test("get'ing a tile with url specified 2.1.0 style (lzma version)",  function(done){
+        var qo = {
+          style: test_style_black_210,
+          style_version: '2.1.0',
+          cache_buster: 5
+        };
+        Step (
+          function compressQuery () {
+            //console.log("Compressing starts");
+            var next = this;
+            helper.lzma_compress_to_base64(JSON.stringify(qo), 1, this);
+          },
+          function sendRequest(err, lzma) {
+            if ( err ) throw err;
+            var next = this;
+            //console.log("Compressing ends: " + typeof(lzma) + " - " + lzma);
+            assert.response(server, {
+                headers: {host: 'localhost'},
+                url: '/tiles/test_table/15/16046/12354.png?lzma=' + encodeURIComponent(lzma),
+                method: 'GET',
+                encoding: 'binary'
+            },{}, function(res) { next(null, res); });
+          },
+          function checkResponse(err, res) {
+            if ( err ) throw err;
+            var next = this;
+            assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+            var ct = res.headers['content-type'];
+            assert.equal(ct, 'image/png');
+            assert.imageEqualsFile(res.body, './test/fixtures/test_table_15_16046_12354_styled_black.png',  2,
+              function(err, similarity) {
+                next(err);
+            });
+          },
+          function finish(err) {
+            done(err);
+          }
+        );
+    });
+
+    // See http://github.com/Vizzuality/Windshaft-cartodb/issues/57
+    test("GET'ing a tile as anonymous with style set by POST",  function(done){
+      var style = querystring.stringify({style: test_style_black_210, style_version: '2.1.0'});
+      Step (
+        function postStyle1() {
+          var next = this;
+          assert.response(server, {
+              method: 'POST',
+              url: '/tiles/test_table/style',
+              headers: {host: 'localhost', 'Content-Type': 'application/x-www-form-urlencoded' },
+              data: querystring.stringify({style: 'Map { background-color:#fff; }', map_key: 1234})
+          },{}, function(res) {
+            assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+            next();
+          });
+        },
+        // Load the new cache with results from Style1 above
+        function getTileAnon1(err) {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              headers: {host: 'localhost'},
+              url: '/tiles/test_table/15/16046/12354.png', 
+              method: 'GET',
+              encoding: 'binary'
+          },{}, function(res){
+            assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+            var ct = res.headers['content-type'];
+            assert.equal(ct, 'image/png');
+            assert.imageEqualsFile(res.body, './test/fixtures/blank.png',  0,
+              function(err, similarity) {
+                if (err) next(err); 
+                else next();
+            });
+          });
+        },
+        // Get again with authentication 
+        function getTileAuth1(err) {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              headers: {host: 'localhost'},
+              url: '/tiles/test_table/15/16046/12354.png?map_key=1234', 
+              method: 'GET',
+              encoding: 'binary'
+          },{}, function(res){
+            assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+            var ct = res.headers['content-type'];
+            assert.equal(ct, 'image/png');
+            assert.imageEqualsFile(res.body, './test/fixtures/blank.png',  0,
+              function(err, similarity) {
+                if (err) next(err); 
+                else next();
+            });
+          });
+        },
+        // Change the style
+        function postStyle2(err) {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              method: 'POST',
+              url: '/tiles/test_table/style',
+              headers: {host: 'localhost', 'Content-Type': 'application/x-www-form-urlencoded' },
+              data: querystring.stringify({style: test_style_black_200, map_key: 1234})
+          },{}, function(res) {
+            try {
+              assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+              next();
+            }
+            catch (err) { next(err); }
+          });
+        },
+        // Verify the Style2 is applied. NOTE: pass the SAME cache_buster as before!
+        function getTileAnon2(err) {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              headers: {host: 'localhost'},
+              url: '/tiles/test_table/15/16046/12354.png', 
+              method: 'GET',
+              encoding: 'binary'
+          },{}, function(res){
+            assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+            var ct = res.headers['content-type'];
+            assert.equal(ct, 'image/png');
+            assert.imageEqualsFile(res.body, './test/fixtures/test_table_15_16046_12354_styled_black.png',  2,
+              function(err, similarity) {
+                // NOTE: we expect them to be EQUAL here
+                if (err) { next(err); return; }
+                next();
+            });
+          });
+        },
+        // Delete the style
+        function delStyle(err) {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              method: 'DELETE',
+              url: '/tiles/test_table/style?map_key=1234',
+              headers: {host: 'localhost'}
+          },{}, function(res) {
+            try {
+              assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+              next();
+            }
+            catch (err) { next(err); }
+          });
+        },
+        // Verify the default style is applied. 
+        function getTileAnon3(err) {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              headers: {host: 'localhost'},
+              url: '/tiles/test_table/15/16046/12354.png?cache_buster=2314',
+              method: 'GET',
+              encoding: 'binary'
+          },{}, function(res){
+            assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+            var ct = res.headers['content-type'];
+            assert.equal(ct, 'image/png');
+            assert.imageEqualsFile(res.body, './test/fixtures/test_table_15_16046_12354_styled_black.png',  2,
+              function(err, similarity) {
+                // NOTE: we expect them to be different here
+                if (err) next(); 
+                else next(new Error('Last posted style still in effect after delete'));
+            });
+          });
+        },
+        function finish(err) {
+          done(err);
+        }
+      );
+    });
+
+    test("uses sqlapi to figure source data of query", function(done){
+        var qo = {
+          sql: "SELECT g.cartodb_id, g.codineprov, t.the_geom_webmercator "
+              + "FROM gadm4 g, test_table t "
+              + "WHERE g.cartodb_id = t.cartodb_id",
+          map_key: 1234
+        };
+        var sqlapi;
+        Step(
+          function sendRequest(err) {
+            var next = this;
+            assert.response(server, {
+                headers: {host: 'localhost'},
+                url: '/tiles/gadm4/6/31/24.png?' + querystring.stringify(qo),
+                method: 'GET'
+            },{}, function(res) { next(null, res); });
+          },
+          function checkResponse(err, res) {
+            if ( err ) throw err;
+            assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+            var ct = res.headers['content-type'];
+            assert.equal(ct, 'image/png');
+            var cc = res.headers['x-cache-channel'];
+            assert(cc);
+            var dbname = 'test_cartodb_user_1_db'
+            assert.equal(cc.substring(0, dbname.length), dbname);
+            var jsonquery = cc.substring(dbname.length+1);
+            var sentquery = JSON.parse(jsonquery);
+            assert.equal(sentquery.api_key, qo.map_key);
+            assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$' + qo.sql + '$windshaft$)');
+            return null;
+          },
+          function finish(err) {
+            done(err);
+          }
+        );
+    });
+
+    test("requests to skip cache on sqlapi error", function(done){
+        var qo = {
+          sql: "SELECT g.cartodb_id, g.codineprov, t.the_geom_webmercator "
+              + ", 'SQLAPIERROR' is not null "
+              + "FROM gadm4 g, test_table t "
+              + "WHERE g.cartodb_id = t.cartodb_id",
+          map_key: 1234
+        };
+        var sqlapi;
+        Step(
+          function sendRequest(err) {
+            var next = this;
+            assert.response(server, {
+                headers: {host: 'localhost'},
+                url: '/tiles/gadm4/6/31/24.png?' + querystring.stringify(qo),
+                method: 'GET'
+            },{}, function(res) { next(null, res); });
+          },
+          function checkResponse(err, res) {
+            if ( err ) throw err;
+            assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+            var ct = res.headers['content-type'];
+            assert.equal(ct, 'image/png');
+            // does NOT send an x-cache-channel
+            assert.ok(!res.headers.hasOwnProperty('x-cache-channel'));
+            return null;
+          },
+          function finish(err) {
+            done(err);
+          }
+        );
+    });
+
+    // Zoom is a special variable
+    test("Specifying zoom level in CartoCSS does not need a 'zoom' variable in SQL output", function(done){
+        // NOTE: may fail if grainstore < 0.3.0 is used by Windshaft
+        var query = querystring.stringify({
+          sql: "SELECT 'SRID=3857;POINT(0 0)'::geometry as the_geom_webmercator, 1::int as cartodb_id",
+          style: '#gadm4 [ zoom>=3] { marker-fill:red; }'
+        });
+        assert.response(server, {
+            headers: {host: 'localhost'},
+            url: '/tiles/gadm4/0/0/0.png?' + query,
+            method: 'GET'
+        },{}, function(res) {
+          assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+          done();
+        });
+    });
+
     /////////////////////////////////////////////////////////////////////////////////
     //
     // DELETE CACHE 
@@ -663,7 +1290,7 @@ suite('server', function() {
         // 'map_style|null|publicuser|my_table',
         redis_client.keys("map_style|*", function(err, matches) {
             _.each(matches, function(k) { redis_client.del(k); });
-            done();
+            sqlapi_server.close(done);
         });
     });
     

test/acceptance/templates.js

@@ -0,0 +1,910 @@
+var assert      = require('../support/assert');
+var tests       = module.exports = {};
+var _           = require('underscore');
+var redis       = require('redis');
+var querystring = require('querystring');
+var semver      = require('semver');
+var mapnik      = require('mapnik');
+var Step        = require('step');
+var strftime    = require('strftime');
+var SQLAPIEmu   = require(__dirname + '/../support/SQLAPIEmu.js');
+var redis_stats_db = 5;
+
+require(__dirname + '/../support/test_helper');
+
+var windshaft_fixtures = __dirname + '/../../node_modules/windshaft/test/fixtures';
+
+var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft');
+var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
+var server = new CartodbWindshaft(serverOptions);
+server.setMaxListeners(0);
+
+suite('template_api', function() {
+
+    var redis_client = redis.createClient(global.environment.redis.port);
+    var sqlapi_server;
+    var expected_last_updated_epoch = 1234567890123; // this is hard-coded into SQLAPIEmu
+    var expected_last_updated = new Date(expected_last_updated_epoch).toISOString();
+
+    suiteSetup(function(done){
+      sqlapi_server = new SQLAPIEmu(global.environment.sqlapi.port, done);
+      // TODO: check redis is clean ?
+    });
+
+    var template_acceptance1 =  {
+        version: '0.0.1',
+        name: 'acceptance1',
+        auth: { method: 'open' },
+        layergroup:  {
+          version: '1.0.0',
+          layers: [
+             { options: {
+                 sql: 'select cartodb_id, ST_Translate(the_geom_webmercator, -5e6, 0) as the_geom_webmercator from test_table limit 2 offset 2',
+                 cartocss: '#layer { marker-fill:blue; marker-allow-overlap:true; }', 
+                 cartocss_version: '2.0.2',
+                 interactivity: 'cartodb_id'
+               } }
+          ]
+        }
+    };
+
+    test("can add template, returning id", function(done) {
+
+      var errors = [];
+      var expected_failure = false;
+      var expected_tpl_id = "localhost@acceptance1";
+      var post_request_1 = {
+          url: '/tiles/template',
+          method: 'POST',
+          headers: {host: 'localhost', 'Content-Type': 'application/json' },
+          data: JSON.stringify(template_acceptance1)
+      }
+      Step(
+        function postUnauthenticated()
+        {
+          var next = this;
+          assert.response(server, post_request_1, {},
+            function(res) { next(null, res); });
+        },
+        function postTemplate(err, res)
+        {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 401);
+          var parsed = JSON.parse(res.body);
+          assert.ok(parsed.hasOwnProperty('error'), res.body);
+          err = parsed.error;
+          assert.ok(err.match(/only.*authenticated.*user/i),
+            'Unexpected error response: ' + err);
+          post_request_1.url += '?api_key=1234';
+          var next = this;
+          assert.response(server, post_request_1, {},
+            function(res) { next(null, res); });
+        },
+        function rePostTemplate(err, res)
+        {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 200, res.body);
+          var parsedBody = JSON.parse(res.body);
+          var expectedBody = { template_id: expected_tpl_id };
+          assert.deepEqual(parsedBody, expectedBody);
+          var next = this;
+          assert.response(server, post_request_1, {},
+            function(res) { next(null, res); });
+        },
+        function checkFailure(err, res)
+        {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 400, res.body);
+          var parsedBody = JSON.parse(res.body);
+          assert.ok(parsedBody.hasOwnProperty('error'), res.body);
+          assert.ok(parsedBody.error.match(/already exists/i),
+            'Unexpected error for pre-existing template name: ' + parsedBody.error);
+          return null;
+        },
+        function finish(err) {
+          if ( err ) errors.push(err);
+          redis_client.keys("map_*|localhost", function(err, keys) {
+              if ( err ) errors.push(err.message);
+              var todrop = _.map(keys, function(m) {
+                if ( m.match(/^map_(tpl|crt)|/) )
+                  return m;
+              });
+              if ( todrop.length != 2 ) {
+                errors.push(new Error("Unexpected keys in redis: " + todrop));
+              } else {
+                if ( todrop.indexOf('map_tpl|localhost') == -1 ) {
+                  errors.push(new Error("Missing 'map_tpl|localhost' key in redis"));
+                }
+                if ( todrop.indexOf('map_crt|localhost') == -1 ) {
+                  errors.push(new Error("Missing 'map_crt|localhost' key in redis"));
+                }
+              }
+              redis_client.del(todrop, function(err) {
+                if ( err ) errors.push(err.message);
+                if ( errors.length ) {
+                  done(new Error(errors));
+                }
+                else done(null);
+              });
+          });
+        }
+      );
+    });
+
+    test("can list templates", function(done) {
+
+      var errors = [];
+      var expected_failure = false;
+      var tplid1, tplid2;
+      Step(
+        function postTemplate1(err, res)
+        {
+          var next = this;
+          var post_request = {
+              url: '/tiles/template?api_key=1234',
+              method: 'POST',
+              headers: {host: 'localhost', 'Content-Type': 'application/json' },
+              data: JSON.stringify(template_acceptance1)
+          }
+          assert.response(server, post_request, {},
+            function(res) { next(null, res); });
+        },
+        function postTemplate2(err, res)
+        {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 200, res.body);
+          var parsed = JSON.parse(res.body);
+          assert.ok(parsed.hasOwnProperty('template_id'),
+            "Missing 'template_id' from response body: " + res.body);
+          tplid1 = parsed.template_id;
+
+          var next = this;
+          var backup_name = template_acceptance1.name;
+          template_acceptance1.name += '_new';
+          var post_request = {
+              url: '/tiles/template?api_key=1234',
+              method: 'POST',
+              headers: {host: 'localhost', 'Content-Type': 'application/json' },
+              data: JSON.stringify(template_acceptance1)
+          }
+          template_acceptance1.name = backup_name;
+          assert.response(server, post_request, {},
+            function(res) { next(null, res); });
+        },
+        function litsTemplatesUnauthenticated(err, res)
+        {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 200, res.body);
+          var parsed = JSON.parse(res.body);
+          assert.ok(parsed.hasOwnProperty('template_id'),
+            "Missing 'template_id' from response body: " + res.body);
+          tplid2 = parsed.template_id;
+          var next = this;
+          var get_request = {
+              url: '/tiles/template',
+              method: 'GET',
+              headers: {host: 'localhost'}
+          }
+          assert.response(server, get_request, {},
+            function(res) { next(null, res); });
+        },
+        function litsTemplates(err, res)
+        {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 401, res.statusCode + ': ' + res.body);
+          var parsed = JSON.parse(res.body);
+          assert.ok(parsed.hasOwnProperty('error'),
+            'Missing error from response: ' + res.body);
+          err = parsed.error;
+          assert.ok(err.match(/authenticated user/), err);
+          var next = this;
+          var get_request = {
+              url: '/tiles/template?api_key=1234',
+              method: 'GET',
+              headers: {host: 'localhost'}
+          }
+          assert.response(server, get_request, {},
+            function(res) { next(null, res); });
+        },
+        function checkList(err, res)
+        {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 200, res.body);
+          var parsed = JSON.parse(res.body);
+          assert.ok(parsed.hasOwnProperty('template_ids'),
+            "Missing 'template_ids' from response body: " + res.body);
+          var ids = parsed.template_ids;
+          assert.equal(ids.length, 2);
+          assert.ok(ids.indexOf(tplid1) != -1,
+            'Missing "' + tplid1 + "' from list response: " + ids.join(','));
+          assert.ok(ids.indexOf(tplid2) != -1,
+            'Missing "' + tplid2 + "' from list response: " + ids.join(','));
+          return null;
+        },
+        function finish(err) {
+          if ( err ) errors.push(err);
+          redis_client.keys("map_*|localhost", function(err, keys) {
+              if ( err ) errors.push(err.message);
+              var todrop = _.map(keys, function(m) {
+                if ( m.match(/^map_(tpl|crt)|/) )
+                  return m;
+              });
+              if ( todrop.length != 2 ) {
+                errors.push(new Error("Unexpected keys in redis: " + todrop));
+              } else {
+                if ( todrop.indexOf('map_tpl|localhost') == -1 ) {
+                  errors.push(new Error("Missing 'map_tpl|localhost' key in redis"));
+                }
+                if ( todrop.indexOf('map_crt|localhost') == -1 ) {
+                  errors.push(new Error("Missing 'map_crt|localhost' key in redis"));
+                }
+              }
+              redis_client.del(todrop, function(err) {
+                if ( err ) errors.push(err.message);
+                if ( errors.length ) {
+                  done(new Error(errors));
+                }
+                else done(null);
+              });
+          });
+        }
+      );
+    });
+
+    test("can update template", function(done) {
+
+      var errors = [];
+      var expected_failure = false;
+      var tpl_id;
+      Step(
+        function postTemplate(err, res)
+        {
+          var next = this;
+          var post_request = {
+              url: '/tiles/template?api_key=1234',
+              method: 'POST',
+              headers: {host: 'localhost', 'Content-Type': 'application/json' },
+              data: JSON.stringify(template_acceptance1)
+          }
+          assert.response(server, post_request, {},
+            function(res) { next(null, res); });
+        },
+        function putMisnamedTemplate(err, res)
+        {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 200, res.body);
+          var parsed = JSON.parse(res.body);
+          assert.ok(parsed.hasOwnProperty('template_id'),
+            "Missing 'template_id' from response body: " + res.body);
+          tpl_id = parsed.template_id;
+          var backup_name = template_acceptance1.name;
+          template_acceptance1.name = 'changed_name';
+          var put_request = {
+              url: '/tiles/template/' + tpl_id + '/?api_key=1234',
+              method: 'PUT',
+              headers: {host: 'localhost', 'Content-Type': 'application/json' },
+              data: JSON.stringify(template_acceptance1)
+          }
+          template_acceptance1.name = backup_name;
+          var next = this;
+          assert.response(server, put_request, {},
+            function(res) { next(null, res); });
+        },
+        function putUnexistentTemplate(err, res)
+        {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 400, res.statusCode + ": " + res.body);
+          var parsedBody = JSON.parse(res.body);
+          assert.ok(parsedBody.hasOwnProperty('error'), res.body);
+          assert.ok(parsedBody.error.match(/cannot update name/i),
+            'Unexpected error for invalid update: ' + parsedBody.error);
+          var put_request = {
+              url: '/tiles/template/unexistent/?api_key=1234',
+              method: 'PUT',
+              headers: {host: 'localhost', 'Content-Type': 'application/json' },
+              data: JSON.stringify(template_acceptance1)
+          }
+          var next = this;
+          assert.response(server, put_request, {},
+            function(res) { next(null, res); });
+        },
+        function putValidTemplate(err, res)
+        {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 400, res.statusCode + ": " + res.body);
+          var parsedBody = JSON.parse(res.body);
+          assert.ok(parsedBody.hasOwnProperty('error'), res.body);
+          assert.ok(parsedBody.error.match(/cannot update name/i),
+            'Unexpected error for invalid update: ' + parsedBody.error);
+          var put_request = {
+              url: '/tiles/template/' + tpl_id + '/?api_key=1234',
+              method: 'PUT',
+              headers: {host: 'localhost', 'Content-Type': 'application/json' },
+              data: JSON.stringify(template_acceptance1)
+          }
+          var next = this;
+          assert.response(server, put_request, {},
+            function(res) { next(null, res); });
+        },
+        function checkValidUpate(err, res)
+        {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 200, res.statusCode + ": " + res.body);
+          var parsed = JSON.parse(res.body);
+          assert.ok(parsed.hasOwnProperty('template_id'),
+            "Missing 'template_id' from response body: " + res.body);
+          assert.equal(tpl_id, parsed.template_id);
+          return null;
+        },
+        function finish(err) {
+          if ( err ) errors.push(err);
+          redis_client.keys("map_*|localhost", function(err, keys) {
+              if ( err ) errors.push(err.message);
+              var todrop = _.map(keys, function(m) {
+                if ( m.match(/^map_(tpl|crt)|/) )
+                  return m;
+              });
+              if ( todrop.length != 2 ) {
+                errors.push(new Error("Unexpected keys in redis: " + todrop));
+              } else {
+                if ( todrop.indexOf('map_tpl|localhost') == -1 ) {
+                  errors.push(new Error("Missing 'map_tpl|localhost' key in redis"));
+                }
+                if ( todrop.indexOf('map_crt|localhost') == -1 ) {
+                  errors.push(new Error("Missing 'map_crt|localhost' key in redis"));
+                }
+              }
+              redis_client.del(todrop, function(err) {
+                if ( err ) errors.push(err.message);
+                if ( errors.length ) {
+                  done(new Error(errors));
+                }
+                else done(null);
+              });
+          });
+        }
+      );
+    });
+
+    test("can get a template by id", function(done) {
+
+      var errors = [];
+      var expected_failure = false;
+      var tpl_id;
+      Step(
+        function postTemplate(err, res)
+        {
+          var next = this;
+          var post_request = {
+              url: '/tiles/template?api_key=1234',
+              method: 'POST',
+              headers: {host: 'localhost', 'Content-Type': 'application/json' },
+              data: JSON.stringify(template_acceptance1)
+          }
+          assert.response(server, post_request, {},
+            function(res) { next(null, res); });
+        },
+        function getTemplateUnauthorized(err, res)
+        {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 200, res.body);
+          var parsed = JSON.parse(res.body);
+          assert.ok(parsed.hasOwnProperty('template_id'),
+            "Missing 'template_id' from response body: " + res.body);
+          tpl_id = parsed.template_id;
+          var get_request = {
+              url: '/tiles/template/' + tpl_id, 
+              method: 'GET',
+              headers: {host: 'localhost'}
+          }
+          var next = this;
+          assert.response(server, get_request, {},
+            function(res) { next(null, res); });
+        },
+        function getTemplate(err, res)
+        {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 401, res.statusCode + ": " + res.body);
+          var parsedBody = JSON.parse(res.body);
+          assert.ok(parsedBody.hasOwnProperty('error'), res.body);
+          assert.ok(parsedBody.error.match(/only.*authenticated.*user/i),
+            'Unexpected error for unauthenticated template get: ' + parsedBody.error);
+          var get_request = {
+              url: '/tiles/template/' + tpl_id + '?api_key=1234', 
+              method: 'GET',
+              headers: {host: 'localhost'}
+          }
+          var next = this;
+          assert.response(server, get_request, {},
+            function(res) { next(null, res); });
+        },
+        function checkReturnTemplate(err, res)
+        {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 200, res.statusCode + ": " + res.body);
+          var parsed = JSON.parse(res.body);
+          assert.ok(parsed.hasOwnProperty('template'),
+            "Missing 'template' from response body: " + res.body);
+          assert.deepEqual(template_acceptance1, parsed.template);
+          return null;
+        },
+        function finish(err) {
+          if ( err ) errors.push(err);
+          redis_client.keys("map_*|localhost", function(err, keys) {
+              if ( err ) errors.push(err.message);
+              var todrop = _.map(keys, function(m) {
+                if ( m.match(/^map_(tpl|crt)|/) )
+                  return m;
+              });
+              if ( todrop.length != 2 ) {
+                errors.push(new Error("Unexpected keys in redis: " + todrop));
+              } else {
+                if ( todrop.indexOf('map_tpl|localhost') == -1 ) {
+                  errors.push(new Error("Missing 'map_tpl|localhost' key in redis"));
+                }
+                if ( todrop.indexOf('map_crt|localhost') == -1 ) {
+                  errors.push(new Error("Missing 'map_crt|localhost' key in redis"));
+                }
+              }
+              redis_client.del(todrop, function(err) {
+                if ( err ) errors.push(err.message);
+                if ( errors.length ) {
+                  done(new Error(errors));
+                }
+                else done(null);
+              });
+          });
+        }
+      );
+    });
+
+    test("can delete a template by id", function(done) {
+
+      var errors = [];
+      var expected_failure = false;
+      var tpl_id;
+      Step(
+        function postTemplate(err, res)
+        {
+          var next = this;
+          var post_request = {
+              url: '/tiles/template?api_key=1234',
+              method: 'POST',
+              headers: {host: 'localhost', 'Content-Type': 'application/json' },
+              data: JSON.stringify(template_acceptance1)
+          }
+          assert.response(server, post_request, {},
+            function(res) { next(null, res); });
+        },
+        function getTemplate(err, res)
+        {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 200, res.body);
+          var parsed = JSON.parse(res.body);
+          assert.ok(parsed.hasOwnProperty('template_id'),
+            "Missing 'template_id' from response body: " + res.body);
+          tpl_id = parsed.template_id;
+          var get_request = {
+              url: '/tiles/template/' + tpl_id + '?api_key=1234', 
+              method: 'GET',
+              headers: {host: 'localhost'}
+          }
+          var next = this;
+          assert.response(server, get_request, {},
+            function(res) { next(null, res); });
+        },
+        function deleteTemplateUnauthorized(err, res)
+        {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 200, res.statusCode + ": " + res.body);
+          var parsed = JSON.parse(res.body);
+          assert.ok(parsed.hasOwnProperty('template'),
+            "Missing 'template' from response body: " + res.body);
+          assert.deepEqual(template_acceptance1, parsed.template);
+          var del_request = {
+              url: '/tiles/template/' + tpl_id,
+              method: 'DELETE',
+              headers: {host: 'localhost'}
+          }
+          var next = this;
+          assert.response(server, del_request, {},
+            function(res) { next(null, res); });
+        },
+        function deleteTemplate(err, res)
+        {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 401, res.statusCode + ": " + res.body);
+          var parsed = JSON.parse(res.body);
+          assert.ok(parsed.hasOwnProperty('error'),
+            "Missing 'error' from response body: " + res.body);
+          assert.ok(parsed.error.match(/only.*authenticated.*user/i),
+            'Unexpected error for unauthenticated template get: ' + parsed.error);
+          var del_request = {
+              url: '/tiles/template/' + tpl_id + '?api_key=1234', 
+              method: 'DELETE',
+              headers: {host: 'localhost'}
+          }
+          var next = this;
+          assert.response(server, del_request, {},
+            function(res) { next(null, res); });
+        },
+        function getMissingTemplate(err, res)
+        {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 204, res.statusCode + ': ' + res.body);
+          assert.ok(!res.body, 'Unexpected body in DELETE /template response');
+          var get_request = {
+              url: '/tiles/template/' + tpl_id + '?api_key=1234', 
+              method: 'GET',
+              headers: {host: 'localhost'}
+          }
+          var next = this;
+          assert.response(server, get_request, {},
+            function(res) { next(null, res); });
+        },
+        function checkGetFailure(err, res)
+        {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 404, res.statusCode + ': ' + res.body);
+          var parsed = JSON.parse(res.body);
+          assert.ok(parsed.hasOwnProperty('error'),
+            "Missing 'error' from response body: " + res.body);
+          assert.ok(parsed.error.match(/cannot find/i),
+            'Unexpected error for missing template: ' + parsed.error);
+          return null;
+        },
+        function finish(err) {
+          if ( err ) errors.push(err);
+          redis_client.keys("map_*|localhost", function(err, keys) {
+              if ( err ) errors.push(err.message);
+              var todrop = _.map(keys, function(m) {
+                if ( m.match(/^map_(tpl|crt)|/) )
+                  return m;
+              });
+              if ( todrop.length ) {
+                errors.push(new Error("Unexpected keys in redis: " + todrop));
+                redis_client.del(todrop, function(err) {
+                  if ( err ) errors.push(err.message);
+                  if ( errors.length ) {
+                    done(new Error(errors));
+                  }
+                  else done(null);
+                });
+              } else {
+                  if ( errors.length ) {
+                    done(new Error(errors));
+                  }
+                  else done(null);
+              }
+          });
+        }
+      );
+    });
+
+    test("can instanciate a template by id", function(done) {
+
+      // This map fetches data from a private table
+      var template_acceptance2 =  {
+          version: '0.0.1',
+          name: 'acceptance1',
+          auth: { method: 'token', valid_tokens: ['valid1','valid2'] },
+          layergroup:  {
+            version: '1.0.0',
+            layers: [
+               { options: {
+                   sql: "select * from test_table_private_1 LIMIT 0",
+                   cartocss: '#layer { marker-fill:blue; marker-allow-overlap:true; }', 
+                   cartocss_version: '2.0.2',
+                   interactivity: 'cartodb_id'
+                 } }
+            ]
+          }
+      };
+
+      var template_params = {};
+
+      var errors = [];
+      var expected_failure = false;
+      var tpl_id;
+      var layergroupid;
+      Step(
+        function postTemplate(err, res)
+        {
+          var next = this;
+          var post_request = {
+              url: '/tiles/template?api_key=1234',
+              method: 'POST',
+              headers: {host: 'localhost', 'Content-Type': 'application/json' },
+              data: JSON.stringify(template_acceptance2)
+          }
+          assert.response(server, post_request, {},
+            function(res) { next(null, res); });
+        },
+        function instanciateNoAuth(err, res)
+        {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 200, res.body);
+          var parsed = JSON.parse(res.body);
+          assert.ok(parsed.hasOwnProperty('template_id'),
+            "Missing 'template_id' from response body: " + res.body);
+          tpl_id = parsed.template_id;
+          var post_request = {
+              url: '/tiles/template/' + tpl_id,
+              method: 'POST',
+              headers: {host: 'localhost', 'Content-Type': 'application/json' },
+              data: JSON.stringify(template_params)
+          }
+          var next = this;
+          assert.response(server, post_request, {},
+            function(res) { next(null, res); });
+        },
+        function instanciateAuth(err, res)
+        {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 401,
+            'Unexpected success instanciating template with no auth: '
+            + res.statusCode + ': ' + res.body);
+          var parsed = JSON.parse(res.body);
+          assert.ok(parsed.hasOwnProperty('error'),
+            "Missing 'error' from response body: " + res.body);
+          assert.ok(parsed.error.match(/unauthorized/i),
+            'Unexpected error for unauthorized instance : ' + parsed.error);
+          var post_request = {
+              url: '/tiles/template/' + tpl_id + '?auth_token=valid2',
+              method: 'POST',
+              headers: {host: 'localhost', 'Content-Type': 'application/json' },
+              data: JSON.stringify(template_params)
+          }
+          var next = this;
+          assert.response(server, post_request, {},
+            function(res) { next(null, res); });
+        },
+        function fetchTileNoAuth(err, res) {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 200,
+            'Instantiating template: ' + res.statusCode + ': ' + res.body);
+          var parsed = JSON.parse(res.body);
+          assert.ok(parsed.hasOwnProperty('layergroupid'),
+            "Missing 'layergroupid' from response body: " + res.body);
+          layergroupid = parsed.layergroupid;
+          assert.ok(layergroupid.match(/^localhost@/),
+            "Returned layergroupid does not start with signer name: "
+            + layergroupid);
+          assert.ok(parsed.hasOwnProperty('last_updated'),
+            "Missing 'last_updated' from response body: " + res.body);
+          // TODO: check value of last_updated ?
+          var get_request = {
+              url: '/tiles/layergroup/' + layergroupid + ':cb0/0/0/0.png',
+              method: 'GET',
+              headers: {host: 'localhost' },
+              encoding: 'binary'
+          }
+          var next = this;
+          assert.response(server, get_request, {},
+            function(res) { next(null, res); });
+        },
+        function fetchTileAuth(err, res) {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 401,
+            'Fetching tile with no auth: ' + res.statusCode + ': ' + res.body);
+          var parsed = JSON.parse(res.body);
+          assert.ok(parsed.hasOwnProperty('error'),
+            "Missing 'error' from response body: " + res.body);
+          assert.ok(parsed.error.match(/permission denied/i),
+            'Unexpected error for unauthorized instance '
+            + '(expected /permission denied): ' + parsed.error);
+          var get_request = {
+              url: '/tiles/layergroup/' + layergroupid + '/0/0/0.png?auth_token=valid1',
+              method: 'GET',
+              headers: {host: 'localhost' },
+              encoding: 'binary'
+          }
+          var next = this;
+          assert.response(server, get_request, {},
+            function(res) { next(null, res); });
+        },
+        function checkTile(err, res) {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 200, 
+            'Unexpected error for authorized instance: '
+            + res.statusCode + ' -- ' + res.body);
+          assert.equal(res.headers['content-type'], "image/png");
+          return null;
+        },
+        function deleteTemplate(err)
+        {
+          if ( err ) throw err;
+          var del_request = {
+              url: '/tiles/template/' + tpl_id + '?api_key=1234', 
+              method: 'DELETE',
+              headers: {host: 'localhost'}
+          }
+          var next = this;
+          assert.response(server, del_request, {},
+            function(res) { next(null, res); });
+        },
+        function fetchTileDeleted(err, res) {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 204,
+            'Deleting template: ' + res.statusCode + ':' + res.body);
+          var get_request = {
+              url: '/tiles/layergroup/' + layergroupid + '/0/0/0.png?auth_token=valid1',
+              method: 'GET',
+              headers: {host: 'localhost' },
+              encoding: 'binary'
+          }
+          var next = this;
+          assert.response(server, get_request, {},
+            function(res) { next(null, res); });
+        },
+        function checkTileDeleted(err, res) {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 401, 
+            'Unexpected statusCode fetch tile after signature revokal: '
+            + res.statusCode + ':' + res.body); 
+          var parsed = JSON.parse(res.body);
+          assert.ok(parsed.hasOwnProperty('error'),
+            "Missing 'error' from response body: " + res.body);
+          assert.ok(parsed.error.match(/permission denied/i),
+            'Unexpected error for unauthorized access : ' + parsed.error);
+          return null;
+        },
+        function finish(err) {
+          if ( err ) errors.push(err);
+          redis_client.keys("map_*|localhost", function(err, keys) {
+              if ( err ) errors.push(err.message);
+              var todrop = _.map(keys, function(m) {
+                if ( m.match(/^map_(tpl|crt)|/) )
+                  return m;
+              });
+              if ( todrop.length ) {
+                errors.push(new Error("Unexpected keys in redis: " + todrop));
+                redis_client.del(todrop, function(err) {
+                  if ( err ) errors.push(err.message);
+                  if ( errors.length ) {
+                    done(new Error(errors));
+                  }
+                  else done(null);
+                });
+              } else {
+                  if ( errors.length ) {
+                    done(new Error(errors));
+                  }
+                  else done(null);
+              }
+          });
+        }
+      );
+    });
+
+    test("template instantiation raises mapviews counter", function(done) {
+      var layergroup =  {
+        stat_tag: 'random_tag',
+        version: '1.0.0',
+        layers: [
+           { options: {
+               sql: 'select 1 as cartodb_id, !pixel_height! as h, '
+                  + 'ST_Buffer(!bbox!, -32*greatest(!pixel_width!,!pixel_height!)) as the_geom_webmercator',
+               cartocss: '#layer { polygon-fill:red; }', 
+               cartocss_version: '2.0.1' 
+             } }
+        ]
+      };
+      var template =  {
+          version: '0.0.1',
+          name: 'stat_gathering',
+          auth: { method: 'open' },
+          layergroup: layergroup
+      };
+      var statskey = "user:localhost:mapviews";
+      var redis_stats_client = redis.createClient(global.environment.redis.port);
+      var template_id; // will be set on template post 
+      var now = strftime("%Y%m%d", new Date());
+      var errors = [];
+      Step(
+        function clean_stats()
+        {
+          var next = this;
+          redis_stats_client.select(redis_stats_db, function(err) {
+            if ( err ) next(err);
+            else redis_stats_client.del(statskey+':global', next);
+          });
+        },
+        function do_post_tempate(err)
+        {
+          if ( err ) throw err;
+          var post_request = {
+              url: '/tiles/template?api_key=1234',
+              method: 'POST',
+              headers: {host: 'localhost', 'Content-Type': 'application/json' },
+              data: JSON.stringify(template)
+          }
+          var next = this;
+          assert.response(server, post_request, {},
+            function(res) { next(null, res); });
+        },
+        function instantiateTemplate(err, res) {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 200, res.body);
+          template_id = JSON.parse(res.body).template_id;
+          var post_request = {
+              url: '/tiles/template/' + template_id,
+              method: 'POST',
+              headers: {host: 'localhost', 'Content-Type': 'application/json' },
+              data: JSON.stringify({})
+          }
+          var next = this;
+          assert.response(server, post_request, {},
+            function(res) { next(null, res); });
+        },
+        function check_global_stats(err, res) {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 200,
+            'Instantiating template: ' + res.statusCode + ': ' + res.body);
+          var parsed = JSON.parse(res.body);
+          assert.ok(parsed.hasOwnProperty('layergroupid'),
+            "Missing 'layergroupid' from response body: " + res.body);
+          layergroupid = parsed.layergroupid;
+          redis_stats_client.zscore(statskey + ":global", now, this);
+        },
+        function check_tag_stats(err, val) {
+          if ( err ) throw err;
+          assert.equal(val, 1, "Expected score of " + now + " in "
+              +  statskey + ":global to be 1, got " + val);
+          redis_stats_client.zscore(statskey+':stat_tag:random_tag', now, this);
+        },
+        function check_tag_stats_value(err, val) {
+          if ( err ) throw err;
+          assert.equal(val, 1, "Expected score of " + now + " in "
+              +  statskey + ":stat_tag:" + layergroup.stat_tag + " to be 1, got " + val);
+          return null;
+        },
+        function deleteTemplate(err)
+        {
+          if ( err ) throw err;
+          var del_request = {
+              url: '/tiles/template/' + template_id + '?api_key=1234', 
+              method: 'DELETE',
+              headers: {host: 'localhost'}
+          }
+          var next = this;
+          assert.response(server, del_request, {},
+            function(res) { next(null, res); });
+        },
+        function cleanup_stats(err, res) {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 204, res.statusCode + ': ' + res.body);
+          if ( err ) errors.push('' + err);
+          redis_client.del([statskey+':global', statskey+':stat_tag:'+layergroup.stat_tag], this);
+        },
+        function finish(err) {
+          if ( err ) errors.push('' + err);
+          if ( errors.length ) done(new Error(errors.join(',')));
+          else done(null);
+        }
+      );
+    });
+
+    suiteTeardown(function(done) {
+
+        // This test will add map_style records, like
+        // 'map_style|null|publicuser|my_table',
+        redis_client.keys("map_*", function(err, keys) {
+            var todrop = _.map(keys, function(m) {
+              if ( m.match(/^map_(tpl|crt|sig)|/) ) return m;
+            });
+            redis_client.del(todrop, function(err) {
+              redis_client.select(5, function(err) {
+                redis_client.keys("user:localhost:mapviews*", function(err, keys) {
+                  redis_client.del(keys, function(err) {
+                    sqlapi_server.close(done);
+                  });
+                });
+              });
+            });
+        });
+
+    });
+    
+});
+

test/fixtures/test_multilayer_bbox.grid.json

@@ -0,0 +1 @@
+{"grid":["                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"],"keys":["","1"],"data":{"1":{"cartodb_id":1}}}

test/fixtures/test_table_0_0_0_multilayer1.layer0.grid.json

@@ -0,0 +1 @@
+{"grid":["                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                      !!!                       ","                                     !!!!!!                     ","                                    !!!!!!!                     ","                                    !!!!!!!!                    ","                                    !!!!!!!!                    ","                                    !!!!!!!!                    ","                                    !!!!!!!                     ","                                     !!!!!!                     ","                                      !!!                       ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                "],"keys":["","2"],"data":{"2":{"cartodb_id":2}}}

test/fixtures/test_table_0_0_0_multilayer1.layer1.grid.json

@@ -0,0 +1 @@
+{"grid":["                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                       !!                                       ","                      !!!                                       ","                       !!                                       ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                "],"keys":["","2"],"data":{"2":{"cartodb_id":4}}}

test/support/SQLAPIEmu.js

@@ -0,0 +1,38 @@
+var http        = require('http');
+var url         = require('url');
+
+var o = function(port, cb) {
+
+  this.queries = [];
+  var that = this;
+  this.sqlapi_server = http.createServer(function(req,res) {
+        var query = url.parse(req.url, true).query;
+        that.queries.push(query);
+        if ( query.q.match('SQLAPIERROR') ) {
+          res.statusCode = 400;
+          res.write(JSON.stringify({'error':'Some error occurred'}));
+        } else if ( query.q.match('EPOCH.* as max') ) {
+          // This is the structure of the known query sent by tiler
+          var row = {
+            'max': 1234567890.123
+          };
+          res.write(JSON.stringify({rows: [ row ]}));
+        } else {
+          var qs = JSON.stringify(query);
+          var row = {
+            // This is the structure of the known query sent by tiler
+            'cdb_querytables': '{' + qs + '}',
+            'max': qs
+          };
+          res.write(JSON.stringify({rows: [ row ]}));
+        }
+        res.end();
+   }).listen(port, cb);
+};
+
+o.prototype.close = function(cb) {
+  this.sqlapi_server.close(cb);
+};
+
+module.exports = o;
+

test/support/assert.js

@@ -8,29 +8,93 @@ var exec = require('child_process').exec;
 
 var assert = module.exports = exports = require('assert');
 
-assert.imageEqualsFile = function(buffer, file_b, callback) {
+// @param tolerance number of tolerated grid cell differences
+assert.utfgridEqualsFile = function(buffer, file_b, tolerance, callback) {
+    fs.writeFileSync('/tmp/grid.json', buffer, 'binary'); // <-- to debug/update
+    var expected_json = JSON.parse(fs.readFileSync(file_b, 'utf8'));
+
+    var err = null;
+
+    var Celldiff = function(x, y, ev, ov) {
+      this.x = x;
+      this.y = y;
+      this.ev = ev;
+      this.ov = ov;
+    };
+
+    Celldiff.prototype.toString = function() {
+      return '(' + this.x + ',' + this.y + ')["' + this.ev + '" != "' + this.ov + '"]';
+    };
+
+    try {
+      var obtained_json = JSON.parse(buffer);
+
+      // compare grid
+      var obtained_grid = obtained_json.grid;
+      var expected_grid = expected_json.grid;
+      var nrows = obtained_grid.length
+      if (nrows != expected_grid.length) {
+        throw new Error( "Obtained grid rows (" + nrows +
+                    ") != expected grid rows (" + expected_grid.length + ")" );
+      }
+      var celldiff = [];
+      for (var i=0; i<nrows; ++i) {
+        var ocols = obtained_grid[i];
+        var ecols = expected_grid[i];
+        var ncols = ocols.length;
+        if ( ncols != ecols.length ) {
+          throw new Error( "Obtained grid cols (" + ncols +
+                   ") != expected grid cols (" + ecols.length +
+                   ") on row " + i ); 
+        }
+        for (var j=0; j<ncols; ++j) {
+          var ocell = ocols[j];
+          var ecell = ecols[j];
+          if ( ocell !== ecell ) {
+            celldiff.push(new Celldiff(i, j, ecell, ocell));
+          }
+        }
+      }
+
+      if ( celldiff.length > tolerance ) {
+        throw new Error( celldiff.length + " cell differences: " + celldiff );
+      }
+
+      assert.deepEqual(obtained_json.keys, expected_json.keys);
+    } catch (e) { err = e; }
+
+    callback(err);
+};
+
+//
+// @param tol tolerated color distance as a percent over max channel value
+//            by default this is zero. For meaningful values, see
+//            http://www.imagemagick.org/script/command-line-options.php#metric
+//
+assert.imageEqualsFile = function(buffer, file_b, tol, callback) {
     if (!callback) callback = function(err) { if (err) throw err; };
     file_b = path.resolve(file_b);
-    var file_a = '/tmp/' + (Math.random() * 1e16);
+    var file_a = '/tmp/windshaft-test-image-test.png'; // + (Math.random() * 1e16); // TODO: make predictable 
     var err = fs.writeFileSync(file_a, buffer, 'binary');
     if (err) throw err;
 
-    exec('compare -metric PSNR "' + file_a + '" "' +
+    var fuzz = tol + '%';
+    exec('compare -fuzz ' + fuzz + ' -metric AE "' + file_a + '" "' +
             file_b + '" /dev/null', function(err, stdout, stderr) {
         if (err) {
             fs.unlinkSync(file_a);
             callback(err);
         } else {
             stderr = stderr.trim();
-            if (stderr === 'inf') {
-                fs.unlinkSync(file_a);
-                callback(null);
+            var similarity = parseFloat(stderr);
+            if ( similarity > 0 ) {
+              var err = new Error('Images not equal(' + similarity + '): ' +
+                      file_a  + '    ' + file_b);
+              err.similarity = similarity;
+              callback(err);
             } else {
-                var similarity = parseFloat(stderr);
-                var err = new Error('Images not equal(' + similarity + '): ' +
-                        file_a  + '    ' + file_b);
-                err.similarity = similarity;
-                callback(err);
+              fs.unlinkSync(file_a);
+              callback(null);
             }
         }
     });

test/support/config.js

@@ -1,5 +1,8 @@
 var _ = require('underscore');
 
+
+require(__dirname + '/test_helper');
+
 module.exports = function(opts) {
 
     var config = {
@@ -7,7 +10,7 @@ module.exports = function(opts) {
             max: 10, 
             idleTimeoutMillis: 1, 
             reapIntervalMillis: 1,
-            port: 6333 // TODO: read from test env ?
+            port: global.environment.redis.port
         }
     }
 

test/support/prepare_db.sh

@@ -10,27 +10,102 @@
 #        TODO: fix that
 #
 
+PREPARE_REDIS=yes
+PREPARE_PGSQL=yes
+
+while [ -n "$1" ]; do
+  if test "$1" = "--skip-pg"; then
+    PREPARE_PGSQL=no
+    shift; continue
+  elif test "$1" = "--skip-redis"; then
+    PREPARE_REDIS=no
+    shift; continue
+  fi
+done
+
 die() {
         msg=$1
         echo "${msg}" >&2
         exit 1
 }
 
-TEST_DB="cartodb_test_user_1_db"
-REDIS_PORT=6333
+# This is where postgresql connection parameters are read from
+TESTENV=../../config/environments/test.js
+if [ \! -r ${TESTENV} ]; then
+  echo "Cannot read ${TESTENV}" >&2
+  exit 1
+fi
 
-echo "preparing postgres..."
-dropdb "${TEST_DB}"
-createdb -Ttemplate_postgis -EUTF8 "${TEST_DB}" || die "Could not create test database"
-psql "${TEST_DB}" < ./sql/windshaft.test.sql
-psql "${TEST_DB}" < ./sql/gadm4.sql
+TESTUSERID=1
 
-echo "preparing redis..."
-echo "HSET rails:users:localhost id 1" | redis-cli -p ${REDIS_PORT} -n 5
-echo 'HSET rails:users:localhost database_name "'"${TEST_DB}"'"' | redis-cli -p ${REDIS_PORT} -n 5
-echo "HSET rails:users:localhost map_key 1234" | redis-cli -p ${REDIS_PORT} -n 5
-echo "SADD rails:users:localhost:map_key 1235" | redis-cli -p ${REDIS_PORT} -n 5
-echo 'HSET rails:'"${TEST_DB}"':my_table infowindow "this, that, the other"' | redis-cli -p ${REDIS_PORT} -n 0
-echo 'HSET rails:'"${TEST_DB}"':test_table_private_1 privacy "0"' | redis-cli -p ${REDIS_PORT} -n 0
+TESTUSER=`node -e "console.log(require('${TESTENV}').postgres_auth_user || '')"`
+if test -z "$TESTUSER"; then
+  echo "Missing postgres_auth_user from ${TESTENV}" >&2
+  exit 1
+fi
+TESTUSER=`echo ${TESTUSER} | sed "s/<%= user_id %>/${TESTUSERID}/"`
+
+TESTPASS=`node -e "console.log(require('${TESTENV}').postgres_auth_pass || 'test')"`
+# TODO: should postgres_auth_pass be optional ?
+if test -z "$TESTPASS"; then
+  echo "Missing postgres_auth_pass from ${TESTENV}" >&2
+  exit 1
+fi
+TESTPASS=`echo ${TESTPASS} | sed "s/<%= user_id %>/${TESTUSERID}/"`
+
+TEST_DB="${TESTUSER}_db"
+
+# NOTE: will be set by caller trough environment
+if test -z "$REDIS_PORT"; then REDIS_PORT=6333; fi
+
+PUBLICUSER=`node -e "console.log(require('${TESTENV}').postgres.user || 'xxx')"`
+PUBLICPASS=`node -e "console.log(require('${TESTENV}').postgres.password || 'xxx')"`
+echo "PUBLICUSER: ${PUBLICUSER}"
+echo "PUBLICPASS: ${PUBLICPASS}"
+echo "TESTUSER: ${TESTUSER}"
+echo "TESTPASS: ${TESTPASS}"
+
+if test x"$PREPARE_PGSQL" = xyes; then
+
+  echo "preparing postgres..."
+  dropdb "${TEST_DB}"
+  createdb -Ttemplate_postgis -EUTF8 "${TEST_DB}" || die "Could not create test database"
+
+  cat sql/windshaft.test.sql sql/gadm4.sql |
+    sed "s/:PUBLICUSER/${PUBLICUSER}/" | 
+    sed "s/:PUBLICPASS/${PUBLICPASS}/" | 
+    sed "s/:TESTUSER/${TESTUSER}/" | 
+    sed "s/:TESTPASS/${TESTPASS}/" | 
+    psql ${TEST_DB}
+
+fi
+
+if test x"$PREPARE_REDIS" = xyes; then
+
+  echo "preparing redis..."
+
+  cat <<EOF | redis-cli -p ${REDIS_PORT} -n 5
+HMSET rails:users:localhost id ${TESTUSERID} \
+                            database_name '${TEST_DB}' \
+                            map_key 1234
+SADD rails:users:localhost:map_key 1235
+EOF
+
+  # A user configured as with cartodb-2.5.0+ 
+  cat <<EOF | redis-cli -p ${REDIS_PORT} -n 5
+HMSET rails:users:cartodb250user id ${TESTUSERID} \
+                                 database_name "${TEST_DB}" \
+                                 database_host "localhost" \
+                                 database_password "${TESTPASS}" \
+                                 map_key 4321
+EOF
+
+  cat <<EOF | redis-cli -p ${REDIS_PORT} -n 0
+HSET rails:${TEST_DB}:my_table infowindow "this, that, the other"
+HSET rails:${TEST_DB}:test_table_private_1 privacy "0"
+EOF
+
+fi
+
+echo "Finished preparing data. Ready to run tests"
 
-echo "Finished preparing data. Run tests with expresso."

test/support/sql/windshaft.test.sql

@@ -15,11 +15,13 @@ SET search_path = public, pg_catalog;
 SET default_tablespace = '';
 SET default_with_oids = false;
 
--- publicuser role
-CREATE USER publicuser;
+-- public user role
+DROP USER IF EXISTS :PUBLICUSER;
+CREATE USER :PUBLICUSER WITH PASSWORD ':PUBLICPASS';
 
 -- db owner role
-CREATE USER test_cartodb_user_1;
+DROP USER IF EXISTS :TESTUSER;
+CREATE USER :TESTUSER WITH PASSWORD ':TESTPASS';
 
 -- first table
 CREATE TABLE test_table (
@@ -51,19 +53,20 @@ SELECT pg_catalog.setval('test_table_cartodb_id_seq', 60, true);
 
 ALTER TABLE test_table ALTER COLUMN cartodb_id SET DEFAULT nextval('test_table_cartodb_id_seq'::regclass);
 
-INSERT INTO test_table VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241');
-INSERT INTO test_table VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241');
-INSERT INTO test_table VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.324', 3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241');
-INSERT INTO test_table VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.329509', 4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241');
-INSERT INTO test_table VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.334931', 5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241');
+INSERT INTO test_table VALUES
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.324', 3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.329509', 4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.334931', 5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241');
 
 ALTER TABLE ONLY test_table ADD CONSTRAINT test_table_pkey PRIMARY KEY (cartodb_id);
 
 CREATE INDEX test_table_the_geom_idx ON test_table USING gist (the_geom);
 CREATE INDEX test_table_the_geom_webmercator_idx ON test_table USING gist (the_geom_webmercator);
 
-GRANT ALL ON TABLE test_table TO test_cartodb_user_1;
-GRANT SELECT ON TABLE test_table TO publicuser;
+GRANT ALL ON TABLE test_table TO :TESTUSER;
+GRANT SELECT ON TABLE test_table TO :PUBLICUSER;
 
 -- second table
 CREATE TABLE test_table_2 (
@@ -95,19 +98,20 @@ SELECT pg_catalog.setval('test_table_2_cartodb_id_seq', 60, true);
 
 ALTER TABLE test_table_2 ALTER COLUMN cartodb_id SET DEFAULT nextval('test_table_2_cartodb_id_seq'::regclass);
 
-INSERT INTO test_table_2 VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241');
-INSERT INTO test_table_2 VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241');
-INSERT INTO test_table_2 VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.324', 3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241');
-INSERT INTO test_table_2 VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.329509', 4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241');
-INSERT INTO test_table_2 VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.334931', 5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241');
+INSERT INTO test_table_2 VALUES
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.324', 3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.329509', 4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.334931', 5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241');
 
 ALTER TABLE ONLY test_table_2 ADD CONSTRAINT test_table_2_pkey PRIMARY KEY (cartodb_id);
 
 CREATE INDEX test_table_2_the_geom_idx ON test_table_2 USING gist (the_geom);
 CREATE INDEX test_table_2_the_geom_webmercator_idx ON test_table_2 USING gist (the_geom_webmercator);
 
-GRANT ALL ON TABLE test_table_2 TO test_cartodb_user_1;
-GRANT SELECT ON TABLE test_table_2 TO publicuser;
+GRANT ALL ON TABLE test_table_2 TO :TESTUSER;
+GRANT SELECT ON TABLE test_table_2 TO :PUBLICUSER;
 
 -- third table
 CREATE TABLE test_table_3 (
@@ -139,19 +143,20 @@ SELECT pg_catalog.setval('test_table_3_cartodb_id_seq', 60, true);
 
 ALTER TABLE test_table_3 ALTER COLUMN cartodb_id SET DEFAULT nextval('test_table_3_cartodb_id_seq'::regclass);
 
-INSERT INTO test_table_3 VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241');
-INSERT INTO test_table_3 VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241');
-INSERT INTO test_table_3 VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.324', 3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241');
-INSERT INTO test_table_3 VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.329509', 4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241');
-INSERT INTO test_table_3 VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.334931', 5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241');
+INSERT INTO test_table_3 VALUES
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.324', 3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.329509', 4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.334931', 5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241');
 
 ALTER TABLE ONLY test_table_3 ADD CONSTRAINT test_table_3_pkey PRIMARY KEY (cartodb_id);
 
 CREATE INDEX test_table_3_the_geom_idx ON test_table_3 USING gist (the_geom);
 CREATE INDEX test_table_3_the_geom_webmercator_idx ON test_table_3 USING gist (the_geom_webmercator);
 
-GRANT ALL ON TABLE test_table_3 TO test_cartodb_user_1;
-GRANT SELECT ON TABLE test_table_3 TO publicuser;
+GRANT ALL ON TABLE test_table_3 TO :TESTUSER;
+GRANT SELECT ON TABLE test_table_3 TO :PUBLICUSER;
 
 -- private table
 CREATE TABLE test_table_private_1 (
@@ -170,4 +175,4 @@ CREATE TABLE test_table_private_1 (
     CONSTRAINT enforce_srid_the_geom_webmercator CHECK ((st_srid(the_geom_webmercator) = 3857))
 );
 
-GRANT ALL ON TABLE test_table_private_1 TO test_cartodb_user_1;
+GRANT ALL ON TABLE test_table_private_1 TO :TESTUSER;

test/support/test_helper.js

@@ -6,6 +6,7 @@
  */
 
 var _ = require('underscore');
+var LZMA  = require('lzma/lzma_worker.js').LZMA;
 
 // set environment specific variables
 global.settings     = require(__dirname + '/../../config/settings');
@@ -13,4 +14,21 @@ global.environment  = require(__dirname + '/../../config/environments/test');
 _.extend(global.settings, global.environment);
 
 
+// Utility function to compress & encode LZMA
+function lzma_compress_to_base64(payload, mode, callback) {
+  LZMA.compress(payload, mode, 
+    function(ints) {
+      ints = ints.map(function(c) { return String.fromCharCode(c + 128) }).join('')
+      var base64 = new Buffer(ints, 'binary').toString('base64');
+      callback(null, base64);
+    },
+    function(percent) {
+      //console.log("Compressing: " + percent + "%");
+    }
+  );
+}
+
+module.exports = {
+  lzma_compress_to_base64: lzma_compress_to_base64
+}
 

test/unit/cartodb/redis_pool.test.js

@@ -1,61 +0,0 @@
-var assert = require('../../support/assert')
-  , _ = require('underscore')
-  , RedisPool = require('../../../lib/cartodb/redis_pool')
-  , tests = module.exports = {};
-
-suite('redis_pool', function() {
-
-    // configure redis pool instance to use in tests
-    var test_opts = require('../../support/config').redis_pool;
-    
-    var redis_pool = new RedisPool(test_opts);
-
-    test('RedisPool object exists', function(done){
-      assert.ok(RedisPool);
-      done();
-    });
-    
-    test('RedisPool can create new redis_pool objects with default settings', function(done){
-      var redis_pool = new RedisPool();
-      done();
-    });
-    
-    test('RedisPool can create new redis_pool objects with specific settings', function(done){
-      var redis_pool = new RedisPool(_.extend({host:'127.0.0.1', port: '6379'}, test_opts));
-      done();
-    });
-    
-    
-    test('pool object has an acquire function', function(done){
-      var found=false;
-      var functions = _.functions(redis_pool);
-      for (var i=0; i<functions.length; ++i) {
-          if ( functions[i] == 'acquire' ) { found=true; break; }
-      }
-      assert.ok(found);
-      done();
-    });
-    
-    test('calling aquire returns a redis client object that can get/set', function(done){
-      redis_pool.acquire(0, function(err, client){
-        client.set("key","value");
-        client.get("key", function(err,data){      
-          assert.equal(data, "value");      
-          redis_pool.release(0, client); // needed to exit tests
-          done();
-        })
-      });    
-    });
-    
-    test('calling aquire on another DB returns a redis client object that can get/set', function(done){
-      redis_pool.acquire(2, function(err, client){
-        client.set("key","value");
-        client.get("key", function(err,data){      
-          assert.equal(data, "value");      
-          redis_pool.release(2, client); // needed to exit tests
-          done();
-        })
-      });      
-    });
-
-});

test/unit/cartodb/req2params.test.js

@@ -15,27 +15,27 @@ suite('req2params', function() {
 
     test('cleans up request', function(done){
       opts.req2params({headers: { host:'localhost' }, query: {dbuser:'hacker',dbname:'secret'}}, function(err, req) {
-          if ( err ) { console.log(err); throw new Error(err); }
+          if ( err ) { done(err); return; }
           assert.ok(_.isObject(req.query), 'request has query');
           assert.ok(!req.query.hasOwnProperty('dbuser'), 'dbuser was removed from query');
           assert.ok(req.hasOwnProperty('params'), 'request has params');
           assert.ok(req.params.hasOwnProperty('interactivity'), 'request params have interactivity');
-          assert.equal(req.params.dbname, 'cartodb_test_user_1_db', 'could forge dbname: '+ req.params.dbname);
-          assert.ok(!req.hasOwnProperty('dbuser'), 'could inject dbuser ('+req.params.dbuser+')');
+          assert.equal(req.params.dbname, 'test_cartodb_user_1_db', 'could forge dbname: '+ req.params.dbname);
+          assert.ok(!req.params.hasOwnProperty('dbuser'), 'could inject dbuser ('+req.params.dbuser+')');
           done();
       });
     });
 
     test('sets dbname from redis metadata', function(done){
       opts.req2params({headers: { host:'localhost' }, query: {} }, function(err, req) {
-          if ( err ) { console.log(err); throw new Error(err); }
+          if ( err ) { done(err); return; }
           //console.dir(req);
           assert.ok(_.isObject(req.query), 'request has query');
           assert.ok(!req.query.hasOwnProperty('dbuser'), 'dbuser was removed from query');
           assert.ok(req.hasOwnProperty('params'), 'request has params');
           assert.ok(req.params.hasOwnProperty('interactivity'), 'request params have interactivity');
           // database_name for user "localhost" (see test/support/prepare_db.sh)
-          assert.equal(req.params.dbname, 'cartodb_test_user_1_db');
+          assert.equal(req.params.dbname, 'test_cartodb_user_1_db');
           // unauthenticated request gets no dbuser
           assert.ok(!req.params.hasOwnProperty('dbuser'), 'could inject dbuser ('+req.params.dbuser+')');
           done();
@@ -44,23 +44,43 @@ suite('req2params', function() {
 
     test('sets also dbuser for authenticated requests', function(done){
       opts.req2params({headers: { host:'localhost' }, query: {map_key: '1234'} }, function(err, req) {
-          if ( err ) { console.log(err); throw new Error(err); }
+          if ( err ) { done(err); return; }
           //console.dir(req);
           assert.ok(_.isObject(req.query), 'request has query');
           assert.ok(!req.query.hasOwnProperty('dbuser'), 'dbuser was removed from query');
           assert.ok(req.hasOwnProperty('params'), 'request has params');
           assert.ok(req.params.hasOwnProperty('interactivity'), 'request params have interactivity');
           // database_name for user "localhost" (see test/support/prepare_db.sh)
-          assert.equal(req.params.dbname, 'cartodb_test_user_1_db');
+          assert.equal(req.params.dbname, 'test_cartodb_user_1_db');
           // id for user "localhost" (see test/support/prepare_db.sh)
-          assert.equal(req.dbuser, 'test_cartodb_user_1');
+          assert.equal(req.params.dbuser, 'test_cartodb_user_1');
  
           opts.req2params({headers: { host:'localhost' }, query: {map_key: '1235'} }, function(err, req) {
               // wrong key resets params to no user
-              assert.ok(!req.hasOwnProperty('dbuser'), 'could inject dbuser ('+req.params.dbuser+')');
+              assert.ok(!req.params.hasOwnProperty('dbuser'), 'could inject dbuser ('+req.params.dbuser+')');
               done();
           });
       });
     });
+
+    test('it should extend params with decoded lzma', function(done) {
+      var qo = {
+        style: 'test',
+        style_version: '2.1.0',
+        cache_buster: 5
+      };
+      test_helper.lzma_compress_to_base64(JSON.stringify(qo), 1, function(err, data) {
+        opts.req2params({ headers: { host:'localhost' }, query: { non_included: 'toberemoved', api_key: 'test', style: 'override', lzma: data }}, function(err, req) {
+          if ( err ) { done(err); return; }
+          var query = req.params
+          assert.equal(qo.style, query.style)
+          assert.equal(qo.style_version, query.style_version)
+          assert.equal(qo.cache_buster, query.cache_buster)
+          assert.equal('test', query.api_key)
+          assert.equal(undefined, query.non_included)
+          done();
+        });
+      });
+    });
     
 });

test/unit/cartodb/signed_maps.test.js

@@ -0,0 +1,85 @@
+var assert = require('assert')
+  //, _ = require('underscore')
+  , RedisPool = require('redis-mpool')
+  , SignedMaps = require('../../../lib/cartodb/signed_maps.js')
+  , test_helper = require('../../support/test_helper')
+  , Step = require('step')
+  , tests = module.exports = {};
+
+suite('signed_maps', function() {
+
+    // configure redis pool instance to use in tests
+    var redis_pool = RedisPool(global.environment.redis);
+    
+    test('can sign map with open and token-based auth', function(done) {
+      var smap = new SignedMaps(redis_pool);
+      assert.ok(smap);
+      var sig = 'sig1';
+      var map = 'map1';
+      var tok = 'tok1';
+      var crt = {
+        version:'0.0.1',
+        layergroup_id:map,
+        auth: {}
+      };
+      var crt1_id; // by token
+      var crt2_id; // open
+      Step(
+        function() {
+          smap.isAuthorized(sig,map,tok,this);
+        },
+        function checkAuthFailure1(err, authorized) {
+          if ( err ) throw err;
+          assert.ok(!authorized, "unexpectedly authorized");
+          crt.auth.method = 'token';
+          crt.auth.valid_tokens = [tok];
+          smap.addSignature(sig, map, crt, this)
+        },
+        function getCert1(err, id) {
+          if ( err ) throw err;
+          assert.ok(id, "undefined signature id");
+          crt1_id = id; // keep note of it
+//console.log("Certificate 1 is " + crt1_id);
+          smap.isAuthorized(sig,map,'',this);
+        },
+        function checkAuthFailure2(err, authorized) {
+          if ( err ) throw err;
+          assert.ok(!authorized, "unexpectedly authorized");
+          smap.isAuthorized(sig,map,tok,this);
+        },
+        function checkAuthSuccess1(err, authorized) {
+          if ( err ) throw err;
+          assert.ok(authorized, "unauthorized :(");
+          crt.auth.method = 'open';
+          delete crt.auth.valid_tokens;
+          smap.addSignature(sig, map, crt, this)
+        },
+        function getCert2(err, id) {
+          if ( err ) throw err;
+          assert.ok(id, "undefined signature id");
+          crt2_id = id; // keep note of it
+//console.log("Certificate 2 is " + crt2_id);
+          smap.isAuthorized(sig,map,'arbitrary',this);
+        },
+        function checkAuthSuccess2_delCert2(err, authorized) {
+          if ( err ) throw err;
+          assert.ok(authorized, "unauthorized :(");
+          var next = this;
+          smap.delCertificate(sig, crt2_id, function(e) {
+            if (e) next(e);
+            else smap.isAuthorized(sig,map,'arbitrary',next);
+          });
+        },
+        function checkAuthFailure3_delCert2(err, authorized) {
+          if ( err ) throw err;
+          assert.ok(!authorized, "unexpectedly authorized");
+          smap.delCertificate(sig, crt1_id, this);
+        },
+        function finish(err) {
+          done(err);
+        }
+      );
+    });
+
+    
+});

test/unit/cartodb/template_maps.test.js

@@ -0,0 +1,400 @@
+var assert = require('assert')
+  //, _ = require('underscore')
+  , RedisPool = require('redis-mpool')
+  , SignedMaps = require('../../../lib/cartodb/signed_maps.js')
+  , TemplateMaps = require('../../../lib/cartodb/template_maps.js')
+  , test_helper = require('../../support/test_helper')
+  , Step = require('step')
+  , tests = module.exports = {};
+
+suite('template_maps', function() {
+
+  // configure redis pool instance to use in tests
+  var redis_pool = RedisPool(global.environment.redis);
+  var signed_maps = new SignedMaps(redis_pool);
+    
+  test('does not accept template with unsupported version', function(done) {
+    var tmap = new TemplateMaps(redis_pool, signed_maps);
+    assert.ok(tmap);
+    var tpl = { version:'6.6.6',
+      name:'k', auth: {}, layergroup: {} };
+    Step(
+      function() {
+        tmap.addTemplate('me', tpl, this);
+      },
+      function checkFailed(err) {
+        assert.ok(err);
+        assert.ok(err.message.match(/unsupported.*version/i), err);
+        return null;
+      },
+      function finish(err) {
+        done(err);
+      }
+    );
+  });
+
+  test('does not accept template with missing name', function(done) {
+    var tmap = new TemplateMaps(redis_pool, signed_maps);
+    assert.ok(tmap);
+    var tpl = { version:'0.0.1',
+      auth: {}, layergroup: {} };
+    Step(
+      function() {
+        tmap.addTemplate('me', tpl, this);
+      },
+      function checkFailed(err) {
+        assert.ok(err);
+        assert.ok(err.message.match(/missing.*name/i), err);
+        return null;
+      },
+      function finish(err) {
+        done(err);
+      }
+    );
+  });
+
+  test('does not accept template with invalid name', function(done) {
+    var tmap = new TemplateMaps(redis_pool, signed_maps);
+    assert.ok(tmap);
+    var tpl = { version:'0.0.1',
+      auth: {}, layergroup: {} };
+    var invalidnames = [ "ab|", "a b", "a@b", "1ab", "_x", "", " x", "x " ];
+    var testNext = function() {
+      if ( ! invalidnames.length ) { done(); return; }
+      var n = invalidnames.pop();
+      tpl.name = n;
+      tmap.addTemplate('me', tpl, function(err) {
+        if ( ! err ) {
+          done(new Error("Unexpected success with invalid name '" + n + "'"));
+        }
+        else if ( ! err.message.match(/template.*name/i) ) {
+          done(new Error("Unexpected error message with invalid name '" + n
+            + "': " + err));
+        }
+        else {
+          testNext();
+        }
+      });
+    };
+    testNext();
+  });
+
+  test('does not accept template with invalid placeholder name', function(done) {
+    var tmap = new TemplateMaps(redis_pool, signed_maps);
+    assert.ok(tmap);
+    var tpl = { version:'0.0.1',
+      name: "valid", placeholders: {},
+      auth: {}, layergroup: {} };
+    var invalidnames = [ "ab|", "a b", "a@b", "1ab", "_x", "", " x", "x " ];
+    var testNext = function() {
+      if ( ! invalidnames.length ) { done(); return; }
+      var n = invalidnames.pop();
+      tpl.placeholders = {};
+      tpl.placeholders[n] = { type:'number', default:1 };
+      tmap.addTemplate('me', tpl, function(err) {
+        if ( ! err ) {
+          done(new Error("Unexpected success with invalid name '" + n + "'"));
+        }
+        else if ( ! err.message.match(/invalid.*name/i) ) {
+          done(new Error("Unexpected error message with invalid name '" + n
+            + "': " + err));
+        }
+        else {
+          testNext();
+        }
+      });
+    };
+    testNext();
+  });
+
+  test('does not accept template with missing placeholder default', function(done) {
+    var tmap = new TemplateMaps(redis_pool, signed_maps);
+    assert.ok(tmap);
+    var tpl = { version:'0.0.1',
+      name: "valid", placeholders: { v: {} },
+      auth: {}, layergroup: {} };
+    tmap.addTemplate('me', tpl, function(err) {
+        if ( ! err ) {
+          done(new Error("Unexpected success with missing placeholder default"));
+        }
+        else if ( ! err.message.match(/missing default/i) ) {
+          done(new Error("Unexpected error message with missing placeholder default: "
+            + err));
+        }
+        else {
+          done();
+        }
+      });
+  });
+
+  test('does not accept template with missing placeholder type', function(done) {
+    var tmap = new TemplateMaps(redis_pool, signed_maps);
+    assert.ok(tmap);
+    var tpl = { version:'0.0.1',
+      name: "valid", placeholders: { v: { default:1 } },
+      auth: {}, layergroup: {} };
+    tmap.addTemplate('me', tpl, function(err) {
+        if ( ! err ) {
+          done(new Error("Unexpected success with missing placeholder type"));
+        }
+        else if ( ! err.message.match(/missing type/i) ) {
+          done(new Error("Unexpected error message with missing placeholder default: "
+            + err));
+        }
+        else {
+          done();
+        }
+      });
+  });
+
+  test('add, get and delete a valid template', function(done) {
+    var tmap = new TemplateMaps(redis_pool, signed_maps);
+    assert.ok(tmap);
+    var expected_failure = false;
+    var tpl_id;
+    var tpl = { version:'0.0.1',
+      name: 'first', auth: {}, layergroup: {} };
+    Step(
+      function() {
+        tmap.addTemplate('me', tpl, this);
+      },
+      function addOmonimousTemplate(err, id) {
+        if ( err ) throw err;
+        tpl_id = id;
+        assert.equal(tpl_id, 'first');
+        expected_failure = true;
+        // should fail, as it already exists
+        tmap.addTemplate('me', tpl, this);
+      },
+      function getTemplate(err) {
+        if ( ! expected_failure && err ) throw err;
+        assert.ok(err);
+        assert.ok(err.message.match(/already exists/i), err);
+        tmap.getTemplate('me', tpl_id, this);
+      },
+      function delTemplate(err, got_tpl) {
+        if ( err ) throw err;
+        assert.deepEqual(got_tpl, tpl);
+        tmap.delTemplate('me', tpl_id, this);
+      },
+      function finish(err) {
+        done(err);
+      }
+    );
+  });
+
+  test('add multiple templates, list them', function(done) {
+    var tmap = new TemplateMaps(redis_pool, signed_maps);
+    assert.ok(tmap);
+    var expected_failure = false;
+    var tpl1 = { version:'0.0.1', name: 'first', auth: {}, layergroup: {} };
+    var tpl1_id;
+    var tpl2 = { version:'0.0.1', name: 'second', auth: {}, layergroup: {} };
+    var tpl2_id;
+    Step(
+      function addTemplate1() {
+        tmap.addTemplate('me', tpl1, this);
+      },
+      function addTemplate2(err, id) {
+        if ( err ) throw err;
+        tpl1_id = id;
+        tmap.addTemplate('me', tpl2, this);
+      },
+      function listTemplates(err, id) {
+        if ( err ) throw err;
+        tpl2_id = id;
+        tmap.listTemplates('me', this);
+      },
+      function checkTemplates(err, ids) {
+        if ( err ) throw err;
+        assert.equal(ids.length, 2);
+        assert.ok(ids.indexOf(tpl1_id) != -1, ids.join(','));
+        assert.ok(ids.indexOf(tpl2_id) != -1, ids.join(','));
+        return null;
+      },
+      function delTemplate1(err) {
+        if ( tpl1_id ) {
+          var next = this;
+          tmap.delTemplate('me', tpl1_id, function(e) {
+            if ( err || e ) next(new Error(err + '; ' + e));
+            else next();
+          });
+        } else {
+          if ( err ) throw err;
+          return null;
+        }
+      },
+      function delTemplate2(err) {
+        if ( tpl2_id ) {
+          var next = this;
+          tmap.delTemplate('me', tpl2_id, function(e) {
+            if ( err || e ) next(new Error(err + '; ' + e));
+            else next();
+          });
+        } else {
+          if ( err ) throw err;
+          return null;
+        }
+      },
+      function finish(err) {
+        done(err);
+      }
+    );
+  });
+
+  test('update templates', function(done) {
+    var tmap = new TemplateMaps(redis_pool, signed_maps);
+    assert.ok(tmap);
+    var expected_failure = false;
+    var owner = 'me';
+    var tpl = { version:'0.0.1',
+      name: 'first',
+      auth: { method: 'open' },
+      layergroup: {}
+    };
+    var tpl_id;
+    Step(
+      function addTemplate() {
+        tmap.addTemplate(owner, tpl, this);
+      },
+      // Updating template name should fail
+      function updateTemplateName(err, id) {
+        if ( err ) throw err;
+        tpl_id = id;
+        expected_failure = true;
+        tpl.name = 'second';
+        tmap.updTemplate(owner, tpl_id, tpl, this);
+      },
+      function updateTemplateAuth(err) {
+        if ( err && ! expected_failure) throw err;
+        expected_failure = false;
+        assert.ok(err);
+        tpl.name = 'first';
+        tpl.auth.method = 'token';
+        tmap.updTemplate(owner, tpl_id, tpl, this);
+      },
+      function updateTemplateWithInvalid(err) {
+        if ( err ) throw err;
+        tpl.version = '999.999.999';
+        expected_failure = true;
+        tmap.updTemplate(owner, tpl_id, tpl, this);
+      },
+      function updateUnexistentTemplate(err) {
+        if ( err && ! expected_failure) throw err;
+        expected_failure = false;
+        assert.ok(err);
+        assert.ok(err.message.match(/unsupported.*version/i), err);
+        tpl.version = '0.0.1';
+        expected_failure = true;
+        tmap.updTemplate(owner, 'unexistent', tpl, this);
+      },
+      function delTemplate(err) {
+        if ( err && ! expected_failure) throw err;
+        expected_failure = false;
+        assert.ok(err);
+        assert.ok(err.message.match(/cannot update name/i), err);
+        tmap.delTemplate(owner, tpl_id, this);
+      },
+      function finish(err) {
+        done(err);
+      }
+    );
+  });
+
+  test('instanciate templates', function() {
+    var tmap = new TemplateMaps(redis_pool, signed_maps);
+    assert.ok(tmap);
+
+    var tpl1 =  {
+        version: '0.0.1',
+        name: 'acceptance1',
+        auth: { method: 'open' },
+        placeholders: {
+          fill: { type: "css_color", default: "red" },
+          color: { type: "css_color", default: "#a0fF9A" },
+          name: { type: "sql_literal", default: "test" },
+          zoom: { type: "number", default: "0" },
+        },
+        layergroup: {
+          version: '1.0.0',
+          global_cartocss_version: '2.0.2',
+          layers: [
+             { options: {
+                 sql: "select '<%= name %>' || id, g from t",
+                 cartocss: '#layer { marker-fill:<%= fill %>; }'
+               } },
+             { options: {
+                 sql: "select fun('<%= name %>') g from x",
+                 cartocss: '#layer { line-color:<%= color %>; marker-fill:<%= color %>; }'
+               } },
+             { options: {
+                 sql: "select g from x",
+                 cartocss: '#layer[zoom=<%=zoom%>] { }'
+               } }
+          ]
+        }
+    };
+
+    var inst = tmap.instance(tpl1, {});
+
+    var lyr = inst.layers[0].options;
+    assert.equal(lyr.sql, "select 'test' || id, g from t");
+    assert.equal(lyr.cartocss, '#layer { marker-fill:red; }');
+
+    lyr = inst.layers[1].options;
+    assert.equal(lyr.sql, "select fun('test') g from x");
+    assert.equal(lyr.cartocss, '#layer { line-color:#a0fF9A; marker-fill:#a0fF9A; }');
+
+    inst = tmap.instance(tpl1, {color:'yellow', name:"it's dangerous"});
+
+    lyr = inst.layers[0].options;
+    assert.equal(lyr.sql, "select 'it''s dangerous' || id, g from t");
+    assert.equal(lyr.cartocss, '#layer { marker-fill:red; }');
+
+    lyr = inst.layers[1].options;
+    assert.equal(lyr.sql, "select fun('it''s dangerous') g from x");
+    assert.equal(lyr.cartocss, '#layer { line-color:yellow; marker-fill:yellow; }');
+
+    // Invalid css_color
+    var err = null;
+    try { inst = tmap.instance(tpl1, {color:'##ff00ff'}); }
+    catch (e) { err = e; }
+    assert.ok(err);
+    assert.ok(err.message.match(/invalid css_color/i), err);
+
+    // Invalid css_color 2 (too few digits)
+    var err = null;
+    try { inst = tmap.instance(tpl1, {color:'#ff'}); }
+    catch (e) { err = e; }
+    assert.ok(err);
+    assert.ok(err.message.match(/invalid css_color/i), err);
+
+    // Invalid css_color 3 (too many digits)
+    var err = null;
+    try { inst = tmap.instance(tpl1, {color:'#1234567'}); }
+    catch (e) { err = e; }
+    assert.ok(err);
+    assert.ok(err.message.match(/invalid css_color/i), err);
+
+    // Invalid number
+    var err = null;
+    try { inst = tmap.instance(tpl1, {zoom:'#'}); }
+    catch (e) { err = e; }
+    assert.ok(err);
+    assert.ok(err.message.match(/invalid number/i), err);
+
+    // Invalid number 2
+    var err = null;
+    try { inst = tmap.instance(tpl1, {zoom:'23e'}); }
+    catch (e) { err = e; }
+    assert.ok(err);
+    assert.ok(err.message.match(/invalid number/i), err);
+
+    // Valid number
+    var err = null;
+    try { inst = tmap.instance(tpl1, {zoom:'-.23e10'}); }
+    catch (e) { err = e; }
+    assert.ok(!err);
+  });
+    
+});

tools/convert_database_styles

@@ -0,0 +1,81 @@
+#!/usr/bin/env node
+
+var path = require('path');
+
+var grainstore = require('../node_modules/windshaft/node_modules/grainstore');
+var mapnik = require('mapnik');
+var redis = require('redis');
+
+function usage(me, exitcode) {
+  console.log("Usage: " + me + " <database_name> <table_name> [<target_mapnik_version>]");
+  process.exit(exitcode);
+}
+
+var node_path = process.argv.shift(); 
+var script_path = process.argv.shift(); 
+var me = path.basename(script_path);
+
+var database_name = process.argv.shift()
+var table_name = process.argv.shift()
+var MAPNIK_VERSION = process.argv.shift()
+
+
+if ( ! MAPNIK_VERSION ) {
+  MAPNIK_VERSION = mapnik.versions.mapnik;
+}
+
+if ( ! database_name || ! table_name) {
+  usage(me, 1);
+}
+
+var REDIS_PORT = 6379; // TODO: make a command line parameter
+
+var dbnum = 0;
+
+var mml_store = new grainstore.MMLStore({port:REDIS_PORT}, {mapnik_version:MAPNIK_VERSION});
+
+var failures = [];
+
+var client = redis.createClient(REDIS_PORT, 'localhost');
+client.on('connect', function() {
+  client.select(dbnum);
+  client.keys('map_style|' + database_name + '|' + table_name, function(err, matches) {
+
+    processNext = function() {
+      if ( ! matches.length ) process.exit(failures.length);
+      var k = matches.shift();
+
+      if ( /map_style\|.*\|.*\|/.test(k) ) {
+        //console.warn("Key " + k + " is EXTENDED, skipping");
+        processNext();
+      } 
+
+      var out = 'map_style|' + database_name + '|' + table_name + ': ';
+
+      var mml_builder = mml_store.mml_builder({
+        dbname:database_name, 
+        table:table_name},
+        function(err, payload) {
+          if ( err ) { 
+            console.warn(out + err.message); 
+            failures.push(k); processNext(); 
+          }
+          else {
+            mml_builder.resetStyle(function(err, data) {
+              if ( err ) { 
+                console.warn(out + err.message); 
+                failures.push(k); 
+              }
+              else console.log(out + 'OK');
+              processNext();
+            }, true);
+          }
+      });
+
+    };
+
+    processNext();
+
+  });
+
+});

tools/create_multilayer

@@ -0,0 +1,40 @@
+#!/bin/sh
+
+verbose=no
+tiler_url=http://dev.localhost.lan:8181/tiles/layergroup
+
+# This is for direct windshaft connection
+#tiler_url=http://dev.localhost.lan:8083/database/cartodb_dev_user_1_db/layergroup
+
+while test -n "$1"; do
+  if test "$1" = "-v"; then
+    verbose=yes
+  elif test -z "$cfg"; then
+    cfg="$1"
+  else
+    tiler_url="$1"
+  fi
+  shift
+done
+
+if test -z "$cfg"; then
+  echo "Usage: $0 [-v] <config_file> [<tiler_url>]" >&2
+  echo "Default <tiler_url> is ${tiler_url}" >&2
+  exit 1
+fi
+
+cmd="curl -skH Content-Type:application/json --data-binary @- ${tiler_url}"
+if test x${verbose} = xyes; then
+  cmd="${cmd} -v"
+fi
+res=`cat ${cfg} | tr '\n' ' ' | ${cmd}`
+if test $? -gt 0; then
+  echo "curl command failed: ${cmd}"
+fi
+
+if test x${verbose} = xyes; then
+  echo "${res}"
+fi
+
+tok=`echo "$res" | sed 's/.*"layergroupid":"\([^"]*\)".*/\1/'`
+echo $tok

tools/create_template

@@ -0,0 +1,49 @@
+#!/bin/sh
+
+verbose=no
+tiler_url=http://dev.localhost.lan:8181/tiles/template
+apikey=${CDB_APIKEY}
+
+while test -n "$1"; do
+  if test "$1" = "-v"; then
+    verbose=yes
+  elif test "$1" = "-k"; then
+    shift
+    apikey="$1"
+  elif test "$1" = "-u"; then
+    shift
+    tiler_url="$1"
+  elif test -z "$cfg"; then
+    cfg="$1"
+  else
+    echo "Unused parameter $1" >&2
+  fi
+  shift
+done
+
+if test -z "$cfg"; then
+  echo "Usage: $0 [-v] [-k <api_key>] [-u <tiler_url>] <template_config>" >&2
+  echo "Default <tiler_url> is ${tiler_url}" >&2
+  echo "Default <api_key> is read from CDB_APIKEY env variable" >&2
+  exit 1
+fi
+
+cmd="curl -skH Content-Type:application/json --data-binary @- ${tiler_url}?api_key=${apikey}"
+if test x${verbose} = xyes; then
+  cmd="${cmd} -v"
+fi
+res=`cat ${cfg} | tr '\n' ' ' | ${cmd}`
+if test $? -gt 0; then
+  echo "curl command failed: ${cmd}"
+fi
+
+if test x${verbose} = xyes; then
+  echo "${res}"
+fi
+
+# Successful response contains no space
+echo "$res" | grep " " && { echo $res && exit 1; }
+
+tok=`echo "$res" | sed 's/.*"template_id":"\([^"]*\)".*/\1/'`
+echo $tok
+

tools/delete_template

@@ -0,0 +1,45 @@
+#!/bin/sh
+
+verbose=no
+tiler_url=http://dev.localhost.lan:8181/tiles/template
+apikey=${CDB_APIKEY}
+
+while test -n "$1"; do
+  if test "$1" = "-v"; then
+    verbose=yes
+  elif test "$1" = "-k"; then
+    shift
+    apikey="$1"
+  elif test "$1" = "-u"; then
+    shift
+    tiler_url="$1"
+  elif test -z "$tpl"; then
+    tpl="$1"
+  else
+    echo "Unused parameter $1" >&2
+  fi
+  shift
+done
+
+if test -z "$tpl"; then
+  echo "Usage: $0 [-v] [-k <api_key>] [-u <tiler_url>] <template_id>" >&2
+  echo "Default <tiler_url> is ${tiler_url}" >&2
+  echo "Default <api_key> is read from CDB_APIKEY env variable" >&2
+  exit 1
+fi
+
+cmd="curl -X DELETE -skH Content-Type:application/json ${tiler_url}/${tpl}?api_key=${apikey}"
+if test x${verbose} = xyes; then
+  cmd="${cmd} -v"
+fi
+res=`${cmd}`
+if test $? -gt 0; then
+  echo "curl command failed: ${cmd}"
+fi
+
+if test x${verbose} = xyes; then
+  echo "${res}"
+fi
+
+tok=`echo "$res" | sed 's/.*"template_id":"\([^"]*\)".*/\1/'`
+echo $tok

tools/instanciate_template

@@ -0,0 +1,53 @@
+#!/bin/sh
+
+verbose=no
+tiler_url=http://dev.localhost.lan:8181/tiles/template
+apikey=${CDB_APIKEY}
+
+while test -n "$1"; do
+  if test "$1" = "-v"; then
+    verbose=yes
+  elif test "$1" = "-k"; then
+    shift
+    apikey="$1"
+  elif test "$1" = "-u"; then
+    shift
+    tiler_url="$1"
+  elif test -z "$tpl"; then
+    tpl="$1"
+  elif test -z "$cfg"; then
+    cfg="$1"
+  else
+    echo "Unused parameter $1" >&2
+  fi
+  shift
+done
+
+if test -z "$tpl"; then
+  echo "Usage: $0 [-v] [-k <api_key>] [-u <tiler_url>] <template_id> [<template_params>]" >&2
+  echo "Default <tiler_url> is ${tiler_url}" >&2
+  echo "Default <api_key> is read from CDB_APIKEY env variable" >&2
+  exit 1
+fi
+
+if test -z "$cfg"; then
+  cfg="/dev/null"
+fi
+
+tiler_url="${tiler_url}/${tpl}"
+
+cmd="curl -X POST -skH Content-Type:application/json --data-binary @- ${tiler_url}?api_key=${apikey}"
+if test x${verbose} = xyes; then
+  cmd="${cmd} -v"
+fi
+res=`cat ${cfg} | tr '\n' ' ' | ${cmd}`
+if test $? -gt 0; then
+  echo "curl command failed: ${cmd}"
+fi
+
+if test x${verbose} = xyes; then
+  echo "${res}"
+fi
+
+tok=`echo "$res" | sed 's/.*"template_id":"\([^"]*\)".*/\1/'`
+echo $tok

tools/munin/Makefile

@@ -0,0 +1,19 @@
+MUNIN_PLUGINS_DIR=/etc/munin/plugins
+MUNIN_PLUGINS_CONFIG_DIR=/etc/munin/plugin-conf.d
+PWD=$(shell pwd)
+
+all: windshaft.conf
+
+windshaft.conf: windshaft.conf.in
+	sed 's#@PWD@#$(PWD)#' < $< > $@
+
+install-munin-plugin-conf: windshaft.conf 
+	install -m 644 $< $(MUNIN_PLUGINS_CONFIG_DIR)/windshaft.conf
+
+install-munin-plugin: windshaft
+	install -m 755 $< $(MUNIN_PLUGINS_DIR)/windshaft
+
+install: install-munin-plugin install-munin-plugin-conf
+
+clean:
+	rm -f windshaft.conf

tools/munin/windshaft

@@ -0,0 +1,87 @@
+#!/bin/sh
+
+envnik=$(basename "${TILER_ENVIRONMENT}" .js)
+
+if test "$1" = "config"; then
+echo "graph_title fd usage (${envnik})"
+cat <<"EOM"
+graph_vlabel number of file descriptors
+graph_category windshaft
+graph_scale no
+procs.label Number of tiler processes
+pgsql.label PostgreSQL connections (max)
+redis.label Redis connections (max)
+http.label Incoming http requests (max)
+caches.label Number of renderer caches (max)
+nfd.label Total file descriptors (max)
+EOM
+exit 0
+fi
+
+if test x"$1" != x; then
+  TILER_ENVIRONMENT=$(cd $(dirname $0); pwd)/../../config/environments/${1}.js
+fi
+
+if test -z "$TILER_ENVIRONMENT"; then
+  echo "Usage: $0 [<environment>]" >&2
+  echo "   or: [TILER_ENVIRONMENT=<environment>] $0" >&2
+  exit 1
+fi
+
+http_port=$(echo "console.log(require('${TILER_ENVIRONMENT}').port)" | node) || exit 1
+pgsql_port=$(echo "console.log(require('${TILER_ENVIRONMENT}').postgres.port)" | node) || exit 1
+redis_port=$(echo "console.log(require('${TILER_ENVIRONMENT}').redis.port)" | node) || exit 1
+
+pids=$(lsof -i :${http_port} | grep LISTEN | awk '{print $2}')
+nworkers=$(echo "${pids}" | wc -l)
+pids=$(echo "${pids}" | paste -sd ' ')
+
+if test -z "${pids}"; then
+  echo "No processes found listening on tcp port '${http_port}'" >&2
+  exit 1
+fi
+
+tmpreport="/tmp/checkfd.$$.txt"
+
+lsof -Pp $(echo "${pids}" | tr ' ' ',') > "${tmpreport}"
+
+maxdb=0
+maxredis=0
+maxhttp=0
+maxtot=0
+maxcache=0
+
+for pid in ${pids}; do
+
+  cnt=$(grep "${pid}" "${tmpreport}" | grep ":${pgsql_port} " | wc -l);
+  if test $cnt -gt $maxdb; then maxdb=$cnt; fi
+  
+  cnt=$(grep "${pid}" "${tmpreport}" | grep ":${redis_port} " | wc -l);
+  if test $cnt -gt $maxredis; then maxredis=$cnt; fi
+
+  cnt=$(grep "${pid}" "${tmpreport}" | grep ":${http_port}-" | grep -v "LISTEN" | wc -l);
+  if test $cnt -gt $maxhttp; then maxhttp=$cnt; fi
+
+  cnt=$(grep "${pid}" "${tmpreport}" | wc -l);
+  if test $cnt -gt $maxtot; then maxtot=$cnt; fi
+
+  log=$(grep "${pid}" "${tmpreport}" | grep -w 1w | awk '{print $9}')
+  if test -e "${log}"; then
+    kill -USR2 "${pid}"
+    cnt=$(tac ${log} | sed -n -e '/ItemKey/p;/^RenderCache/q' | wc -l)
+    if test $cnt -gt $maxcache; then maxcache=$cnt; fi
+  else
+    # report the error...
+    maxcache=-1
+  fi
+
+done
+
+echo "procs.value ${nworkers}"
+echo "pgsql.value ${maxdb}"
+echo "redis.value ${maxredis}"
+echo "http.value ${maxhttp}"
+echo "caches.value ${maxcache}"
+echo "nfd.value ${maxtot}"
+
+rm -f "${tmpreport}"

tools/munin/windshaft.conf.in

@@ -0,0 +1,5 @@
+# Configuration file for munin plugin
+
+[windshaft]
+user root
+env.TILER_ENVIRONMENT @PWD@/../../config/environments/production.js

tools/performance/stresstest_templates.sh

@@ -0,0 +1,54 @@
+#!/bin/sh
+
+verbose=no
+tiler_url=http://dev.localhost.lan:8181/tiles/template
+apikey=${CDB_APIKEY}
+
+while test -n "$1"; do
+  if test "$1" = "-k"; then
+    shift
+    apikey="$1"
+  elif test "$1" = "-u"; then
+    shift
+    tiler_url="$1"
+  elif test -z "$tpl"; then
+    tpl="$1"
+  else
+    echo "Unused parameter $1" >&2
+  fi
+  shift
+done
+
+if test -z "$tpl"; then
+  echo "Usage: $0 [-v] [-k <api_key>] [-u <tiler_url>] <template_config>" >&2
+  echo "Default <tiler_url> is ${tiler_url}" >&2
+  echo "Default <api_key> is read from CDB_APIKEY env variable" >&2
+  exit 1
+fi
+
+basedir=$(cd $(dirname $0); cd ..; pwd)
+export CDB_APIKEY=${apikey}
+max=3000000
+i=0
+while test "$i" -le "$max"; do
+  tpln=`cat ${tpl} | sed "s/\"name\":\"\(.*\)\"/\"name\":\"\1${i}\"/"`
+  tpl_id=`echo ${tpln} | ${basedir}/create_template -u ${tiler_url} /dev/stdin`
+  if test $? -ne 0; then
+    echo $tpl_id >&2
+    break
+  fi
+  tpl_id=`echo ${tpln} | ${basedir}/update_template -u ${tiler_url} ${tpl_id} /dev/stdin`
+  if test $? -ne 0; then
+    echo $tpl_id >&2
+    break
+  fi
+  out=`${basedir}/delete_template -u ${tiler_url} ${tpl_id}`
+  if test $? -ne 0; then
+    echo $out >&2
+    break
+  fi
+  i=$((i+1))
+  if test `expr $i % 100` -eq 0; then
+    echo -n "."
+  fi
+done

tools/reset_styles

@@ -1,35 +1,105 @@
 #!/usr/bin/env node
 
-// Reset redis-stored XML styles so that they are regenerated
-// from CartoCSS on next tile request
+/*
 
-var redis = require('redis')
+This scripts drops all extended map_style keys in redis and regenerates
+the XML caches in all the base ones to target the configured mapnik_version.
 
-var REDIS_PORT = 6379; // TODO: make a parameter
+Optionally (with --convert) it also re-writes the CartoCSS if needed
+to target the configured mapnik_version.
+
+It is recommended to make a backup of the redis database before using
+this script.
+
+*/
+
+var path = require('path');
+
+// Reset all styles in the store
+var grainstore = require('../node_modules/windshaft/node_modules/grainstore/lib/grainstore');
+var mapnik = require('mapnik');
+var redis = require('redis');
+
+function usage(me, exitcode) {
+  console.log("Usage: " + me + " [--convert] <environment>");
+  process.exit(exitcode);
+}
+
+var doConvert = false;
+
+var node_path = process.argv.shift(); 
+var script_path = process.argv.shift(); 
+var me = path.basename(script_path);
+var ENV;
+var arg;
+while ( arg = process.argv.shift() ) {
+  if ( arg == '--convert' ) {
+    doConvert = true;
+  }
+  else if ( ! ENV ) {
+    ENV = arg;
+  }
+  else {
+    usage(me, 1);
+  }
+}
+
+if ( ! ENV ) usage(me, 1);
+
+global.environment = require('../config/environments/' + ENV);
+var serverOptions = require('../lib/cartodb/server_options'); // _after_ setting global.environment
+
+var MAPNIK_VERSION = global.environment.mapnik_version || mapnik.versions.mapnik;
+
+console.log( (doConvert ? "Converting" : "Resetting" ) + ' all styles to target ' + MAPNIK_VERSION);
 
 var dbnum = 0;
 
-var client = redis.createClient(REDIS_PORT, 'localhost');
+var mml_store = new grainstore.MMLStore(serverOptions.redis, serverOptions.grainstore);
+
+var failures = [];
+
+var client = redis.createClient(serverOptions.redis.port, serverOptions.redis.host);
 client.on('connect', function() {
   client.select(dbnum);
   client.keys('map_style|*', function(err, matches) {
 
     processNext = function() {
-      if ( ! matches.length ) process.exit(0);
+      if ( ! matches.length ) process.exit(failures.length);
       var k = matches.shift();
-      console.log("Resetting XML in key: " + k);
-      client.get(k, function(err, val) {
-        if ( err ) throw err;
-        val = JSON.parse(val);
-        delete val.xml;
-        client.set(k, JSON.stringify(val), function() {
-          console.log("done with style " + k);
+
+      if ( /map_style\|.*\|.*\|/.test(k) ) {
+        // See https://github.com/Vizzuality/Windshaft-cartodb/issues/58
+        //console.warn("Key " + k + " is EXTENDED, dropping");
+        client.del(k, function(err) {
+          if ( err ) console.warn("Error dropping key " + k);
           processNext();
         });
+        return;
+      } 
+
+      var params = RegExp(/map_style\|(.*)\|(.*)/).exec(k);
+      var db = params[1];
+      var tab = params[2];
+      var out = 'map_style|' + db + '|' + tab + ': ';
+
+      var mml_builder = mml_store.mml_builder({dbname:db, table:tab},
+                                              function(err, payload) {
+
+          if ( err ) { console.warn(out + err.message); failures.push(k); processNext(); }
+          else {
+            mml_builder.resetStyle(function(err, data) {
+              if ( err ) { console.warn(out + err.message); failures.push(k); }
+              else console.log(out + 'OK' + ( doConvert ? ' (converted)' : '' ));
+              processNext();
+            }, doConvert);
+          }
       });
-    }
+
+    };
 
     processNext();
 
   });
+
 });

tools/show_style

@@ -0,0 +1,29 @@
+#!/bin/sh
+
+# TODO: port to node, if you really need it
+
+REDIS_PORT=6379 # default port
+
+
+if test -z "$1"; then
+  echo "Usage: $0 <username> [<tablename>|~<token>]" >&2
+  exit 1
+fi
+
+username="$1"
+token="$2"
+
+dbname=`redis-cli -p ${REDIS_PORT} -n 5 hget "rails:users:${username}" "database_name"`
+if test $? -ne 0; then
+  exit 1
+fi
+if test -z "${dbname}"; then
+  echo "Username ${username} unknown by redis (try CARTODB/script/restore_redis?)" >&2
+  exit 1
+fi
+echo "Database name for user ${username}: ${dbname}" # only if verbose?
+if test -n "$token"; then
+  redis-cli get "map_style|${dbname}|${token}" | sed -e 's/\\n/\n/g' -e 's/\\//g'
+else
+  redis-cli keys "map_style|${dbname}|*"
+fi

tools/update_template

@@ -0,0 +1,47 @@
+#!/bin/sh
+
+verbose=no
+tiler_url=http://dev.localhost.lan:8181/tiles/template
+apikey=${CDB_APIKEY}
+
+while test -n "$1"; do
+  if test "$1" = "-v"; then
+    verbose=yes
+  elif test "$1" = "-k"; then
+    shift
+    apikey="$1"
+  elif test "$1" = "-u"; then
+    shift
+    tiler_url="$1"
+  elif test -z "$tpl"; then
+    tpl="$1"
+  elif test -z "$cfg"; then
+    cfg="$1"
+  else
+    echo "Unused parameter $1" >&2
+  fi
+  shift
+done
+
+if test -z "$cfg"; then
+  echo "Usage: $0 [-v] [-k <api_key>] [-u <tiler_url>] <template_id> <template_config>" >&2
+  echo "Default <tiler_url> is ${tiler_url}" >&2
+  echo "Default <api_key> is read from CDB_APIKEY env variable" >&2
+  exit 1
+fi
+
+cmd="curl -X PUT -skH Content-Type:application/json --data-binary @- ${tiler_url}/${tpl}?api_key=${apikey}"
+if test x${verbose} = xyes; then
+  cmd="${cmd} -v"
+fi
+res=`cat ${cfg} | tr '\n' ' ' | ${cmd}`
+if test $? -gt 0; then
+  echo "curl command failed: ${cmd}"
+fi
+
+if test x${verbose} = xyes; then
+  echo "${res}"
+fi
+
+tok=`echo "$res" | sed 's/.*"template_id":"\([^"]*\)".*/\1/'`
+echo $tok