Set 1.8.0 release date

See https://github.com/CartoDB/Windshaft/issues/117

.gitignore

@@ -1,2 +1,7 @@
-node_modules
-.idea
+node_modules*
+config.status*
+config/environments/*.js
+.idea
+tools/munin/windshaft.conf
+logs/
+pids/

.travis.yml

@@ -0,0 +1,20 @@
+before_install:
+  - sudo apt-add-repository --yes ppa:mapnik/v2.1.0
+  - sudo apt-get update -q
+  - sudo apt-get install -q libmapnik-dev
+  - createdb template_postgis
+  - psql -c "CREATE EXTENSION postgis" template_postgis
+
+env:
+  - NPROCS=1 JOBS=1
+
+language: node_js
+node_js:
+  - "0.8"
+  - "0.10"
+
+notifications:
+  irc:
+    channels:
+      - "irc.freenode.org#cartodb"
+    use_notice: true

HOWTO_RELEASE

@@ -0,0 +1,13 @@
+1. Ensure proper version in package.json 
+2. Ensure NEWS section exists for the new version, review it, add release date
+3. Drop npm-shrinkwrap.json
+4. Run npm install
+5. Test (make check or npm test), fix if broken before proceeding
+6. Run npm shrinkwrap
+7. Set "from" in npm-shrinkwrap.json for known packages
+   (windshaft, node-varnish, grainstore...)
+8. Commit package.json, npm-shrinwrap.json, NEWS
+9. git tag -a Major.Minor.Patch # use NEWS section as content
+10. Announce
+11. Stub NEWS/package for next version
+

Makefile

@@ -1,8 +1,35 @@
+srcdir=$(shell pwd)
+
 all:
 	npm install
 
 clean:
 	rm -rf node_modules/*
 
-check:
-	./run_tests.sh
+distclean: clean
+	rm config.status*
+
+config.status--test:
+	./configure --environment=test
+
+config/environments/test.js: config.status--test
+	./config.status--test 
+
+check-local: config/environments/test.js
+	./run_tests.sh ${RUNTESTFLAGS} \
+    test/unit/cartodb/*.js \
+    test/acceptance/*.js 
+
+check-submodules:
+	PATH="$$PATH:$(srcdir)/node_modules/.bin/"; \
+	for sub in windshaft grainstore node-varnish mapnik; do \
+	  if test -e node_modules/$${sub}; then \
+	      echo "Testing submodule $${sub}"; \
+	      make -C node_modules/$${sub} check || exit 1; \
+	  fi; \
+	done
+
+check-full: check-local check-submodules
+
+check: check-local
+

NEWS.md

@@ -1,3 +1,313 @@
+1.8.0 -- 2014-02-18
+-------------------
+
+Enhancements:
+
+ * Add script to flush caches (#140)
+ * Add script to list templates
+ * Add statsd support (#139)
+ * Add support for specifying a varnish password
+ * Avoid sending multiple varnish invalidation at once (#135)
+ * Tested with node-0.10 (#141)
+ * Use single redis pooler for torque and grainstore
+ * Reduce cost of garbage collection for localized resources
+ * Allow limiting number of templates for each user (#136)
+ * Allow configuring TTL of mapConfigs via "mapConfigTTL"
+
+1.7.1 -- 2014-02-11
+-------------------
+
+Enhancements:
+
+ * Disable debug logging unless "debug" config param evaluates to true (#137)
+ * Require windshaft 0.17.2 for further reducing log noise (#137)
+
+1.7.0 -- 2014-02-11
+-------------------
+
+New features:
+
+ * Add support for torque tiles (#112)
+ * Add attributes service (#118)
+ * Implement Unified Map API (#126)
+ * Make endpoints configurable (#127)
+
+Enhancements:
+
+ * Allow specifying fixed sqlapi host address (#117)
+ * Include template hash in template instance response, to keep caches
+   of different instances separated (#105)
+
+Bug fixes:
+
+ * Allow space padding in template variables usage (#129)
+ * Allow passing numbers as values for numeric template variables (#130)
+
+
+1.6.3 -- 2014-01-30
+-------------------
+
+Bug fixes: 
+
+* layergroup accept both map_key and api_key (#91)
+* Fix public instanciation of signed template accessing private data (#114)
+* Fix show_style in presence of complex styles
+* Fix use of maxzoom in layergroup config (via windshaft-0.15.1)
+
+Enhancements:
+
+* Add support for instanciating a template map with JSONP (#116)
+* Stop processing XML on renderer creation, not needed anymore since 1.6.1
+  introduced on-demand XML generation.
+
+1.6.2 -- 2014-01-23
+-------------------
+
+Bug fixes:
+
+* Fix support for long (>64k chars) queries in layergroup creation (#111)
+
+Enhancements:
+
+* Enhance tools/show_style to accept an environment parameter and
+  print XML style now it is not in redis anymore (#110)
+* Support CORS in template instanciation endpoint (#113)
+
+1.6.1 -- 2014-01-15
+-------------------
+
+Bug fixes:
+
+* Drop cache headers from error responses (#107)
+* Localize external CartoCSS resources at renderer creation time (#108)
+
+1.6.0 -- 2014-01-10
+-------------------
+
+New features:
+
+ * Add 'user_from_host' directive to generalize username extraction (#100)
+ * Implement signed template maps (#98)
+
+
+Other changes:
+
+ * Update cartodb-redis dependency to "~0.3.0"
+ * Update redis-server dependency to "2.4.0+"
+
+1.5.2 -- 2013-12-05
+-------------------
+
+Bug fixes:
+
+* Fix configuration-level compatibility with versions prior to 1.5 (#96)
+* Fix use of old layergroups on mapnik upgrade (#97)
+
+1.5.1 -- 2013-11-28
+-------------------
+
+Bug fixes:
+
+* Survive presence of malformed CartoCSS in redis (#94)
+* Accept useless point-transform:scale directives (#93)
+
+1.5.0 -- 2013-11-19
+-------------------
+
+NOTE: new configuration directives `postgres_auth_pass` and
+      `postgres.password` added; see config/environments/*.example
+      for documentation.
+
+Improvements:
+
+* Add support for configuring database connection passwords
+* Optionally read user-specific database_host and database_password
+  from redis as per CartoDB-2.5.0 model (#88, #89)
+* Do not force ending dot in SQL-API hostname, for easier testing
+
+Bug fixes:
+
+* Return CORS headers when creating layergroups via GET (windshaft/#92)
+* Fix http status on database authentication error (windshaft/#94)
+* Fix text-face-name error at layergroup creation (windshaft/#93)
+
+Other changes:
+
+* CartoDB redis interaction delegated to "cartodb-redis" module
+
+
+1.4.1 -- 2013-11-08
+-------------------
+
+* Fix support for exponential notation in CartoCSS filter values (#87)
+
+1.4.0 -- 2013-10-31
+-------------------
+
+* Add Support for Mapnik-2.2.0 (#78)
+
+1.3.6 -- 2013-10-11
+-------------------
+
+* Restore support for node-0.8.9 accidentally dropped by 1.3.5
+  NOTE: needs removing node_modules/windshaft and re-running npm install
+
+1.3.5 -- 2013-10-03
+-------------------
+
+* Fixing apostrophes in CartoCSS
+* Fix "sql/table must contain zoom variable" error when using
+  "[ zoom > 3]" CartoCSS snippets (note the space)
+* Fix backward compatibility handling of sqlapi.host configuration (#82)
+* Fix error for invalid text-name in CartoCSS (#81)
+* Do not let anonymous requests use authorized renderer caches 
+
+1.3.4
+------
+
+NOTE: configuration sqlapi.host renamed to sqlapi.domain
+      (support for "sqlapi.host" is retained for backward compatibility)
+
+* Improve empty CartoCSS error message
+* Improve invalid mapnik-geometry-type CSS error message
+* Fix race condition in localization of network resources
+
+1.3.3
+------
+* Set Last-Modified header to allow for 304 responses
+* Add profiling support (needs useProfiler in env config file)
+* Fix double-checking for layergroups with no interactivity
+* Log full layergroup config at creation time (#76)
+
+1.3.2
+------
+* Set default layergroup TTL to 2 hours
+* Serve multilayer tiles and grid with persistent cache control
+
+1.3.1
+------
+* Fix deadlock on new style creation
+* Fix database authentication with multi-table layergroups
+* Add tile and grid fetching checks at layergroup creation time
+* Fix SQL error reporting to NOT split on newline
+* Fix support for CartoCSS attachments
+
+1.3.0
+------
+* Change stats format for multilayer map token request, see
+  http://github.com/Vizzuality/Windshaft-cartodb/wiki/Redis-stats-format
+
+1.2.1
+------
+* Fix multilayer post from firefox
+* Fix multilayer cartocss layer name handling
+
+1.2.0
+------
+* Multilayer API changes
+  * Layers passed by index in grid fetching url
+  * Interactivity only specified in layergroup config
+  * Embed cache_buster within token 
+  * Use ISO format for last_modified timestamp
+* Expected LZMA encoding changed to base64
+
+1.1.10
+------
+* Fix regression with default interactivity parameter (#74)
+* More verbose logging for SQL api connection errors
+* Write stats for multilayer map token request
+
+1.1.9
+-----
+* Handle SQL API errors by requesting no Varnish cache
+* Fix X-Cache-Channel for multilayer (by token) responses
+* Add last_modified field to layergroup creation response (#72)
+* Deprecate signal handler for USR1, add handler for USR2 (#71)
+* Fix support for ampersend characters in CartoCSS
+* Add support for LZMA compressed GET parameters
+* Add support for creating layergroups via GET
+
+1.1.8
+-----
+* Require Windshaft-0.9.1, to reduce harmfulness of cache_buster param
+
+1.1.7 (DD//MM//YY)
+-----
+* Do not let /etc/services confuse FD checker (munin plugin)
+* Multilayer support (#72)
+* Expose renderer settings in the environment config files
+
+1.1.6 (19//02//13)
+-----
+* Require windshaft 0.8.5, fixing some stability issues
+  and providing cache info on request
+* Require grainstore 0.10.9, fixing an issue with multi-geom markers
+* Enhance run_tests.sh to allow running single tests and skipping preparation
+* Fix async throws in getGeometryType, getInfoWindow and getMapMetadata
+* Survive connection refusals from redis
+* Add maxConnection environment configuration, default to 128
+
+1.1.5 (DD//MM//YY)
+-----
+* Fix bogus cached return of utf grid for fully contained tiles (#67)
+
+1.1.4 (DD//MM//YY)
+-----
+* Reduce default extent to allow for consistent proj4 round-tripping 
+* Enhance reset_styles script to use full configuration (#62)
+* Have reset_styles script also drop extended keys (#58)
+* Fix example postgis parameter for simplifying input geoms (#63)
+* Add row_limit to example config (#64)
+
+1.1.3 (30//11//12)
+-----
+* Fix reset_styles script to really skip extended keys
+* CartoCSS versioning
+ * Mapnik-version dependent default styles
+ * Enhance 2.0 -> 2.1 transforms:
+  * styles with conditional markers
+  * scale arrow markers by 50%
+
+1.1.2 (DD//MM//YY)
+-----
+* CartoCSS versioning
+ * Fix use of "style_version" with GET (inline styles)
+ * Enhance 2.0 -> 2.1 transforms:
+  * styles with no semicolon
+  * markers shift due to geometry clipping
+
+1.1.1 (DD//MM//YY)
+-----
+* Add support for persistent client cache headers
+* Fix crash on unknown user (#55)
+* Add /version entry point
+* CartoCSS versioning
+ * Include style_version in GET /style response
+ * Support style_version and style_convert parameters in POST /style request
+ * Support style_version in GET /:z/:x/:y request
+
+1.1.0 (30/10/12)
+=======
+* Add /version entry point
+* CartoCSS versioning
+ * Include version in GET /style response
+ * Support version and convert parameters in POST /style request
+ * Autodetect target mapnik version and let config override it
+ * Add tools/reset_styles script to batch-reset (and optionally convert) styles
+* Configurable logging format (#4)
+* Detailed error on missing user metadata 
+* Properly handle unauthenticated requests for metadata
+* Accept "api_key" in addition to "map_key",
+  both in query_string and POST body (#38)
+* Add ./configure script
+* Allow listening on host IP
+* Replaced environment configs by .example ones
+* Fixed some issues with cluster2
+
+1.0.0 (03/10/12)
+-----
+* Migrated to node 0.8.x.
+
 0.9.0 (25/09/12)
 -----
 * External resources in CartoCSS

README.md

@@ -1,21 +1,53 @@
 Windshaft-CartoDB
 ==================
 
-NOTE: requires node-0.4.x
+[![Build Status](http://travis-ci.org/CartoDB/Windshaft-cartodb.png)]
+(http://travis-ci.org/CartoDB/Windshaft-cartodb)
 
 This is the CartoDB map tiler. It extends Windshaft with some extra
 functionality and custom filters for authentication
 
 * reads dbname from subdomain and cartodb redis for pretty tile urls
-* configures windshaft to publish cartodb_id as the interactivity layer
+* configures windshaft to publish ``cartodb_id`` as the interactivity layer
 * gets the default geometry type from the cartodb redis store
 * allows tiles to be styled individually
 * provides a link to varnish high speed cache
-* provides a infowindow endpoint for windshaft
-* provides a map_metadata endpoint for windshaft
+* provides a ``infowindow`` endpoint for windshaft (DEPRECATED)
+* provides a ``map_metadata`` endpoint for windshaft (DEPRECATED)
+* provides signed template maps API
+  (http://github.com/CartoDB/Windshaft-cartodb/wiki/Template-maps)
 
-Install
--------
+Requirements
+------------
+
+ [core]
+ - node-0.8.x+
+ - PostgreSQL-8.3+
+ - PostGIS-1.5.0+
+ - Redis 2.4.0+ (http://www.redis.io)
+ - Mapnik 2.0 or 2.1 
+
+ [for cache control]
+ - CartoDB-SQL-API 1.0.0+
+ - CartoDB 0.9.5+ (for ``CDB_QueryTables``)
+ - Varnish (http://www.varnish-cache.org)
+
+ [for running the testsuite]
+ - Imagemagick (http://www.imagemagick.org)
+
+Configure
+---------
+
+Create the config/environments/<env>.js files (there are .example files
+to start from). You can optionally use the ./configure script for this,
+see ```./configure --help``` to see available options.
+
+Look at lib/cartodb/server_options.js for more on config
+
+Build/install
+-------------
+
+To fetch and build all node-based dependencies, run:
 
 ```
 git clone
@@ -28,22 +60,18 @@ happen to have startup errors you may need to force rebuilding those
 modules. At any time just wipe out the node_modules/ directory and run
 ```npm install``` again.
 
-Configure
----------
-
-Edit config/environments/<env>.js files
-
-Look at lib/cartodb/server_options for more on config
 
 Run
 ---
 
 ```
-node app.js [development | staging | production]
+node app.js <env> 
 ```
 
-Note that caches are kept in redis. If you're not seeing what
-you expect there may be out-of-sync records in there.
+Where <env> is the name of a configuration file under config/environments/.
+
+Note that caches are kept in redis. If you're not seeing what you expect
+there may be out-of-sync records in there.
 Take a look: http://redis.io/commands
 
 
@@ -58,8 +86,16 @@ Args:
 
 * sql - plain SQL arguments
 * interactivity - specify the column to use in UTFGrid
-* cache_buster - if needed you can add a cachebuster to make sure you're
-  rendering new
+* cache_buster - Specify an identifier for the internal tile cache.
+                 Requesting tiles with the same cache_buster value may
+                 result in being served a cached version of the tile
+                 (even when requesting a tile for the first time, as tiles
+                 can be prepared in advance)
+* cache_policy - Set to "persist" to have the server send an Cache-Control
+                 header requesting caching devices to keep the response
+                 cached as much as possible. This is best used with a
+                 timestamp value in cache_buster for manual control of
+                 updates.
 * geom_type - override the cartodb default
 * style - override the default map style with Carto
 
@@ -71,6 +107,8 @@ Args:
 Args:
 
 * style - the style in CartoCSS you want to set
+* style_version - the version of the style for POST
+* style_convert - request conversion to target version (both POST and GET)
 
 
 **INFOWINDOW**

app.js

@@ -10,28 +10,45 @@
 
 // sanity check
 var ENV = process.argv[2]
-if (ENV != 'development' && ENV != 'production'){
+if (ENV != 'development' && ENV != 'production' && ENV != 'staging' ){
     console.error("\nnode app.js [environment]");
-    console.error("environments: [development, production]\n");
+    console.error("environments: [development, production, staging]\n");
     process.exit(1);
 }
 
 var _ = require('underscore')
     , Step       = require('step')
-	, CartodbWindshaft = require('./lib/cartodb/cartodb_windshaft');
-
+    ;
 // set environment specific variables
 global.settings     = require(__dirname + '/config/settings');
 global.environment  = require(__dirname + '/config/environments/' + ENV);
 _.extend(global.settings, global.environment);
 
-// Include cart_data.js only _after_ the "global" variable is set
+// Include cartodb_windshaft only _after_ the "global" variable is set
 // See https://github.com/Vizzuality/Windshaft-cartodb/issues/28
-var cartoData = require('./lib/cartodb/carto_data');
-
-var Windshaft = require('windshaft');
+var CartodbWindshaft = require('./lib/cartodb/cartodb_windshaft');
 var serverOptions = require('./lib/cartodb/server_options');
 
 ws = CartodbWindshaft(serverOptions);
-ws.listen(global.environment.port);
-console.log("Windshaft tileserver started on port " + global.environment.port);
+
+// Maximum number of connections for one process
+// 128 is a good number if you have up to 1024 filedescriptors
+// 4 is good if you have max 32 filedescriptors
+// 1 is good if you have max 16 filedescriptors
+ws.maxConnections = global.environment.maxConnections || 128;
+
+ws.listen(global.environment.port, global.environment.host);
+
+ws.on('listening', function() {
+  console.log("Windshaft tileserver started on " + global.environment.host + ':' + global.environment.port);
+});
+
+// DEPRECATED, use SIGUSR2
+process.on('SIGUSR1', function() {
+  console.log('WARNING: handling of SIGUSR1 by Windshaft-CartoDB is deprecated, please send SIGUSR2 instead');
+  ws.dumpCacheStats();
+});
+
+process.on('SIGUSR2', function() {
+  ws.dumpCacheStats();
+});

cluster.js

@@ -1,45 +0,0 @@
-/*
- * Windshaft-CartoDB
- * ===============
- *
- * ./app.js [environment]
- *
- * environments: [development, production]
- */
-
-var cluster = require('cluster');
-
-// sanity check
-var ENV = process.argv[2]
-if (ENV != 'development' && ENV != 'production' && ENV != 'staging'){
-    console.error("\nnode app.js [environment]");
-    console.error("environments: [development, production, staging]\n");
-    process.exit(1);
-}
-
-var _ = require('underscore')
-    , Step       = require('step')
-	, CartodbWindshaft = require('./lib/cartodb/cartodb_windshaft');
-    
-
-// set environment specific variables
-global.settings     = require(__dirname + '/config/settings');
-global.environment  = require(__dirname + '/config/environments/' + ENV);
-_.extend(global.settings, global.environment);
-
-// Include cart_data.js only _after_ the "global" variable is set
-// See https://github.com/Vizzuality/Windshaft-cartodb/issues/28
-var cartoData = require('./lib/cartodb/carto_data');
-
-var Windshaft = require('windshaft');
-var serverOptions = require('./lib/cartodb/server_options');
-
-ws = CartodbWindshaft(serverOptions);
-cluster(ws)
-  .use(cluster.logger('logs'))
-  .use(cluster.stats())
-  .use(cluster.pidfiles('pids'))
-  .set('workers', 1)
-  .listen(global.environment.port, global.environment.host);
-
-console.log("Windshaft tileserver started on port " + global.environment.port);

config/environments/development.js

@@ -1,43 +0,0 @@
-var config = {
-     environment: 'development'
-    ,port: 8181
-    ,host: '127.0.0.1'
-    ,enable_cors: true
-    ,cache_enabled: false
-    ,postgres_auth_user: 'development_cartodb_user_<%= user_id %>'
-    ,postgres: {
-        type: "postgis",
-        user: "publicuser",
-        host: '127.0.0.1',
-        port: 5432,
-        extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188",
-        /* experimental
-        geometry_field: "the_geom",
-        extent: "-180,-90,180,90",
-        srid: 4326,
-        */
-        simplify: true
-    }
-    ,millstone: {
-        cache_basedir: '/tmp/cdb-tiler-dev/millstone-dev'
-    }
-    ,redis: {
-        host: '127.0.0.1',
-        port: 6379,
-        idleTimeoutMillis: 1,
-        reapIntervalMillis: 1
-    }
-    ,sqlapi: {
-        protocol: 'http',
-        host: 'localhost.lan',
-        port: 8080,
-        version: 'v1'
-    }
-    ,varnish: {
-        host: 'localhost',
-        port: 6082,
-        ttl: 86400
-    }
-};
-
-module.exports = config;

config/environments/development.js.example

@@ -0,0 +1,118 @@
+var config = {
+     environment: 'development'
+    ,port: 8181
+    ,host: '127.0.0.1'
+    // Regular expression pattern to extract username
+    // from hostname. Must have a single grabbing block.
+    ,user_from_host: '^(.*)\\.localhost'
+
+    // Base URLs for the APIs
+    //
+    // See http://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API
+    // 
+    // Base url for the Templated Maps API
+    // "/api/v1/map/named" is the new API,
+    // "/tiles/template" is for compatibility with versions up to 1.6.x
+    ,base_url_templated: '(?:/api/v1/map/named|/tiles/template)'
+    // Base url for the Detached Maps API
+    // "maps" is the the new API,
+    // "tiles/layergroup" is for compatibility with versions up to 1.6.x
+    ,base_url_detached: '(?:/api/v1/map|/tiles/layergroup)'
+    // Base url for the Inline Maps and Table Maps API
+    ,base_url_legacy: '/tiles/:table'
+
+    // Maximum number of connections for one process
+    // 128 is a good value with a limit of 1024 open file descriptors
+    ,maxConnections:128
+    // Maximum number of templates per user. Unlimited by default.
+    ,maxUserTemplates:1024
+    // Seconds since "last creation" before a detached
+    // or template instance map expires. Or: how long do you want
+    // to be able to navigate the map without a reload ?
+    // Defaults to 7200 (2 hours)
+    ,mapConfigTTL: 7200
+    // idle socket timeout, in miliseconds 
+    ,socket_timeout: 600000
+    ,enable_cors: true
+    ,cache_enabled: false
+    ,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
+    // Templated database username for authorized user
+    // Supported labels: 'user_id' (read from redis)
+    ,postgres_auth_user: 'development_cartodb_user_<%= user_id %>'
+    // Templated database password for authorized user
+    // Supported labels: 'user_id', 'user_password' (both read from redis)
+    ,postgres_auth_pass: '<%= user_password %>'
+    ,postgres: {
+        // Parameters to pass to datasource plugin of mapnik
+        // See http://github.com/mapnik/mapnik/wiki/PostGIS
+        type: "postgis",
+        user: "publicuser",
+        password: "public",
+        host: '127.0.0.1',
+        port: 5432,
+        extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
+        /* experimental
+        geometry_field: "the_geom",
+        extent: "-180,-90,180,90",
+        srid: 4326,
+        */
+        row_limit: 65535,
+        simplify_geometries: true,
+        max_size: 500
+    }
+    ,mapnik_version: undefined
+    ,statsd: {
+        host: 'localhost',
+        port: 8125
+        // support all allowed node-statsd options
+    }
+    ,renderer: {
+      // Milliseconds since last access before renderer cache item expires
+      cache_ttl: 60000,
+      metatile: 4,
+      bufferSize: 64
+    }
+    ,millstone: {
+        // Needs to be writable by server user
+        cache_basedir: '/tmp/cdb-tiler-dev/millstone-dev'
+    }
+    ,redis: {
+        host: '127.0.0.1',
+        port: 6379,
+        // Max number of connections in each pool.
+        // Users will be put on a queue when the limit is hit.
+        // Set to maxConnection to have no possible queues.
+        // There are currently 3 pools involved in serving
+        // windshaft-cartodb requests so multiply this number
+        // by 3 to know how many possible connections will be
+        // kept open by the server. The default is 50.
+        max: 50,
+        idleTimeoutMillis: 1, // idle time before dropping connection
+        reapIntervalMillis: 1 // time between cleanups
+    }
+    ,sqlapi: {
+        protocol: 'http',
+        // If "host" is given, it will be used
+        // to connect to the SQL-API without a
+        // DNS lookup
+        host: '127.0.0.1',
+        port: 8080,
+        // The "domain" part will be appended to 
+        // the cartodb username and passed to
+        // SQL-API requests in the Host HTTP header
+        domain: 'localhost.lan',
+        version: 'v1'
+    }
+    ,varnish: {
+        host: 'localhost',
+        port: 6082,
+        secret: 'xxx',
+        ttl: 86400
+    }
+    // If useProfiler is true every response will be served with an
+    // X-Tiler-Profile header containing elapsed timing for various 
+    // steps taken for producing the response.
+    ,useProfiler:true
+};
+
+module.exports = config;

config/environments/production.js

@@ -1,35 +0,0 @@
-var config = {
-    environment: 'production'
-    ,port: 8181
-    ,host: '127.0.0.1'
-    ,enable_cors: true
-    ,cache_enabled: true
-    ,postgres_auth_user: 'cartodb_user_<%= user_id %>'
-    ,postgres: {
-        user: "publicuser",
-        host: '127.0.0.1',
-        port: 6432,
-        extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188",
-        simplify: true
-    }
-    ,millstone: {
-        cache_basedir: '/home/ubuntu/tile_assets/'
-    }
-    ,redis: {
-        host: '127.0.0.1',
-        port: 6379
-    }
-    ,sqlapi: {
-        protocol: 'https',
-        host: 'cartodb.com',
-        port: 8080,
-        version: 'v2'
-    }
-    ,varnish: {
-        host: 'localhost',
-        port: 6082,
-        ttl: 86400
-    }
-};
-
-module.exports = config;

config/environments/production.js.example

@@ -0,0 +1,112 @@
+var config = {
+    environment: 'production'
+    ,port: 8181
+    ,host: '127.0.0.1'
+    // Regular expression pattern to extract username
+    // from hostname. Must have a single grabbing block.
+    ,user_from_host: '^(.*)\\.cartodb\\.com$'
+
+    // Base URLs for the APIs
+    //
+    // See http://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API
+    // 
+    // Base url for the Templated Maps API
+    // "/api/v1/map/named" is the new API,
+    // "/tiles/template" is for compatibility with versions up to 1.6.x
+    ,base_url_templated: '(?:/api/v1/map/named|/tiles/template)'
+    // Base url for the Detached Maps API
+    // "maps" is the the new API,
+    // "tiles/layergroup" is for compatibility with versions up to 1.6.x
+    ,base_url_detached: '(?:/api/v1/map|/tiles/layergroup)'
+    // Base url for the Inline Maps and Table Maps API
+    ,base_url_legacy: '/tiles/:table'
+
+    // Maximum number of connections for one process
+    // 128 is a good value with a limit of 1024 open file descriptors
+    ,maxConnections:128
+    // Maximum number of templates per user. Unlimited by default.
+    ,maxUserTemplates:1024
+    // Seconds since "last creation" before a detached
+    // or template instance map expires. Or: how long do you want
+    // to be able to navigate the map without a reload ?
+    // Defaults to 7200 (2 hours)
+    ,mapConfigTTL: 7200
+    // idle socket timeout, in miliseconds
+    ,socket_timeout: 600000 
+    ,enable_cors: true
+    ,cache_enabled: true
+    ,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
+    // Templated database username for authorized user
+    // Supported labels: 'user_id' (read from redis)
+    ,postgres_auth_user: 'cartodb_user_<%= user_id %>'
+    // Templated database password for authorized user
+    // Supported labels: 'user_id', 'user_password' (both read from redis)
+    ,postgres_auth_pass: '<%= user_password %>'
+    ,postgres: {
+        // Parameters to pass to datasource plugin of mapnik
+        // See http://github.com/mapnik/mapnik/wiki/PostGIS
+        user: "publicuser",
+        password: "public",
+        host: '127.0.0.1',
+        port: 6432,
+        extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
+        row_limit: 65535,
+        simplify_geometries: true,
+        max_size: 500
+    }
+    ,mapnik_version: undefined
+    ,statsd: {
+        host: 'localhost',
+        port: 8125
+        // support all allowed node-statsd options
+    }
+    ,renderer: {
+      // Milliseconds since last access before renderer cache item expires
+      cache_ttl: 60000,
+      metatile: 4,
+      bufferSize: 64
+    }
+    ,millstone: {
+        // Needs to be writable by server user
+        cache_basedir: '/home/ubuntu/tile_assets/'
+    }
+    ,redis: {
+        host: '127.0.0.1',
+        port: 6379,
+        // Max number of connections in each pool.
+        // Users will be put on a queue when the limit is hit.
+        // Set to maxConnection to have no possible queues.
+        // There are currently 3 pools involved in serving
+        // windshaft-cartodb requests so multiply this number
+        // by 3 to know how many possible connections will be
+        // kept open by the server. The default is 50.
+        max: 50,
+        idleTimeoutMillis: 30000, // idle time before dropping connection
+        reapIntervalMillis: 1000 // time between cleanups
+    }
+    ,sqlapi: {
+        protocol: 'https',
+        // If "host" is given, it will be used
+        // to connect to the SQL-API without a
+        // DNS lookup
+        //host: '127.0.0.1',
+        port: 8080,
+        // The "domain" part will be appended to 
+        // the cartodb username and passed to
+        // SQL-API requests in the Host HTTP header
+        domain: 'cartodb.com',
+        version: 'v2'
+    }
+    ,varnish: {
+        host: 'localhost',
+        port: 6082,
+        secret: 'xxx',
+        ttl: 86400
+    }
+    // If useProfiler is true every response will be served with an
+    // X-Tiler-Profile header containing elapsed timing for various 
+    // steps taken for producing the response.
+    ,useProfiler:false
+};
+
+module.exports = config;

config/environments/staging.js

@@ -1,35 +0,0 @@
-var config = {
-    environment: 'production'
-    ,port: 8181
-    ,host: '127.0.0.1'
-    ,enable_cors: true
-    ,cache_enabled: true
-    ,postgres_auth_user: 'cartodb_staging_user_<%= user_id %>'
-    ,postgres: {
-        user: "publicuser",
-        host: '127.0.0.1',
-        port: 6432,
-        extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188",
-        simplify: true
-    }
-    ,millstone: {
-        cache_basedir: '/home/ubuntu/tile_assets/'
-    }
-    ,redis: {
-        host: '127.0.0.1',
-        port: 6379
-    }
-    ,sqlapi: {
-        protocol: 'https',
-        host: 'cartodb.com',
-        port: 8080,
-        version: 'v2'
-    }
-    ,varnish: {
-        host: 'localhost',
-        port: 6082,
-        ttl: 86400
-    }
-};
-
-module.exports = config;

config/environments/staging.js.example

@@ -0,0 +1,112 @@
+var config = {
+    environment: 'production'
+    ,port: 8181
+    ,host: '127.0.0.1'
+    // Regular expression pattern to extract username
+    // from hostname. Must have a single grabbing block.
+    ,user_from_host: '^(.*)\\.cartodb\\.com$'
+
+    // Base URLs for the APIs
+    //
+    // See http://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API
+    // 
+    // Base url for the Templated Maps API
+    // "/api/v1/maps/named" is the new API,
+    // "/tiles/template" is for compatibility with versions up to 1.6.x
+    ,base_url_templated: '(?:/api/v1/maps/named|/tiles/template)'
+    // Base url for the Detached Maps API
+    // "/api/v1/maps" is the the new API,
+    // "/tiles/layergroup" is for compatibility with versions up to 1.6.x
+    ,base_url_detached: '(?:/api/v1/maps|/tiles/layergroup)'
+    // Base url for the Inline Maps and Table Maps API
+    ,base_url_legacy: '/tiles/:table'
+
+    // Maximum number of connections for one process
+    // 128 is a good value with a limit of 1024 open file descriptors
+    ,maxConnections:128
+    // Maximum number of templates per user. Unlimited by default.
+    ,maxUserTemplates:1024
+    // Seconds since "last creation" before a detached
+    // or template instance map expires. Or: how long do you want
+    // to be able to navigate the map without a reload ?
+    // Defaults to 7200 (2 hours)
+    ,mapConfigTTL: 7200
+    // idle socket timeout, in miliseconds
+    ,socket_timeout: 600000 
+    ,enable_cors: true
+    ,cache_enabled: true
+    ,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms (:res[X-Tiler-Profiler]) -> :res[Content-Type]'
+    // Templated database username for authorized user
+    // Supported labels: 'user_id' (read from redis)
+    ,postgres_auth_user: 'cartodb_staging_user_<%= user_id %>'
+    // Templated database password for authorized user
+    // Supported labels: 'user_id', 'user_password' (both read from redis)
+    ,postgres_auth_pass: '<%= user_password %>'
+    ,postgres: {
+        // Parameters to pass to datasource plugin of mapnik
+        // See http://github.com/mapnik/mapnik/wiki/PostGIS
+        user: "publicuser",
+        password: "public",
+        host: '127.0.0.1',
+        port: 6432,
+        extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
+        row_limit: 65535,
+        simplify_geometries: true,
+        max_size: 500
+    }
+    ,mapnik_version: undefined
+    ,statsd: {
+        host: 'localhost',
+        port: 8125
+        // support all allowed node-statsd options
+    }
+    ,renderer: {
+      // Milliseconds since last access before renderer cache item expires
+      cache_ttl: 60000,
+      metatile: 4,
+      bufferSize: 64
+    }
+    ,millstone: {
+        // Needs to be writable by server user
+        cache_basedir: '/home/ubuntu/tile_assets/'
+    }
+    ,redis: {
+        host: '127.0.0.1',
+        port: 6379,
+        // Max number of connections in each pool.
+        // Users will be put on a queue when the limit is hit.
+        // Set to maxConnection to have no possible queues.
+        // There are currently 3 pools involved in serving
+        // windshaft-cartodb requests so multiply this number
+        // by 3 to know how many possible connections will be
+        // kept open by the server. The default is 50.
+        max: 50,
+        idleTimeoutMillis: 30000, // idle time before dropping connection
+        reapIntervalMillis: 1000 // time between cleanups
+    }
+    ,sqlapi: {
+        protocol: 'https',
+        // If "host" is given, it will be used
+        // to connect to the SQL-API without a
+        // DNS lookup
+        //host: '127.0.0.1',
+        port: 8080,
+        // The "domain" part will be appended to 
+        // the cartodb username and passed to
+        // SQL-API requests in the Host HTTP header
+        domain: 'cartodb.com',
+        version: 'v2'
+    }
+    ,varnish: {
+        host: 'localhost',
+        port: 6082,
+        secret: 'xxx',
+        ttl: 86400
+    }
+    // If useProfiler is true every response will be served with an
+    // X-Tiler-Profile header containing elapsed timing for various 
+    // steps taken for producing the response.
+    ,useProfiler:true
+};
+
+module.exports = config;

config/environments/test.js

@@ -1,38 +0,0 @@
-var config = {
-     environment: 'test'
-    ,port: 8888
-    ,host: '127.0.0.1'
-    ,enable_cors: true
-    ,cache_enabled: false
-    ,postgres_auth_user: 'test_cartodb_user_<%= user_id %>'
-    ,postgres: {
-        user: "publicuser",
-        host: '127.0.0.1',
-        port: 5432,
-        srid: 4326,
-        extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188",
-        simplify: true
-    }
-    ,millstone: {
-        cache_basedir: '/tmp/cdb-tiler-test/millstone'
-    }
-    ,redis: {
-        host: '127.0.0.1',
-        port: 6333,
-        idleTimeoutMillis: 1,
-        reapIntervalMillis: 1
-    }
-    ,sqlapi: {
-        protocol: 'http',
-        host: 'localhost.lan',
-        port: 8080,
-        version: 'v1'
-    }
-    ,varnish: {
-        host: '',
-        port: null,
-        ttl: 86400
-    }
-};
-
-module.exports = config;

config/environments/test.js.example

@@ -0,0 +1,114 @@
+var config = {
+     environment: 'test'
+    ,port: 8888
+    ,host: '127.0.0.1'
+    // Regular expression pattern to extract username
+    // from hostname. Must have a single grabbing block.
+    ,user_from_host: '(.*)'
+
+    // Base URLs for the APIs
+    //
+    // See https://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API
+    // 
+    // Base url for the Templated Maps API
+    // "/api/v1/map/named" is the new API,
+    // "/tiles/template" is for compatibility with versions up to 1.6.x
+    ,base_url_templated: '(?:/api/v1/map/named|/tiles/template)'
+    // Base url for the Detached Maps API
+    // "maps" is the the new API,
+    // "tiles/layergroup" is for compatibility with versions up to 1.6.x
+    ,base_url_detached: '(?:/api/v1/map|/tiles/layergroup)'
+    // Base url for the Inline Maps and Table Maps API
+    ,base_url_legacy: '/tiles/:table'
+
+    // Maximum number of connections for one process
+    // 128 is a good value with a limit of 1024 open file descriptors
+    ,maxConnections:128
+    // Maximum number of templates per user. Unlimited by default.
+    ,maxUserTemplates:1024
+    // Seconds since "last creation" before a detached
+    // or template instance map expires. Or: how long do you want
+    // to be able to navigate the map without a reload ?
+    // Defaults to 7200 (2 hours)
+    ,mapConfigTTL: 7200
+    // idle socket timeout, in miliseconds
+    ,socket_timeout: 600000 
+    ,enable_cors: true
+    ,cache_enabled: false
+    ,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
+    // Templated database username for authorized user
+    // Supported labels: 'user_id' (read from redis)
+    ,postgres_auth_user: 'test_cartodb_user_<%= user_id %>'
+    // Templated database password for authorized user
+    // Supported labels: 'user_id', 'user_password' (both read from redis)
+    ,postgres_auth_pass: 'test_cartodb_user_<%= user_id %>_pass'
+    ,postgres: {
+        // Parameters to pass to datasource plugin of mapnik
+        // See http://github.com/mapnik/mapnik/wiki/PostGIS
+        user: "testpublicuser",
+        password: "public",
+        host: '127.0.0.1',
+        port: 5432,
+        extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
+        row_limit: 65535,
+        simplify_geometries: true,
+        max_size: 500
+    }
+    ,mapnik_version: ''
+    ,statsd: {
+        host: 'localhost',
+        port: 8125
+        // support all allowed node-statsd options
+    }
+    ,renderer: {
+      // Milliseconds since last access before renderer cache item expires
+      cache_ttl: 60000,
+      metatile: 4,
+      bufferSize: 64
+    }
+    ,millstone: {
+        // Needs to be writable by server user
+        cache_basedir: '/tmp/cdb-tiler-test/millstone'
+    }
+    ,redis: {
+        host: '127.0.0.1',
+        port: 6335,
+        // Max number of connections in each pool.
+        // Users will be put on a queue when the limit is hit.
+        // Set to maxConnection to have no possible queues.
+        // There are currently 3 pools involved in serving
+        // windshaft-cartodb requests so multiply this number
+        // by 3 to know how many possible connections will be
+        // kept open by the server. The default is 50.
+        max: 50,
+        idleTimeoutMillis: 1, // idle time before dropping connection
+        reapIntervalMillis: 1 // time between cleanups
+    }
+    ,sqlapi: {
+        protocol: 'http',
+        // If "host" is given, it will be used
+        // to connect to the SQL-API without a
+        // DNS lookup
+        host: '127.0.0.1',
+        port: 1080,
+        // The "domain" part will be appended to 
+        // the cartodb username and passed to
+        // SQL-API requests in the Host HTTP header
+        domain: 'donot_look_this_up',
+        // This port will be used by "make check" for testing purposes
+        // It must be available
+        version: 'v1'
+    }
+    ,varnish: {
+        host: '',
+        port: null,
+        secret: 'xxx',
+        ttl: 86400
+    }
+    // If useProfiler is true every response will be served with an
+    // X-Tiler-Profile header containing elapsed timing for various 
+    // steps taken for producing the response.
+    ,useProfiler:false
+};
+
+module.exports = config;

configure

@@ -0,0 +1,92 @@
+#!/bin/sh
+
+#
+# This script creates config/environments/*.js files using
+# config/environments/*.js.example files as input and performing
+# settings substitutions.
+#
+# It relies on a known format of the .js.example files which haven't
+# been made easier to parse to still let humans copy them manually and
+# do further editing or leave them as such to get the same setup as before
+# the introduction of this script.
+#
+# The script is a work in progress. Available switches are printed
+# by invoking with the --help switch. More switches will be added
+# as the need/request for them arises.
+#
+# --strk(2012-07-23)
+#
+
+ENVDIR=config/environments
+
+PGPORT=
+SQLAPI_PORT=
+MAPNIK_VERSION=
+ENVIRONMENT=development
+
+STATUS="$0 $*"
+
+usage() {
+  echo "Usage: $0 [OPTION]"
+  echo 
+  echo "Configuration:"
+  echo "  --help                        display this help and exit"
+  echo "  --with-pgport=NUM             access PostgreSQL server on TCP port NUM [$PGPORT]"
+  echo "  --with-sqlapi-port=NUM        access SQL-API server on TCP port NUM [$SQLAPI_PORT]"
+  echo "  --with-mapnik-version=STRING  set mapnik version string [$MAPNIK_VERSION]"
+  echo "  --environment=STRING          set output environment name [$ENVIRONMENT]"
+}
+
+while test -n "$1"; do
+  case "$1" in
+    --help|-h)
+      usage
+      exit 0
+      ;;
+    --with-pgport=*)
+      PGPORT=`echo "$1" | cut -d= -f2`
+      ;;
+    --with-sqlapi-port=*)
+      SQLAPI_PORT=`echo "$1" | cut -d= -f2`
+      ;;
+    --with-mapnik-version=*)
+      MAPNIK_VERSION=`echo "$1" | cut -d= -f2`
+      ;;
+    --environment=*)
+      ENVIRONMENT=`echo "$1" | cut -d= -f2`
+      ;;
+    *)
+      echo "Unknown option '$1'" >&2
+      usage >&2
+      exit 1
+  esac
+  shift
+done
+
+ENVEX=./${ENVDIR}/${ENVIRONMENT}.js.example
+
+if [ -z "$PGPORT" ]; then
+  PGPORT=`node -e "console.log(require('${ENVEX}').postgres.port)"`
+fi
+if [ -z "$SQLAPI_PORT" ]; then
+  SQLAPI_PORT=`node -e "console.log(require('${ENVEX}').sqlapi.port)"`
+fi
+
+echo "PGPORT: $PGPORT"
+echo "SQLAPI_PORT: $SQLAPI_PORT"
+echo "MAPNIK_VERSION: $MAPNIK_VERSION"
+echo "ENVIRONMENT: $ENVIRONMENT"
+
+o=`dirname "${ENVEX}"`/`basename "${ENVEX}" .example`
+echo "Writing $o"
+
+# See http://austinmatzko.com/2008/04/26/sed-multi-line-search-and-replace/
+sed -n "1h;1!H;\${;g;s/\(,postgres: {[^}]*port: *'\?\)[^',]*\('\?,\)/\1$PGPORT\2/;p;}" < "${ENVEX}" \
+  | sed "s/mapnik_version:.*/mapnik_version: '$MAPNIK_VERSION'/" \
+  | sed -n "1h;1!H;\${;g;s/\(,sqlapi: {[^}]*port: *'\?\)[^',]*\('\?,\)/\1$SQLAPI_PORT\2/;p;}" \
+  > "$o"
+
+STATUSFILE=config.status--${ENVIRONMENT}
+echo "Writing ${STATUSFILE}"
+echo ${STATUS} > ${STATUSFILE} && chmod +x ${STATUSFILE}
+

lib/cartodb/cache_validator.js

@@ -1,89 +1,26 @@
 var _             = require('underscore'),
     Varnish       = require('node-varnish'),
-    request       = require('request'),
-    crypto      = require('crypto'),
-    channelCache  = {},
     varnish_queue = null;
 
-function init(host, port) {
-    varnish_queue = new Varnish.VarnishQueue(host, port);
+function init(host, port, secret) {
+    varnish_queue = new Varnish.VarnishQueue(host, port, secret);
+    varnish_queue.on('error', function(e) {
+        console.log("[CACHE VALIDATOR ERROR] " + e);
+    });
 }
 
 function invalidate_db(dbname, table) {
+    var cmd = 'purge obj.http.X-Cache-Channel ~ "^' + dbname +
+              ':(.*'+ table +'.*)|(table)$"';
     try{
-        varnish_queue.run_cmd('purge obj.http.X-Cache-Channel ~ "^' + dbname + ':(.*'+ table +'.*)|(table)$"');
-        console.log('[SUCCESS FLUSHING CACHE]');
+        varnish_queue.run_cmd(cmd, false);
     } catch (e) {
-        console.log("[ERROR FLUSHING CACHE] Is enable_cache set to true? Failed for: " + 'purge obj.http.X-Cache-Channel ~ "^' + dbname + ':(.*'+ table +'.*)|(table)$"');
+        console.log("[CACHE VALIDATOR ERROR] could not queue command " +
+                    cmd + " -- " + e);
     }
 }
 
-function generateCacheChannel(req, callback){
-    var cacheChannel = "";
-
-    // use key to call sql api with sql request if present, else just return dbname and table name
-    // base key
-    var tableNames = req.params.table;
-    var dbName     = req.params.dbname;
-    var username   = req.headers.host.split('.')[0];
-
-    // replace tableNames with the results of the explain if present
-    if (_.isString(req.params.sql) && req.params.sql != ''){
-        // initialise MD5 key of sql for cache lookups
-        var sql_md5 = generateMD5(req.params.sql);
-        var api = global.environment.sqlapi;
-        var qs  = {};
-
-        // use cache if present
-        if (!_.isNull(channelCache[sql_md5]) && !_.isUndefined(channelCache[sql_md5])) {
-            callback(channelCache[sql_md5]);
-        } else{
-            // strip out windshaft/mapnik inserted sql if present
-            var sql = req.params.sql.match(/^\((.*)\)\sas\scdbq$/);
-            sql = (sql != null) ? sql[1] : req.params.sql;
-
-            // build up api string
-            var sqlapi = api.protocol + '://' + username + '.' + api.host + ':' + api.port + '/api/' + api.version + '/sql'
-
-            // add query to querystring
-            qs.q = 'SELECT CDB_QueryTables($windshaft$' + sql + '$windshaft$)';
-
-            // add api_key if present in tile request (means table is private)
-            if (_.isString(req.params.map_key) && req.params.map_key != ''){
-                qs.api_key = req.params.map_key;
-            }
-
-            // call sql api
-            request.get({url:sqlapi, qs:qs, json:true}, function(err, response, body){
-                if (!err && response.statusCode == 200) {
-                    tableNames = body.rows[0].cdb_querytables.split(/^\{(.*)\}$/)[1];
-                } else {
-                    //oops, no SQL API. Just cache using fallback 'table' key
-                    tableNames = 'table';
-                }
-                cacheChannel = buildCacheChannel(dbName,tableNames);
-                channelCache[sql_md5] = cacheChannel; // store for caching
-                callback(cacheChannel);
-            });
-        }
-    } else {
-        cacheChannel = buildCacheChannel(dbName,tableNames);
-        callback(cacheChannel);
-    }
-}
-
-function buildCacheChannel(dbName, tableNames){
-    return dbName + ':' + tableNames;
-}
-
-function generateMD5(data){
-    var hash = crypto.createHash('md5');
-    hash.update(data);
-    return hash.digest('hex');
-}
-
 module.exports = {
     init: init,
-    invalidate_db: invalidate_db,
-    generateCacheChannel: generateCacheChannel
+    invalidate_db: invalidate_db
 }

lib/cartodb/carto_data.js

@@ -1,223 +0,0 @@
-/**
- * User: simon
- * Date: 30/08/2011
- * Time: 21:10
- * Desc: CartoDB helper.
- *       Retrieves dbname (based on subdomain/username)
- *       and geometry type from the redis stores of cartodb
- */
-
-var   RedisPool = require("./redis_pool")
-    , _ = require('underscore')
-    , Step = require('step');
-
-module.exports = function() {
-    var redis_pool = new RedisPool(global.environment.redis);
-
-
-    var me = {
-        user_metadata_db: 5,
-        table_metadata_db: 0,
-        user_key:  "rails:users:<%= username %>",
-        table_key: "rails:<%= database_name %>:<%= table_name %>"
-    };
-
-
-    /**
-     * Get the database name for this particular subdomain/username
-     *
-     * @param req - standard express req object. importantly contains host information
-     * @param callback
-     */
-    me.getDatabase = function(req, callback) {
-        // strip subdomain from header host
-        var username = req.headers.host.split('.')[0]
-        var redisKey = _.template(this.user_key, {username: username});
-
-        this.retrieve(this.user_metadata_db, redisKey, 'database_name', callback);
-    };
-
-
-
-    /**
-     * Get the user id for this particular subdomain/username
-     *
-     * @param req - standard express req object. importantly contains host information
-     * @param callback
-     */
-    me.getId= function(req, callback) {
-        // strip subdomain from header host
-        var username = req.headers.host.split('.')[0];
-        var redisKey = _.template(this.user_key, {username: username});
-
-        this.retrieve(this.user_metadata_db, redisKey, 'id', callback);
-    };
-
-    /**
-     * Check the user map key for this particular subdomain/username
-     *
-     * @param req - standard express req object. importantly contains host information
-     * @param callback
-     */
-    me.checkMapKey = function(req, callback) {
-        // strip subdomain from header host
-        var username = req.headers.host.split('.')[0];
-        var redisKey = "rails:users:" + username; 
-        this.retrieve(this.user_metadata_db, redisKey, "map_key", function(err, val) {
-            var valid = 0;
-            if ( val && val == req.query.map_key ) valid = 1;
-            callback(err, valid);
-        });
-    };
-
-    /**
-     * Get privacy for cartodb table
-     *
-     * @param req - standard req object. Importantly contains table and host information
-     * @param callback - is the table private or not?
-     */
-    me.authorize= function(req, callback) {
-        var that = this;
-
-        Step(
-            function(){
-                that.checkMapKey(req, this);
-            },
-            function checkIfInternal(err, check_result){
-                if (err) throw err;
-                if (check_result === 1) {
-                    // authorized by key, login as db owner 
-                    that.getId(req, function(err, user_id) {
-                        if (err) throw new Error(err);
-                        var dbuser = _.template(global.settings.postgres_auth_user, {user_id: user_id});
-                        _.extend(req, {dbuser:dbuser});
-                        callback(err, true); 
-                    });
-                } else {
-                    return true; // continue to check if the table is public/private
-                }
-            }
-            ,function (err, data){
-                if (err) throw err;
-                that.getDatabase(req, this);
-            },
-            function(err, data){
-                if (err) throw err;
-                var redisKey = _.template(that.table_key, {database_name: data, table_name: req.params.table});
-
-                that.retrieve(that.table_metadata_db, redisKey, 'privacy', this);
-            },
-            function(err, data){
-                if (err) throw err;
-                callback(err, data);
-             }
-        );
-    };
-
-
-    /**
-     * Get the geometry type for this particular table;
-     * @param req - standard req object. Importantly contains table and host information
-     * @param callback
-     */
-    me.getGeometryType = function(req, callback){
-        var that = this;
-
-        Step(
-            function(){
-                that.getDatabase(req, this)
-            },
-            function(err, data){
-                if (err) throw err;
-                var redisKey = _.template(that.table_key, {database_name: data, table_name: req.params.table});
-
-                that.retrieve(that.table_metadata_db, redisKey, 'the_geom_type', this);
-            },
-            function(err, data){
-                if (err) throw err;
-                callback(err, data);
-            }
-        );
-    };
-
-
-    me.getInfowindow = function(req, callback){
-        var that = this;
-
-        Step(
-            function(){
-                that.getDatabase(req, this);
-            },
-            function(err, data) {
-                if (err) throw err;
-                var redisKey = _.template(that.table_key, {database_name: data, table_name: req.params.table});
-                that.retrieve(that.table_metadata_db, redisKey, 'infowindow', this);
-            },
-            function(err, data){
-                if (err) throw err;
-                callback(err, data);
-            }
-        );
-    };
-
-
-    me.getMapMetadata = function(req, callback){
-        var that = this;
-
-        Step(
-            function(){
-                that.getDatabase(req, this);
-            },
-            function(err, data) {
-                if (err) throw err;
-                var redisKey = _.template(that.table_key, {database_name: data, table_name: req.params.table});
-
-                that.retrieve(that.table_metadata_db, redisKey, 'map_metadata', this);
-            },
-            function(err, data){
-                if (err) throw err;
-                callback(err, data);
-            }
-        );
-    };
-
-    // Redis Hash lookup
-    me.retrieve = function(db, redisKey, hashKey, callback) {
-        this.redisCmd(db,'HGET',[redisKey, hashKey], callback);
-    };
-
-    // Redis Set member check
-    me.inSet = function(db, setKey, member, callback) {
-        this.redisCmd(db,'SISMEMBER',[setKey, member], callback);
-    };
-
-    /**
-     * Use Redis
-     *
-     * @param db - redis database number
-     * @param redisFunc - the redis function to execute
-     * @param redisArgs - the arguments for the redis function in an array
-     * @param callback - function to pass results too.
-     */
-    me.redisCmd = function(db, redisFunc, redisArgs, callback) {
-        var redisClient;
-
-        Step(
-            function getRedisClient() {
-                redis_pool.acquire(db, this);
-            },
-            function executeQuery(err, data) {
-                redisClient = data;
-                redisArgs.push(this);
-                redisClient[redisFunc.toUpperCase()].apply(redisClient, redisArgs);
-            },
-            function releaseRedisClient(err, data) {
-                if (err) throw err;
-                redis_pool.release(db, redisClient);
-                callback(err, data);
-            }
-        );
-    };
-
-    return me;
-}();

lib/cartodb/cartodb_windshaft.js

@@ -2,13 +2,19 @@
 var _ = require('underscore')
     , Step       = require('step')
     , Windshaft = require('windshaft')
+    , redisPool = new require('redis-mpool')(global.environment.redis)
+    // TODO: instanciate cartoData with redisPool
+    , cartoData  = require('cartodb-redis')(global.environment.redis)
+    , SignedMaps = require('./signed_maps.js')
+    , TemplateMaps = require('./template_maps.js')
     , Cache = require('./cache_validator');
 
 var CartodbWindshaft = function(serverOptions) {
+   var debug = global.environment.debug;
 
     if(serverOptions.cache_enabled) {
         console.log("cache invalidation enabled, varnish on ", serverOptions.varnish_host, ' ', serverOptions.varnish_port);
-        Cache.init(serverOptions.varnish_host, serverOptions.varnish_port);
+        Cache.init(serverOptions.varnish_host, serverOptions.varnish_port, serverOptions.varnish_secret);
         serverOptions.afterStateChange = function(req, data, callback) {
             Cache.invalidate_db(req.params.dbname, req.params.table);
             callback(null, data);
@@ -17,15 +23,45 @@ var CartodbWindshaft = function(serverOptions) {
 
     serverOptions.beforeStateChange = function(req, callback) {
         var err = null;
-        if ( ! req.hasOwnProperty('dbuser') ) {
+        if ( ! req.params.hasOwnProperty('_authorizedByApiKey') ) {
           err = new Error("map state cannot be changed by unauthenticated request!");
         }
         callback(err, req);
     }
 
+    serverOptions.signedMaps = new SignedMaps(redisPool);
+    var templateMapsOpts = {
+      max_user_templates: global.environment.maxUserTemplates
+    }
+    var templateMaps = new TemplateMaps(redisPool, serverOptions.signedMaps, templateMapsOpts);
+
     // boot
     var ws = new Windshaft.Server(serverOptions);
 
+    // Override getVersion to include cartodb-specific versions
+    var wsversion = ws.getVersion;
+    ws.getVersion = function() {
+      var version = wsversion();
+      version.windshaft_cartodb = require('../../package.json').version;
+      return version;
+    }
+
+    // Override sendError to drop added cache headers (if any)
+    // See http://github.com/CartoDB/Windshaft-cartodb/issues/107
+    var ws_sendError = ws.sendError;
+    ws.sendError = function(res) {
+      // NOTE: the "res" object will have no _headers when
+      //       faked by Windshaft, see
+      //       http://github.com/CartoDB/Windshaft-cartodb/issues/109
+      //
+      if ( res._headers ) {
+        delete res._headers['cache-control'];
+        delete res._headers['last-modified'];
+        delete res._headers['x-cache-channel'];
+      } 
+      ws_sendError.apply(this, arguments);
+    };
+
     /**
      * Helper to allow access to the layer to be used in the maps infowindow popup.
      */
@@ -37,7 +73,8 @@ var CartodbWindshaft = function(serverOptions) {
             },
             function(err, data){
                 if (err){
-                    res.send({error: err.message}, 500);
+                    ws.sendError(res, {error: err.message}, 500, 'GET INFOWINDOW', err);
+                    //res.send({error: err.message}, 500);
                 } else {
                     res.send({infowindow: data}, 200);
                 }
@@ -57,7 +94,8 @@ var CartodbWindshaft = function(serverOptions) {
             },
             function(err, data){
                 if (err){
-                    res.send(err.message, 500);
+                    ws.sendError(res, {error: err.message}, 500, 'GET MAP_METADATA', err);
+                    //res.send(err.message, 500);
                 } else {
                     res.send({map_metadata: data}, 200);
                 }
@@ -72,18 +110,481 @@ var CartodbWindshaft = function(serverOptions) {
     ws.del(serverOptions.base_url + '/flush_cache', function(req, res){
         ws.doCORS(res);
         Step(
-            function(){
-                serverOptions.flushCache(req, Cache, this);
+            function flushCache(){
+                serverOptions.flushCache(req, serverOptions.cache_enabled ? Cache : null, this);
             },
-            function(err, data){
+            function sendResponse(err, data){
                 if (err){
-                    res.send(500);
+                    ws.sendError(res, {error: err.message}, 500, 'DELETE CACHE', err);
+                    //res.send(500);
                 } else {
                     res.send({status: 'ok'}, 200);
                 }
             }
         );
     });
+
+    // ---- Template maps interface starts @{
+
+    ws.userByReq = function(req) {
+        return serverOptions.userByReq(req);
+    }
+
+    // This is for Templated maps
+    //
+    // "named" is the official, "template" is for backward compatibility up to 1.6.x
+    //
+    var template_baseurl = global.environment.base_url_templated || '(?:/maps/named|/tiles/template)';
+
+    // Add a template
+    ws.post(template_baseurl, function(req, res) {
+      ws.doCORS(res);
+      var that = this;
+      var response = {};
+      var cdbuser = ws.userByReq(req);
+      Step(
+        function checkPerms(){
+            serverOptions.authorizedByAPIKey(req, this);
+        },
+        function addTemplate(err, authenticated) {
+          if ( err ) throw err;
+          if (authenticated !== 1) {
+            err = new Error("Only authenticated user can create templated maps");
+            err.http_status = 401;
+            throw err;
+          }
+          var next = this;
+          if ( ! req.headers['content-type'] || req.headers['content-type'].split(';')[0] != 'application/json' )
+            throw new Error('template POST data must be of type application/json');
+          var cfg = req.body;
+          templateMaps.addTemplate(cdbuser, cfg, this);
+        },
+        function prepareResponse(err, tpl_id){
+          if ( err ) throw err;
+          // NOTE: might omit "cdbuser" if == dbowner ...
+          return { template_id: cdbuser + '@' + tpl_id };
+        },
+        function finish(err, response){
+            if ( req.profiler ) {
+              var report = req.profiler.toString();
+              res.header('X-Tiler-Profiler', report);
+            }
+            if (err){
+                response = { error: ''+err };
+                var statusCode = 400;
+                if ( ! _.isUndefined(err.http_status) ) {
+                  statusCode = err.http_status;
+                }
+                ws.sendError(res, response, statusCode, 'POST TEMPLATE', err);
+            } else {
+              res.send(response, 200);
+            }
+        }
+      );
+    });
+
+    // Update a template
+    ws.put(template_baseurl + '/:template_id', function(req, res) {
+      ws.doCORS(res);
+      var that = this;
+      var response = {};
+      var cdbuser = ws.userByReq(req);
+      var template;
+      var tpl_id;
+      Step(
+        function checkPerms(){
+            serverOptions.authorizedByAPIKey(req, this);
+        },
+        function updateTemplate(err, authenticated) {
+          if ( err ) throw err;
+          if (authenticated !== 1) {
+            err = new Error("Only authenticated user can list templated maps");
+            err.http_status = 401;
+            throw err;
+          }
+          if ( ! req.headers['content-type'] || req.headers['content-type'].split(';')[0] != 'application/json' )
+            throw new Error('template PUT data must be of type application/json');
+          template = req.body;
+          tpl_id = req.params.template_id.split('@');
+          if ( tpl_id.length > 1 ) {
+            if ( tpl_id[0] != cdbuser ) {
+              err = new Error("Invalid template id '"
+                + req.params.template_id + "' for user '" + cdbuser + "'");
+              err.http_status = 404;
+              throw err;
+            }
+            tpl_id = tpl_id[1];
+          }
+          templateMaps.updTemplate(cdbuser, tpl_id, template, this);
+        },
+        function prepareResponse(err){
+          if ( err ) throw err;
+          return { template_id: cdbuser + '@' + tpl_id };
+        },
+        function finish(err, response){
+            if ( req.profiler ) {
+              var report = req.profiler.toString();
+              res.header('X-Tiler-Profiler', report);
+            }
+            if (err){
+                var statusCode = 400;
+                response = { error: ''+err };
+                if ( ! _.isUndefined(err.http_status) ) {
+                  statusCode = err.http_status;
+                }
+                ws.sendError(res, response, statusCode, 'PUT TEMPLATE', err);
+            } else {
+              res.send(response, 200);
+            }
+        }
+      );
+    });
+
+    // Get a specific template
+    ws.get(template_baseurl + '/:template_id', function(req, res) {
+      ws.doCORS(res);
+      var that = this;
+      var response = {};
+      var cdbuser = ws.userByReq(req);
+      var template;
+      var tpl_id;
+      Step(
+        function checkPerms(){
+            serverOptions.authorizedByAPIKey(req, this);
+        },
+        function updateTemplate(err, authenticated) {
+          if ( err ) throw err;
+          if (authenticated !== 1) {
+            err = new Error("Only authenticated users can get template maps");
+            err.http_status = 401;
+            throw err;
+          }
+          tpl_id = req.params.template_id.split('@');
+          if ( tpl_id.length > 1 ) {
+            if ( tpl_id[0] != cdbuser ) {
+              var err = new Error("Cannot get template id '"
+                + req.params.template_id + "' for user '" + cdbuser + "'");
+              err.http_status = 404;
+              throw err;
+            }
+            tpl_id = tpl_id[1];
+          }
+          templateMaps.getTemplate(cdbuser, tpl_id, this);
+        },
+        function prepareResponse(err, tpl_val){
+          if ( err ) throw err;
+          if ( ! tpl_val ) {
+            err = new Error("Cannot find template '" + tpl_id + "' of user '" + cdbuser + "'");
+            err.http_status = 404;
+            throw err;
+          }
+          // auth_id was added by ourselves,
+          // so we remove it before returning to the user
+          delete tpl_val.auth_id;
+          return { template: tpl_val };
+        },
+        function finish(err, response){
+            if (err){
+                var statusCode = 400;
+                response = { error: ''+err };
+                if ( ! _.isUndefined(err.http_status) ) {
+                  statusCode = err.http_status;
+                }
+                ws.sendError(res, response, statusCode, 'GET TEMPLATE', err);
+            } else {
+              res.send(response, 200);
+            }
+        }
+      );
+    });
+
+    // Delete a specific template
+    ws.delete(template_baseurl + '/:template_id', function(req, res) {
+      ws.doCORS(res);
+      var that = this;
+      var response = {};
+      var cdbuser = ws.userByReq(req);
+      var template;
+      var tpl_id;
+      Step(
+        function checkPerms(){
+            serverOptions.authorizedByAPIKey(req, this);
+        },
+        function updateTemplate(err, authenticated) {
+          if ( err ) throw err;
+          if (authenticated !== 1) {
+            err = new Error("Only authenticated users can delete template maps");
+            err.http_status = 401;
+            throw err;
+          }
+          tpl_id = req.params.template_id.split('@');
+          if ( tpl_id.length > 1 ) {
+            if ( tpl_id[0] != cdbuser ) {
+              var err = new Error("Cannot find template id '"
+                + req.params.template_id + "' for user '" + cdbuser + "'");
+              err.http_status = 404;
+              throw err;
+            }
+            tpl_id = tpl_id[1];
+          }
+          templateMaps.delTemplate(cdbuser, tpl_id, this);
+        },
+        function prepareResponse(err, tpl_val){
+          if ( err ) throw err;
+          return { status: 'ok' };
+        },
+        function finish(err, response){
+            if (err){
+                var statusCode = 400;
+                response = { error: ''+err };
+                if ( ! _.isUndefined(err.http_status) ) {
+                  statusCode = err.http_status;
+                }
+                ws.sendError(res, response, statusCode, 'DELETE TEMPLATE', err);
+            } else {
+              res.send('', 204);
+            }
+        }
+      );
+    });
+
+    // Get a list of owned templates 
+    ws.get(template_baseurl, function(req, res) {
+      ws.doCORS(res);
+      var that = this;
+      var response = {};
+      var cdbuser = ws.userByReq(req);
+      Step(
+        function checkPerms(){
+            serverOptions.authorizedByAPIKey(req, this);
+        },
+        function listTemplates(err, authenticated) {
+          if ( err ) throw err;
+          if (authenticated !== 1) {
+            err = new Error("Only authenticated user can list templated maps");
+            err.http_status = 401;
+            throw err;
+          }
+          templateMaps.listTemplates(cdbuser, this);
+        },
+        function prepareResponse(err, tpl_ids){
+          if ( err ) throw err;
+          // NOTE: might omit "cbduser" if == dbowner ...
+          var ids = _.map(tpl_ids, function(id) { return cdbuser + '@' + id; })
+          return { template_ids: ids };
+        },
+        function finish(err, response){
+            var statusCode = 200;
+            if (err){
+                response = { error: ''+err };
+                if ( ! _.isUndefined(err.http_status) ) {
+                  statusCode = err.http_status;
+                }
+                ws.sendError(res, response, statusCode, 'GET TEMPLATE LIST', err);
+            } else {
+              res.send(response, statusCode);
+            }
+        }
+      );
+    });
+
+    ws.setDBParams = function(cdbuser, params, callback) {
+      Step(
+        function setAuth() {
+          serverOptions.setDBAuth(cdbuser, params, this);
+        },
+        function setConn(err) {
+          if ( err ) throw err;
+          serverOptions.setDBConn(cdbuser, params, this);
+        },
+        function finish(err) {
+          callback(err);
+        }
+      );
+    };
+
+    ws.options(template_baseurl + '/:template_id', function(req, res) {
+      ws.doCORS(res, "Content-Type");
+      return next();
+    });
+
+    // Instantiate a template
+    function instanciateTemplate(req, res, template_params, callback) {
+      ws.doCORS(res);
+      if ( req.profiler ) req.profiler.done('cors');
+      var that = this;
+      var response = {};
+      var template;
+      var signedMaps = serverOptions.signedMaps;
+      var layergroup;
+      var layergroupid;
+      var fakereq; // used for call to createLayergroup
+      var cdbuser = ws.userByReq(req);
+      // Format of template_id: [<template_owner>]@<template_id>
+      var tpl_id = req.params.template_id.split('@');
+      if ( tpl_id.length > 1 ) {
+        if ( tpl_id[0] ) cdbuser = tpl_id[0];
+        tpl_id = tpl_id[1];
+      }
+      var auth_token = req.query.auth_token;
+      Step(
+        function getTemplate(){
+          templateMaps.getTemplate(cdbuser, tpl_id, this);
+        },
+        function checkAuthorized(err, data) {
+          if ( req.profiler ) req.profiler.done('getTemplate');
+          if ( err ) throw err;
+          if ( ! data ) {
+            err = new Error("Template '" + tpl_id + "' of user '" + cdbuser + "' not found");
+            err.http_status = 404;
+            throw err;
+          }
+          template = data;
+          var cert = templateMaps.getTemplateCertificate(template);
+          var authorized = false;
+          try {
+            // authorizedByCert will throw if unauthorized
+            authorized = signedMaps.authorizedByCert(cert, auth_token);
+          } catch (err) {
+            // we catch to add http_status
+            err.http_status = 401;
+            throw err;
+          }
+          if ( ! authorized ) {
+            err = new Error('Unauthorized template instanciation');
+            err.http_status = 401;
+            throw err;
+          }
+          /*if ( (! req.headers['content-type'] || req.headers['content-type'].split(';')[0] != 'application/json') && req.query.callback === undefined) {
+            throw new Error('template POST data must be of type application/json, it is instead ');
+          }*/
+          //var template_params = req.body;
+          if ( req.profiler ) req.profiler.done('authorizedByCert');
+          return templateMaps.instance(template, template_params);
+        },
+        function prepareParams(err, instance){
+          if ( req.profiler ) req.profiler.done('TemplateMaps_instance');
+          if ( err ) throw err;
+          layergroup = instance;
+          fakereq = { query: {}, params: {}, headers: _.clone(req.headers),
+            profiler: req.profiler
+          };
+          ws.setDBParams(cdbuser, fakereq.params, this);
+        },
+        function setApiKey(err){
+          if ( req.profiler ) req.profiler.done('setDBParams');
+          if ( err ) throw err;
+          cartoData.getUserMapKey(cdbuser, this);
+        },
+        function createLayergroup(err, val) {
+          if ( req.profiler ) req.profiler.done('getUserMapKey');
+          if ( err ) throw err;
+          fakereq.params.api_key = val;
+          ws.createLayergroup(layergroup, fakereq, this);
+        },
+        function signLayergroup(err, resp) {
+          // NOTE: createLayergroup uses profiler.start()/end() internally
+          //if ( req.profiler ) req.profiler.done('createLayergroup');
+          if ( err ) throw err;
+          response = resp;
+          var signer = cdbuser;
+          var map_id = response.layergroupid.split(':')[0]; // dropping last_updated 
+          var crt_id = template.auth_id; // check ?
+          if ( ! crt_id ) {
+            var errmsg = "Template '" + tpl_id + "' of user '" + cdbuser + "' has no signature";
+            // Is this really illegal ?
+            // Maybe we could just return an unsigned layergroupid
+            // in this case...
+            err = new Error(errmsg);
+            err.http_status = 403; // Forbidden, we refuse to respond to this
+            throw err;
+          }
+          signedMaps.signMap(signer, map_id, crt_id, this);
+        },
+        function prepareResponse(err) {
+          if ( req.profiler ) req.profiler.done('signMap');
+          if ( err ) throw err;
+          //console.log("Response from createLayergroup: "); console.dir(response);
+          // Add the signature part to the token!
+          var tplhash = templateMaps.fingerPrint(template).substring(0,8);
+          if ( req.profiler ) req.profiler.done('fingerPrint');
+          response.layergroupid = cdbuser + '@' + tplhash + '@' + response.layergroupid;
+          return response;
+        },
+        callback
+      );
+    }
+
+    function finish_instanciation(err, response, res, req) {
+        if ( req.profiler ) {
+          var report = req.profiler.toString();
+          res.header('X-Tiler-Profiler', report);
+        }
+        if (err) {
+            var statusCode = 400;
+            response = { error: ''+err };
+            if ( ! _.isUndefined(err.http_status) ) {
+              statusCode = err.http_status;
+            }
+            if(debug) {
+              response.stack = err.stack;
+            }
+            ws.sendError(res, response, statusCode, 'POST INSTANCE TEMPLATE', err);
+        } else {
+          res.send(response, 200);
+        }
+        if ( req.profiler && req.profiler.statsd_client) {
+          req.profiler.sendStats();
+        }
+    }
+
+    ws.post(template_baseurl + '/:template_id', function(req, res) {
+      if ( req.profiler && req.profiler.statsd_client) {
+        req.profiler.start('windshaft-cartodb.instance_template_post');
+      }
+      Step(
+        function() {
+          if ( ! req.headers['content-type'] || req.headers['content-type'].split(';')[0] != 'application/json') {
+            throw new Error('template POST data must be of type application/json, it is instead ');
+          }
+          instanciateTemplate(req, res, req.body, this);
+        }, function(err, response) {
+          finish_instanciation(err, response, res, req);
+        }
+      );
+    });
+
+    /**
+     * jsonp endpoint, allows to instanciate a template with a json call.
+     * callback query argument is mandartoy
+     */
+    ws.get(template_baseurl + '/:template_id/jsonp', function(req, res) {
+      if ( req.profiler && req.profiler.statsd_client) {
+        req.profiler.start('windshaft-cartodb.instance_template_get');
+      }
+      Step(
+        function() {
+          if ( req.query.callback === undefined || req.query.callback.length === 0) {
+            throw new Error('callback parameter should be present and be a function name');
+          }
+          var config = {};
+          if(req.query.config) {
+            try {
+              config = JSON.parse(req.query.config);
+            } catch(e) {
+              throw new Error('badformed config parameter, should be a valid JSON');
+            }
+          }
+          instanciateTemplate(req, res, config, this);
+        }, function(err, response) {
+          finish_instanciation(err, response, res, req);
+        }
+      );
+    });
+
+
+    // ---- Template maps interface ends @}
+
     return ws;
 }
 

lib/cartodb/redis_pool.js

@@ -1,77 +0,0 @@
-/**
- * RedisPool. A database specific redis pooling lib
- *
- */
-
-var redis  = require('redis')
-  , _      = require('underscore')
-  , Pool   = require('generic-pool').Pool;
-
-// constructor.
-// 
-// - `opts` {Object} optional config for redis and pooling
-var RedisPool = function(opts){
-  var opts = opts || {};
-  var defaults = {
-    host: '127.0.0.1', 
-    port: '6379', 
-    max: 50, 
-    idleTimeoutMillis: 10000, 
-    reapIntervalMillis: 1000, 
-    log: false
-  };    
-  var options = _.defaults(opts, defaults)
-
-  var me = {
-    pools: {} // cached pools by DB name
-  };
-  
-  // Acquire resource.
-  //
-  // - `database` {String} redis database name
-  // - `callback` {Function} callback to call once acquired. Takes the form
-  //   `callback(err, resource)`  
-  me.acquire = function(database, callback) {
-      if (!this.pools[database]) {
-        this.pools[database] = this.makePool(database);            
-      }
-      this.pools[database].acquire(function(err,resource) {
-        callback(err, resource);
-      });
-  };
-  
-  // Release resource.
-  //
-  // - `database` {String} redis database name
-  // - `resource` {Object} resource object to release
-  me.release = function(database, resource) {
-      this.pools[database] && this.pools[database].release(resource);
-  };
-    
-  // Factory for pool objects.
-  me.makePool = function(database) {
-    return Pool({
-      name: database,
-      create: function(callback){
-        var client = redis.createClient(options.port, options.host);          
-        client.on('connect', function () {
-          client.send_anyway = true;
-          client.select(database);  
-          client.send_anyway = false;
-        });    
-        return callback(null, client);
-      },
-      destroy: function(client) { 
-        return client.quit(); 
-      },
-      max: options.max, 
-      idleTimeoutMillis: options.idleTimeoutMillis, 
-      reapIntervalMillis: options.reapIntervalMillis, 
-      log: options.log 
-    });
-  };
-      
-  return me;
-};
-
-module.exports = RedisPool;

lib/cartodb/server_options.js

@@ -1,21 +1,249 @@
 var   _          = require('underscore')
     , Step       = require('step')
-    , cartoData  = require('./carto_data')
-    , Cache = require('./cache_validator');
+    , cartoData  = require('cartodb-redis')(global.environment.redis)
+    , Cache = require('./cache_validator')
+    , mapnik     = require('mapnik')
+    , crypto     = require('crypto')
+    , request    = require('request')
+    , LZMA       = require('lzma/lzma_worker.js').LZMA
+  ;
+
+// This is for backward compatibility with 1.3.3
+if ( _.isUndefined(global.environment.sqlapi.domain) ) {
+  // Only use "host" as "domain" if it contains alphanumeric characters
+  var host = global.environment.sqlapi.host;
+  if ( host && host.match(/[a-zA-Z]/) ) {
+    global.environment.sqlapi.domain = host;
+  }
+}
 
 module.exports = function(){
+
+    var rendererConfig = _.defaults(global.environment.renderer || {}, {
+      cache_ttl: 60000, // milliseconds
+      metatile: 4,
+      bufferSize: 64
+    });
+
     var me = {
-        base_url: '/tiles/:table',
+        // This is for inline maps and table maps
+        base_url: global.environment.base_url_legacy || '/tiles/:table',
+
+        /// @deprecated with Windshaft-0.17.0
+        ///base_url_notable: '/tiles',
+
+        // This is for Detached maps
+        //
+        // "maps" is the official, while
+        // "tiles/layergroup" is for backward compatibility up to 1.6.x
+        //
+        base_url_mapconfig: global.environment.base_url_detached || '(?:/maps|/tiles/layergroup)',
+
         grainstore: {
+          map: {
+            // TODO: allow to specify in configuration
+            srid: 3857
+          },
           datasource: global.environment.postgres,
-          cachedir: global.environment.millstone.cache_basedir
+          cachedir: global.environment.millstone.cache_basedir,
+          mapnik_version: global.environment.mapnik_version || mapnik.versions.mapnik,
+          default_layergroup_ttl: global.environment.mapConfigTTL || 7200, 
+          gc_prob: 0.01 // @deprecated since Windshaft-1.8.0
+        },
+        mapnik: {
+          metatile: rendererConfig.metatile,
+          bufferSize: rendererConfig.bufferSize
+        },
+        statsd: global.environment.statsd,
+        renderCache: {
+          ttl: rendererConfig.cache_ttl
         },
         redis: global.environment.redis,
         enable_cors: global.environment.enable_cors,
         varnish_host: global.environment.varnish.host,
         varnish_port: global.environment.varnish.port,
+        varnish_secret: global.environment.varnish.secret,
         cache_enabled: global.environment.cache_enabled,
-        log_format: '[:date] :req[X-Real-IP] \033[90m:method\033[0m \033[36m:req[Host]:url\033[0m \033[90m:status :response-time ms -> :res[Content-Type]\033[0m'
+        log_format: global.environment.log_format,
+        useProfiler: global.environment.useProfiler
+    };
+
+    // Be nice and warn if configured mapnik version
+    // is != instaled mapnik version
+    if ( mapnik.versions.mapnik != me.grainstore.mapnik_version ) {
+      console.warn("WARNING: detected mapnik version ("
+        + mapnik.versions.mapnik + ") != configured mapnik version ("
+        + me.grainstore.mapnik_version + ")");
+    }
+
+/* This whole block is about generating X-Cache-Channel { */
+
+    // TODO: review lifetime of elements of this cache
+    // NOTE: by-token indices should only be dropped when
+    //       the corresponding layegroup is dropped, because
+    //       we have no SQL after layer creation.
+    me.channelCache = {};
+
+    // Run a query through the SQL api
+    me.sqlQuery = function (username, api_key, sql, callback) {
+      var api = global.environment.sqlapi;
+
+      // build up api string
+      var sqlapihostname = username;
+      if ( api.domain ) sqlapihostname += '.' + api.domain;
+
+      var sqlapi = api.protocol + '://';
+      if ( api.host && api.host != api.domain ) sqlapi += api.host;
+      else sqlapi += sqlapihostname;
+      sqlapi += ':' + api.port + '/api/' + api.version + '/sql';
+
+      var qs  = { q: sql }
+
+      // add api_key if given
+      if (_.isString(api_key) && api_key != '') { qs.api_key = api_key; }
+
+      // call sql api
+      //
+      // NOTE: using POST to avoid size limits:
+      // See http://github.com/CartoDB/Windshaft-cartodb/issues/111
+      //
+      // NOTE: uses "host" header to allow IP based specification
+      //       of sqlapi address (and avoid a DNS lookup)
+      //
+      // NOTE: allows for keeping up to "maxConnections" concurrent
+      //       sockets opened per SQL-API host.
+      // See http://nodejs.org/api/http.html#http_agent_maxsockets
+      //
+      var maxSockets = global.environment.maxConnections || 128;
+      request.post({
+          url:sqlapi, body:qs, json:true,
+          headers:{host: sqlapihostname}
+          // http://nodejs.org/api/http.html#http_agent_maxsockets
+          ,pool:{maxSockets:maxSockets}
+          //,timeout:100
+        }, function(err, res, body)
+      {
+          if (err){
+            console.log('ERROR connecting to SQL API on ' + sqlapi + ': ' + err);
+            callback(err);
+            return;
+          }
+          if (res.statusCode != 200) {
+              var msg = res.body.error ? res.body.error : res.body;
+              callback(new Error(msg));
+              console.log('unexpected response status (' + res.statusCode + ') for sql query: ' + sql + ': ' + msg);
+              return;
+          }
+          callback(null, body.rows);
+      });
+    };
+
+    //
+    // Invoke callback with number of milliseconds since
+    // last update in any of the given tables
+    //
+    me.findLastUpdated = function (username, api_key, tableNames, callback) {
+      var sql = 'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max FROM CDB_TableMetadata m WHERE m.tabname::name = any (\'{'
+        + tableNames.join(',') + '}\')';
+
+      // call sql api
+      me.sqlQuery(username, api_key, sql, function(err, rows){
+          if (err){
+            var msg = err.message ? err.message : err;
+            callback(new Error('could not find last updated timestamp: ' + msg));
+            return;
+          }
+          // when the table has not updated_at means it hasn't been changed so a default last_updated is set
+          var last_updated = 0;
+          if(rows.length !== 0) {
+            last_updated = rows[0].max || 0;
+          }
+          callback(null, last_updated*1000);
+      });
+    };
+
+    me.affectedTables = function (username, api_key, sql, callback) {
+
+      // Replace mapnik tokens
+      sql = sql.replace(RegExp('!bbox!', 'g'), 'ST_MakeEnvelope(0,0,0,0)')
+               .replace(RegExp('!pixel_width!', 'g'), '1')
+               .replace(RegExp('!pixel_height!', 'g'), '1')
+            ;
+
+      // Pass to CDB_QueryTables
+      sql = 'SELECT CDB_QueryTables($windshaft$' + sql + '$windshaft$)';
+
+      // call sql api
+      me.sqlQuery(username, api_key, sql, function(err, rows){
+          if (err){
+            var msg = err.message ? err.message : err;
+            callback(new Error('could not fetch source tables: ' + msg));
+            return;
+          }
+          var qtables = rows[0].cdb_querytables;
+          var tableNames = qtables.split(/^\{(.*)\}$/)[1];
+          tableNames = tableNames.split(',');
+          callback(null, tableNames);
+      });
+    };
+
+    me.buildCacheChannel = function (dbName, tableNames){
+      return dbName + ':' + tableNames.join(',');
+    };
+
+    me.generateMD5 = function(data){
+        var hash = crypto.createHash('md5');
+        hash.update(data);
+        return hash.digest('hex');
+    }
+
+    me.generateCacheChannel = function(req, callback){
+
+      // use key to call sql api with sql request if present, else
+      // just return dbname and table name base key
+      var dbName     = req.params.dbname;
+
+      var cacheKey = [ dbName ];
+      if ( req.params.token ) cacheKey.push(req.params.token);
+      else if ( req.params.sql ) cacheKey.push( me.generateMD5(req.params.sql) );
+      cacheKey = cacheKey.join(':');
+
+      if ( me.channelCache.hasOwnProperty(cacheKey) ) {
+        callback(null, me.channelCache[cacheKey]);
+        return;
+      }
+      else if ( req.params.token ) {
+        // cached cache channel for token-based access should be constructed
+        // at cache creation time
+        callback(new Error('missing channel cache for token ' + req.params.token));
+        return;
+      }
+
+      if ( ! req.params.sql && ! req.params.token ) {
+        var cacheChannel = me.buildCacheChannel(dbName, [req.params.table]);
+        // not worth caching this
+        callback(null, cacheChannel);
+        return;
+      }
+
+      if ( ! req.params.sql ) {
+        callback(new Error("this request doesn't need an X-Cache-Channel generated"));
+        return;
+      }
+
+      var dbName     = req.params.dbname;
+      var username   = this.userByReq(req);
+
+      // strip out windshaft/mapnik inserted sql if present
+      var sql = req.params.sql.match(/^\((.*)\)\sas\scdbq$/);
+      sql = (sql != null) ? sql[1] : req.params.sql;
+
+      me.affectedTables(username, req.params.map_key, sql, function(err, tableNames) {
+          if ( err ) { callback(err); return; }
+          var cacheChannel = me.buildCacheChannel(dbName,tableNames);
+          me.channelCache[cacheKey] = cacheChannel; // store for caching
+          callback(null, cacheChannel);
+      });
     };
 
     // Set the cache chanel info to invalidate the cache on the frontend server
@@ -31,15 +259,337 @@ module.exports = function(){
         // skip non-GET requests, or requests for which there's no response
         if ( req.method != 'GET' || ! req.res ) { cb(null, null); return; }
         var res = req.res;
-        var ttl = global.environment.varnish.ttl || 86400;
-        Cache.generateCacheChannel(req, function(channel){
-            res.header('X-Cache-Channel', channel);
-            res.header('Last-Modified', new Date().toUTCString());
-            res.header('Cache-Control', 'no-cache,max-age='+ttl+',must-revalidate, public');
-            cb(null, channel); // add last-modified too ?
+        var cache_policy = req.query.cache_policy;
+        if ( req.params.token ) cache_policy = 'persist';
+        if ( cache_policy == 'persist' ) {
+          res.header('Cache-Control', 'public,max-age=31536000'); // 1 year
+        } else {
+          var ttl = global.environment.varnish.ttl || 86400;
+          res.header('Cache-Control', 'no-cache,max-age='+ttl+',must-revalidate, public');
+        }
+
+        // Set Last-Modified header
+        var lastUpdated;
+        if ( req.params.cache_buster ) {
+          // Assuming cache_buster is a timestamp
+          // FIXME: store lastModified in the cache channel instead
+          lastUpdated = new Date(parseInt(req.params.cache_buster));
+        } else {
+          lastUpdated = new Date();
+        }
+        res.header('Last-Modified', lastUpdated.toUTCString());
+
+        me.generateCacheChannel(req, function(err, channel){
+            if ( ! err ) {
+              res.header('X-Cache-Channel', channel);
+              cb(null, channel);
+            } else {
+              console.log('ERROR generating cache channel: ' + ( err.message ? err.message : err ));
+              // TODO: evaluate if we should bubble up the error instead
+              cb(null, 'ERROR');
+            }
         });
+    };
+
+    me.afterLayergroupCreate = function(req, mapconfig, response, callback) {
+        var token = response.layergroupid;
+
+        var username = this.userByReq(req);
+
+        var tasksleft = 2; // redis key and affectedTables
+        var errors = [];
+
+        var done = function(err) {
+          if ( err ) {
+            errors.push('' + err);
+          }
+          if ( ! --tasksleft ) {
+            err = errors.length ? new Error(errors.join('\n')) : null;
+            callback(err);
+          }
+        }
+
+        // Don't wait for the mapview count increment to
+        // take place before proceeding. Error will be logged
+        // asyncronously
+        cartoData.incMapviewCount(username, mapconfig.stat_tag, function(err) {
+          if (req.profiler) req.profiler.done('incMapviewCount');
+          if ( err ) console.log("ERROR: failed to increment mapview count for user '" + username + "': " + err);
+          done();
+        });
+
+        var sql = [];
+        _.each(mapconfig.layers, function(lyr) {
+          sql.push(lyr.options.sql);
+        });
+        sql = sql.join(';');
+
+        var dbName = req.params.dbname;
+        var usr    = this.userByReq(req);
+        var key    = req.params.map_key || req.params.api_key;
+
+        var cacheKey = dbName + ':' + token;
+
+        me.affectedTables(usr, key, sql, function(err, tableNames) {
+            if (req.profiler) req.profiler.done('affectedTables');
+
+            if ( err ) { done(err); return; }
+            var cacheChannel = me.buildCacheChannel(dbName,tableNames);
+            me.channelCache[cacheKey] = cacheChannel; // store for caching
+            // find last updated
+            me.findLastUpdated(usr, key, tableNames, function(err, lastUpdated) {
+              if (req.profiler) req.profiler.done('findLastUpdated');
+              if ( err ) { done(err); return; }
+              response.layergroupid = response.layergroupid + ':' + lastUpdated; // use epoch
+              response.last_updated = new Date(lastUpdated).toISOString(); // TODO: use ISO format
+              done(null);
+            });
+        });
+    };
+
+/* X-Cache-Channel generation } */
+
+    me.re_userFromHost = new RegExp(
+      global.environment.user_from_host ||
+      '^([^\\.]+)\\.' // would extract "strk" from "strk.cartodb.com"
+    );
+
+    me.userByReq = function(req) {
+      var host = req.headers.host;
+      var mat = host.match(this.re_userFromHost);
+      if ( ! mat ) {
+        console.error("ERROR: user pattern '" + this.re_userFromHost
+          + "' does not match hostname '" + host + "'");
+        return;
+      }
+      // console.log("Matches: "); console.dir(mat);
+      if ( ! mat.length === 2 ) {
+        console.error("ERROR: pattern '" + this.re_userFromHost
+          + "' gave unexpected matches against '" + host + "': " + mat);
+        return;
+      }
+      return mat[1];
     }
 
+    // Set db authentication parameters to those of the given username
+    //
+    // @param username the cartodb username, mapped to a database username
+    //                 via CartodbRedis metadata records
+    //
+    // @param params the parameters to set auth options into
+    //               added params are: "dbuser" and "dbpassword"
+    //
+    // @param callback function(err) 
+    //
+    me.setDBAuth = function(username, params, callback) {
+
+      var user_params = {};
+      var auth_user = global.environment.postgres_auth_user;
+      var auth_pass = global.environment.postgres_auth_pass;
+      Step(
+        function getId() {
+          cartoData.getUserId(username, this);
+        },
+        function(err, user_id) {
+          if (err) throw err;
+          user_params['user_id'] = user_id;
+          var dbuser = _.template(auth_user, user_params);
+          _.extend(params, {dbuser:dbuser});
+
+          // skip looking up user_password if postgres_auth_pass
+          // doesn't contain the "user_password" label 
+          if (!auth_pass || ! auth_pass.match(/\buser_password\b/) ) return null;
+
+          cartoData.getUserDBPass(username, this);
+        },
+        function(err, user_password) {
+          if (err) throw err;
+          user_params['user_password'] = user_password;
+          if ( auth_pass ) {
+            var dbpass = _.template(auth_pass, user_params);
+            _.extend(params, {dbpassword:dbpass});
+          }
+          return true;
+        },
+        function finish(err) {
+          callback(err); 
+        }
+      );
+    };
+
+    // Set db connection parameters to those for the given username
+    //
+    // @param dbowner cartodb username of database owner,
+    //                mapped to a database username
+    //                via CartodbRedis metadata records
+    //
+    // @param params the parameters to set connection options into
+    //               added params are: "dbname", "dbhost"
+    //
+    // @param callback function(err) 
+    //
+    me.setDBConn = function(dbowner, params, callback) {
+      Step(
+        function getDatabaseHost(){
+            cartoData.getUserDBHost(dbowner, this);
+        },
+        function getDatabase(err, data){
+            if(err) throw err;
+            if ( data ) _.extend(params, {dbhost:data});
+            cartoData.getUserDBName(dbowner, this);
+        },
+        function extendParams(err, data){
+            if (err) throw err;
+            if ( data ) _.extend(params, {dbname:data});
+            return null;
+        },
+        function finish(err) {
+          callback(err); 
+        }
+      );
+    };
+
+
+    // Check if a request is authorized by a signer
+    //
+    // Any existing signature for the given request will verified
+    // for authorization to this specific request (may require auth_token)
+    // See https://github.com/CartoDB/Windshaft-cartodb/wiki/Signed-maps
+    //
+    // @param req express request object
+    // @param callback function(err, signed_by) signed_by will be
+    //                 null if the request is not signed by anyone
+    //                 or will be a string cartodb username otherwise.
+    //                 
+    me.authorizedBySigner = function(req, callback)
+    {
+      if ( ! req.params.token || ! req.params.signer ) {
+        //console.log("No signature provided"); // debugging
+        callback(null, null); // no signer requested
+        return;
+      }
+
+      var signer = req.params.signer;
+      var layergroup_id = req.params.token;
+      var auth_token = req.params.auth_token;
+
+      //console.log("Checking authorization from signer " + signer + " for resource " + layergroup_id + " with auth_token " + auth_token);
+
+      me.signedMaps.isAuthorized(signer, layergroup_id, auth_token,
+        function(err, authorized) {
+          callback(err, authorized ? signer : null);
+        });
+    };
+
+    // Check if a request is authorized by api_key
+    //
+    // @param req express request object
+    // @param callback function(err, authorized) 
+    //        NOTE: authorized is expected to be 0 or 1 (integer)
+    //                 
+    me.authorizedByAPIKey = function(req, callback)
+    {
+        var givenKey = req.query.api_key || req.query.map_key;
+        if ( ! givenKey && req.body ) {
+          // check also in request body
+          givenKey = req.body.api_key || req.body.map_key;
+        }
+        if ( ! givenKey ) {
+          callback(null, 0); // no api key, no authorization...
+          return;
+        }
+        //console.log("given ApiKey: " + givenKey);
+        var user = me.userByReq(req);
+        Step(
+          function (){
+              cartoData.getUserMapKey(user, this);
+          },
+          function checkApiKey(err, val){
+              if (err) throw err;
+              return ( val && givenKey == val ) ? 1 : 0;
+          },
+          function finish(err, authorized) {
+              callback(err, authorized);
+          }
+        );
+    };
+
+    /**
+     * Check access authorization
+     *
+     * @param req - standard req object. Importantly contains table and host information
+     * @param callback function(err, allowed) is access allowed not?
+     */
+    me.authorize = function(req, callback) {
+        var that = this;
+        var user = me.userByReq(req);
+
+        Step(
+            function (){
+                that.authorizedByAPIKey(req, this);
+            },
+            function checkApiKey(err, authorized){
+                if (req.profiler) req.profiler.done('authorizedByAPIKey');
+                if (err) throw err;
+
+                // if not authorized by api_key, continue 
+                if (authorized !== 1)  {
+                  // not authorized by api_key, 
+                  // check if authorized by signer
+                  that.authorizedBySigner(req, this);
+                  return;
+                }
+
+                _.extend(req.params, { _authorizedByApiKey: true });
+
+                // authorized by api key, login as the given username and stop
+                that.setDBAuth(user, req.params, function(err) {
+                  callback(err, true); // authorized (or error)
+                });
+            },
+            function checkSignAuthorized(err, signed_by){
+                if (err) throw err;
+                if (req.profiler) {
+                  if ( req.params._authorizedByApiKey ) {
+                    req.profiler.done('setDBAuth');
+                  } else {
+                    req.profiler.done('authorizedBySigner');
+                  }
+                }
+
+                if ( ! signed_by ) {
+                  // request not authorized by signer,
+                  // continue to check table privacy,
+                  // if table was given
+                  if ( req.params.table ) return null;
+                  // otherwise return no authorization
+                  callback(err, null);
+                  return;
+                }
+
+                // Authorized by "signed_by" !
+                that.setDBAuth(signed_by, req.params, function(err) {
+                  if (req.profiler) req.profiler.done('setDBAuth');
+                  callback(err, true); // authorized (or error)
+                });
+            },
+            function getDatabase(err){
+                if (err) throw err;
+                // NOTE: only used to get to table privacy
+                cartoData.getUserDBName(user, this);
+            },
+            function getPrivacy(err, dbname){
+                if (err) throw err;
+                if (req.profiler) req.profiler.done('tablePrivacy_getUserDBName');
+                cartoData.getTablePrivacy(dbname, req.params.table, this);
+            },
+            function(err, privacy){
+                if (req.profiler) req.profiler.done('getTablePrivacy');
+                callback(err, privacy);
+            }
+        );
+    };
+
     /**
      * Whitelist input and get database name & default geometry type from
      * subdomain/user metadata held in CartoDB Redis
@@ -48,55 +598,108 @@ module.exports = function(){
      */
     me.req2params = function(req, callback){
 
+        if ( req.query.lzma ) {
+
+          // TODO: check ?
+          //console.log("type of req.query.lzma is " + typeof(req.query.lzma));
+
+          // Decode (from base64)
+          var lzma = (new Buffer(req.query.lzma, 'base64').toString('binary')).split('').map(function(c) { return c.charCodeAt(0) - 128 })
+
+          // Decompress
+          LZMA.decompress(
+            lzma,
+            function(result) {
+              if (req.profiler) req.profiler.done('LZMA decompress');
+              try {
+                delete req.query.lzma
+                _.extend(req.query, JSON.parse(result))
+                me.req2params(req, callback);
+              } catch (err) {
+                callback(new Error('Error parsing lzma as JSON: ' + err));
+              }
+            },
+            function(percent) { // progress
+              //console.log("LZMA decompression " + percent + "%");
+            }
+          );
+          return;
+        }
+
         // Whitelist query parameters and attach format
-        var good_query = ['sql', 'geom_type', 'cache_buster','callback', 'interactivity', 'map_key', 'style'];
+        var good_query = ['sql', 'geom_type', 'cache_buster', 'cache_policy', 'callback', 'interactivity', 'map_key', 'api_key', 'auth_token', 'style', 'style_version', 'style_convert', 'config' ];
         var bad_query  = _.difference(_.keys(req.query), good_query);
 
         _.each(bad_query, function(key){ delete req.query[key]; });
         req.params =  _.extend({}, req.params); // shuffle things as request is a strange array/object
 
+        if ( req.params.token ) {
+          //console.log("Request parameters include token " + req.params.token);
+          var tksplit = req.params.token.split(':');
+          req.params.token = tksplit[0];
+          if ( tksplit.length > 1 ) req.params.cache_buster= tksplit[1];
+          tksplit = req.params.token.split('@');
+          if ( tksplit.length > 1 ) {
+            req.params.signer = tksplit.shift();
+            if ( ! req.params.signer ) req.params.signer = this.userByReq(req);
+            if ( tksplit.length > 1 ) {
+              var template_hash = tksplit.shift(); // unused
+            }
+            req.params.token = tksplit.shift(); 
+            //console.log("Request for token " + req.params.token + " with signature from " + req.params.signer);
+          }
+        }
+
         // bring all query values onto req.params object
         _.extend(req.params, req.query);
 
         // for cartodb, ensure interactivity is cartodb_id or user specified
         req.params.interactivity = req.params.interactivity || 'cartodb_id';
 
-        req.params.processXML = function(req, xml, callback) {
-          var dbuser = req.dbuser ? req.dbuser : global.settings.postgres.user;
-          if ( ! me.rx_dbuser ) me.rx_dbuser = /(<Parameter name="user"><!\[CDATA\[)[^\]]*(]]><\/Parameter>)/;
-          xml = xml.replace(me.rx_dbuser, "$1" + dbuser + "$2");
-          callback(null, xml);
-        }
-
         var that = this;
 
+        if (req.profiler) req.profiler.done('req2params.setup');
+
+        var user = me.userByReq(req);
+
         Step(
             function getPrivacy(){
-                cartoData.authorize(req, this);
+                me.authorize(req, this);
             },
             function gatekeep(err, data){
+                if (req.profiler) req.profiler.done('authorize');
                 if(err) throw err;
                 if(data === "0") throw new Error("Sorry, you are unauthorized (permission denied)");
                 return data;
             },
             function getDatabase(err, data){
                 if(err) throw err;
-
-                cartoData.getDatabase(req, this);
+                that.setDBConn(user, req.params, this);
             },
-            function getGeometryType(err, data){
+            function getGeometryType(err){
+                if (req.profiler) req.profiler.done('setDBConn');
                 if (err) throw err;
-                _.extend(req.params, {dbname:data});
-
-                cartoData.getGeometryType(req, this);
+                if ( ! req.params.table ) return null;
+                cartoData.getTableGeometryType(req.params.dbname, req.params.table, this);
             },
             function finishSetup(err, data){
+                if (req.profiler) req.profiler.done('cartoData.getTableGeometryType');
                 if ( err ) { callback(err, req); return; }
 
                 if (!_.isNull(data))
                     _.extend(req.params, {geom_type: data});
 
-                that.addCacheChannel(req, function(err, chan) {
+                // Add default database connection parameters
+                // if none given
+                _.defaults(req.params, {
+                  dbuser: global.environment.postgres.user,
+                  dbpassword: global.environment.postgres.password,
+                  dbhost: global.environment.postgres.host,
+                  dbport: global.environment.postgres.port
+                });
+
+                that.addCacheChannel(req, function(err) {
+                  if (req.profiler) req.profiler.done('addCacheChannel');
                   callback(err, req);
                 });
             }
@@ -110,14 +713,23 @@ module.exports = function(){
      */
     me.getInfowindow = function(req, callback){
         var that = this;
+        var user = me.userByReq(req);
 
         Step(
             function(){
+                // TODO: if this step really needed ?
                 that.req2params(req, this);
             },
+            function getDatabase(err){
+                if (err) throw err;
+                cartoData.getUserDBName(user, this);
+            },
+            function getInfowindow(err, dbname){
+                if (err) throw err;
+                cartoData.getTableInfowindow(dbname, req.params.table, this);
+            },
             function(err, data){
-                if (err) callback(err, null);
-                else cartoData.getInfowindow(data, callback);
+                callback(err, data);
             }
         );
     };
@@ -129,14 +741,23 @@ module.exports = function(){
      */
     me.getMapMetadata = function(req, callback){
         var that = this;
+        var user = me.userByReq(req);
 
         Step(
             function(){
+                // TODO: if this step really needed ?
                 that.req2params(req, this);
             },
-            function(err, data){
+            function getDatabase(err){
                 if (err) throw err;
-                cartoData.getMapMetadata(data, callback);
+                cartoData.getUserDBName(user, this);
+            },
+            function getMapMetadata(err, dbname){
+                if (err) throw err;
+                cartoData.getTableMapMetadata(dbname, req.params.table, this);
+            },
+            function(err, data){
+                callback(err, data);
             }
         );
     };
@@ -150,12 +771,20 @@ module.exports = function(){
         var that = this;
 
         Step(
-            function(){
+            function getParams(){
+                // this is mostly to compute req.params.dbname
                 that.req2params(req, this);
             },
-            function(err, data){
-                if (err) throw err;
-                Cache.invalidate_db(req.params.dbname, req.params.table);
+            function flushInternalCache(err){
+                // TODO: implement this, see
+                // http://github.com/Vizzuality/Windshaft-cartodb/issues/73
+                return true;
+            },
+            function flushVarnishCache(err){
+                if (err) { callback(err); return; }
+                if(Cache) {
+                  Cache.invalidate_db(req.params.dbname, req.params.table);
+                }
                 callback(null, true);
             }
         );

lib/cartodb/signed_maps.js

@@ -0,0 +1,401 @@
+var crypto    = require('crypto');
+var Step      = require('step');
+var _         = require('underscore');
+
+var debug = global.environment ? global.environment.debug : undefined;
+
+// Class handling map signatures and user certificates
+//
+// See https://github.com/CartoDB/Windshaft-cartodb/wiki/Signed-maps
+//
+// @param redis_pool an instance of a "redis-mpool"
+//        See https://github.com/CartoDB/node-redis-mpool
+//        Needs version 0.x.x of the API.
+//
+function SignedMaps(redis_pool) {
+  this.redis_pool = redis_pool;
+
+  // Database containing signatures
+  // TODO: allow configuring ?
+  // NOTE: currently it is the same as
+  //       the one containing layergroups
+  this.db_signatures = 0;
+
+  //
+  // Map signatures in redis are reference to signature certificates
+  // We have the following datastores:
+  //
+  //  1. User certificates: set of per-user authorization certificates
+  //  2. Map signatures: set of per-map certificate references 
+  //  3. Certificate applications: set of per-certificate signed maps
+
+  // User certificates (HASH:crt_id->crt_val)
+  this.key_map_crt = "map_crt|<%= signer %>";
+
+  // Map signatures (SET:crt_id)
+  this.key_map_sig = "map_sig|<%= signer %>|<%= map_id %>";
+  
+  // Certificates applications (SET:map_id)
+  //
+  // Everytime a map is signed, the map identifier (layergroup_id)
+  // is added to this set. The purpose of this set is to drop
+  // all map signatures when a certificate is removed
+  //
+  this.key_crt_sig = "crt_sig|<%= signer %>|<%= crt_id %>";
+
+};
+
+var o = SignedMaps.prototype;
+
+//--------------- PRIVATE METHODS --------------------------------
+
+o._acquireRedis = function(callback) {
+  this.redis_pool.acquire(this.db_signatures, callback);
+};
+
+o._releaseRedis = function(client) {
+  this.redis_pool.release(this.db_signatures, client);
+};
+
+/**
+ * Internal function to communicate with redis
+ *
+ * @param redisFunc - the redis function to execute
+ * @param redisArgs - the arguments for the redis function in an array
+ * @param callback - function to pass results too.
+ */
+o._redisCmd = function(redisFunc, redisArgs, callback) {
+  var redisClient;
+  var that = this;
+  var db = that.db_signatures;
+
+  Step(
+    function getRedisClient() {
+      that.redis_pool.acquire(db, this);
+    },
+    function executeQuery(err, data) {
+      if ( err ) throw err;
+      redisClient = data;
+      redisArgs.push(this);
+      redisClient[redisFunc.toUpperCase()].apply(redisClient, redisArgs);
+    },
+    function releaseRedisClient(err, data) {
+      if ( ! _.isUndefined(redisClient) ) that.redis_pool.release(db, redisClient);
+      callback(err, data);
+    }
+  );
+};
+
+o._getAuthMethod = function(auth) {
+  return auth.method || 'open';
+};
+
+//--------------- PUBLIC API -------------------------------------
+
+/// Check formal validity of a certificate
+//
+/// Return an Error instance if invalid, null otherwise
+///
+o.checkInvalidCertificate = function(cert) {
+  //console.log("Checking cert: "); console.dir(cert);
+  if ( cert.version !== "0.0.1" ) {
+    return new Error("Unsupported certificate version " + cert.version);
+  }
+
+  if ( ! cert.auth ) {
+console.log("Cert is : "); console.dir(cert);
+    return new Error("No certificate authorization");
+  }
+
+  var method = this._getAuthMethod(cert.auth);
+
+  switch ( method ) {
+    case 'open':
+      break;
+    case 'token':
+      if ( ! _.isArray(cert.auth.valid_tokens) )
+        return new Error("Invalid 'token' authentication: missing valid_tokens");
+      if ( ! cert.auth.valid_tokens.length )
+        return new Error("Invalid 'token' authentication: no valid_tokens");
+      break;
+    default:
+      return new Error("Unsupported authentication method: " + cert.auth.method);
+      break;
+  }
+
+  return null; // all valid
+}
+
+// Check if the given certificate authorizes waiver of "auth"
+o.authorizedByCert = function(cert, auth) {
+
+  var err = this.checkInvalidCertificate(cert);
+  if ( err ) throw err;
+
+  var method = this._getAuthMethod(cert.auth);
+
+  // Open authentication certificates are always authorized
+  if ( method === 'open' ) return true;
+
+  // Token based authentication requires valid token
+  if ( method === 'token' ) {
+    var found = cert.auth.valid_tokens.indexOf(auth);
+    //if ( found !== -1 ) {
+      //console.log("Token " + auth + " is found at position " + found + " in valid tokens " + cert.auth.valid_tokens);
+    //  return true;
+    //} else return false;
+    return cert.auth.valid_tokens.indexOf(auth) !== -1;
+  }
+
+  throw new Error("Unsupported authentication method: " + cert.auth.method);
+};
+
+// Check if shown credential are authorized to access a map
+// by the given signer.
+//
+// @param signer a signer name (cartodb username)
+// @param map_id a layergroup_id
+// @param auth an authentication token, or undefined if none
+//                    (can still be authorized by signature)
+//
+// @param callback function(Error, Boolean)
+//
+o.isAuthorized = function(signer, map_id, auth, callback) {
+  var that = this;
+  var redisClient;
+  var db = that.db_signatures;
+  var authorized = false;
+  var certificate_id_list;
+  var missing_certificates = [];
+  if ( debug ) {
+    console.log("Check auth from signer '" + signer + "' on map '" + map_id + "' with auth '" + auth + "'");
+  }
+  Step(
+    function getRedisClient() {
+      that.redis_pool.acquire(db, this);
+    },
+    function getMapSignatures(err, client) {
+      if ( err ) throw err;
+      redisClient = client;
+      var map_sig_key = _.template(that.key_map_sig, {signer:signer, map_id:map_id});
+      redisClient.SMEMBERS(map_sig_key, this);
+      //that._redisCmd('SMEMBERS', [ map_sig_key ], this);
+    },
+    function getCertificates(err, crt_lst) {
+      if ( err ) throw err;
+      if ( debug ) {
+        console.log("Map '" + map_id + "' is signed by " + crt_lst.length + " certificates of user '" + signer);
+      }
+      certificate_id_list = crt_lst;
+      if ( ! crt_lst.length ) {
+        // No certs, avoid calling redis with short args list.
+        // Next step expects a list of certificate values so
+        // we directly send the empty list.
+        return crt_lst;
+      }
+      var map_crt_key = _.template(that.key_map_crt, {signer:signer});
+      //that._redisCmd('HMGET', [ map_crt_key ].concat(crt_lst), this);
+      redisClient.HMGET(map_crt_key, crt_lst, this);
+    },
+    function checkCertificates(err, certs) {
+      if ( err ) throw err;
+      for (var i=0; i<certs.length; ++i) {
+        var crt_id = certificate_id_list[i];
+        if ( _.isNull(certs[i]) ) {
+          missing_certificates.push(crt_id);
+          continue;
+        }
+        var cert;
+        try {
+          //console.log("cert " + crt_id + ": " + certs[i]);
+          cert = JSON.parse(certs[i]);
+          authorized = that.authorizedByCert(cert, auth);
+        } catch (err) {
+          console.log("Certificate " + certificate_id_list[i] + " by user '" + signer + "' is malformed: " + err);
+          continue;
+        }
+        if ( authorized ) {
+          if ( debug ) {
+            console.log("Access to map '" + map_id + "' authorized by cert '"
+                        + certificate_id_list[i] + "' of user '" + signer + "'");
+          }
+          //console.dir(cert);
+          break; // no need to further check certs
+        }
+      }
+      return null;
+    },
+    function finish(err) {
+      if ( missing_certificates.length ) {
+        console.log("WARNING: map '" + map_id + "' is signed by '" + signer
+          + "' with " + missing_certificates.length
+          + " missing certificates: "
+          + missing_certificates + " (TODO: give cleanup instructions)");
+      }
+      if ( redisClient ) that.redis_pool.release(db, redisClient);
+      callback(err, authorized);
+    }
+  );
+};
+
+// Add an authorization certificate from a user.
+//
+// @param signer a signer name (cartodb username)
+// @param cert certificate object, see
+//             http://github.com/CartoDB/Windshaft-cartodb/wiki/Signed-maps
+//
+// @param callback function(err, crt_id) return certificate id
+//
+// TODO: allow for requesting error when certificate already exists ?
+//
+o.addCertificate = function(signer, cert, callback) {
+  var crt_val = JSON.stringify(cert);
+  var crt_id = crypto.createHash('md5').update(crt_val).digest('hex');
+
+  var usr_crt_key = _.template(this.key_map_crt, {signer:signer});
+  this._redisCmd('HSET', [ usr_crt_key, crt_id, crt_val ], function(err, created) {
+    // NOTE: created would be 0 if the field already existed, 1 otherwise
+    callback(err, crt_id);
+  });
+};
+
+// Remove an authorization certificate of a user, also removing
+// any signature made with the certificate.
+//
+// @param signer a signer name (cartodb username)
+// @param crt_id certificate identifier, as returned by addCertificate
+// @param callback function(err)
+//
+o.delCertificate = function(signer, crt_id, callback) {
+  var db = this.db_signatures;
+  var crt_sig_key = _.template(this.key_crt_sig, {signer:signer, crt_id:crt_id});
+  var signed_map_list;
+  var redis_client;
+  var that = this;
+  Step (
+    function getRedisClient() {
+      that._acquireRedis(this);
+    },
+    function removeCertificate(err, data) {
+      if ( err ) throw err;
+      redis_client = data;
+      // Remove the certificate (would be enough to stop authorizing uses)
+      var usr_crt_key = _.template(that.key_map_crt, {signer:signer});
+      redis_client.HDEL(usr_crt_key, crt_id, this);
+    },
+    function getMapSignatures(err, deleted) {
+      if ( err ) throw err;
+      if ( ! deleted ) {
+        // debugging (how can this be possible?)
+        console.log("WARNING: authorization certificate '" + crt_id
+          + "' by user '" + signer + "' did not exist on delete request");
+      }
+      // Get all signatures by this certificate
+      redis_client.SMEMBERS(crt_sig_key, this);
+    },
+    function delMapSignaturesReference(err, map_id_list) {
+      if ( err ) throw err;
+      signed_map_list = map_id_list;
+      if ( debug ) {
+        console.log("Certificate '" + crt_id + "' from user '" + signer
+          + "' was used to sign " + signed_map_list.length + " maps");
+      }
+      redis_client.DEL(crt_sig_key, this);
+    },
+    function delMapSignatures(err) {
+      if ( err ) throw err;
+      var crt_sig_key = _.template(that.key_crt_sig, {signer:signer, crt_id:crt_id});
+      var tx = redis_client.MULTI();
+      for (var i=0; i<signed_map_list.length; ++i) {
+        var map_id = signed_map_list[i];
+        var map_sig_key = _.template(that.key_map_sig, {signer:signer, map_id:map_id});
+        //console.log("Queuing removal of '" + crt_id + "' from '" + map_sig_key + "'");
+        tx.SREM( map_sig_key, crt_id )
+      }
+      tx.EXEC(this);
+    },
+    function reportTransaction(err, rets) {
+      if ( err ) throw err;
+      if ( debug ) {
+        for (var i=0; i<signed_map_list.length; ++i) {
+          var ret = rets[i];
+          if ( ! ret ) {
+            console.log("No signature with certificate '" + crt_id
+              + "' of user '" + signer + "' found in map '"
+              + signed_map_list[i] + "'");
+          } else {
+            console.log("Signature with certificate '" + crt_id
+              + "' of user '" + signer + "' removed from map '"
+              + signed_map_list[i] + "'");
+          }
+        }
+      }
+      return null;
+    },
+    function finish(err) {
+      if ( ! _.isUndefined(redis_client) ) {
+        that._releaseRedis(redis_client);
+      }
+      callback(err);
+    }
+  );
+};
+
+// Sign a map with a certificate reference
+//
+// @param signer a signer name (cartodb username)
+// @param map_id a layergroup_id
+// @param crt_id signature certificate identifier
+//
+// @param callback function(Error) 
+//
+o.signMap = function(signer, map_id, crt_id, callback) {
+  var that = this;
+  Step(
+    function addMapSignature() {
+      var map_sig_key = _.template(that.key_map_sig, {signer:signer, map_id:map_id});
+      if ( debug ) {
+        console.log("Adding " + crt_id + " to " + map_sig_key);
+      }
+      that._redisCmd('SADD', [ map_sig_key, crt_id ], this);
+    },
+    function addCertificateUsage(err) {
+      // Add the map to the set of maps signed by the given cert
+      if ( err ) throw err;
+      var crt_sig_key = _.template(that.key_crt_sig, {signer:signer, crt_id:crt_id});
+      that._redisCmd('SADD', [ crt_sig_key, map_id ], this);
+    },
+    function finish(err) {
+      callback(err);
+    }
+  );
+};
+
+// Sign a map with a full certificate
+//
+// @param signer a signer name (cartodb username)
+// @param map_id a layergroup_id
+// @param cert_id signature certificate identifier
+//
+// @param callback function(Error, String) return certificate id
+//
+o.addSignature = function(signer, map_id, cert, callback) {
+  var that = this;
+  var certificate_id;
+  Step(
+    function addCertificate() {
+      that.addCertificate(signer, cert, this);
+    },
+    function signMap(err, cert_id) {
+      if ( err ) throw err;
+      if ( ! cert_id ) throw new Error("addCertificate returned no certificate id");
+      certificate_id = cert_id;
+      that.signMap(signer, map_id, cert_id, this);
+    },
+    function finish(err) {
+      callback(err, certificate_id);
+    }
+  );
+};
+
+module.exports = SignedMaps;

lib/cartodb/template_maps.js

@@ -0,0 +1,615 @@
+var crypto    = require('crypto');
+var Step      = require('step');
+var _         = require('underscore');
+
+// Templates in this hash (keyed as <username>@<template_name>)
+// are being worked on.
+var user_template_locks = {};
+
+// Class handling map templates
+//
+// See http://github.com/CartoDB/Windshaft-cartodb/wiki/Template-maps
+//
+// @param redis_pool an instance of a "redis-mpool"
+//        See https://github.com/CartoDB/node-redis-mpool
+//        Needs version 0.x.x of the API.
+//
+// @param signed_maps an instance of a "signed_maps" class,
+//        See signed_maps.js
+//
+// @param opts TemplateMap options. Supported elements:
+//    'max_user_templates' limit on the number of per-user
+//
+// 
+function TemplateMaps(redis_pool, signed_maps, opts) {
+  this.redis_pool = redis_pool;
+  this.signed_maps = signed_maps;
+  this.opts = opts || {};
+
+  // Database containing templates
+  // TODO: allow configuring ?
+  // NOTE: currently it is the same as
+  //       the one containing layergroups
+  this.db_signatures = 0;
+
+  //
+  // Map templates are owned by a user that specifies access permissions
+  // for their instances.
+  // 
+  // We have the following datastores:
+  //
+  //  1. User teplates: set of per-user map templates
+  //     NOTE: each template would have an associated auth
+  //           reference, see signed_maps.js
+
+  // User templates (HASH:tpl_id->tpl_val)
+  this.key_usr_tpl = "map_tpl|<%= owner %>";
+
+  // User template locks (HASH:tpl_id->ctime)
+  this.key_usr_tpl_lck = "map_tpl|<%= owner %>|locks";
+
+};
+
+var o = TemplateMaps.prototype;
+
+//--------------- PRIVATE METHODS --------------------------------
+
+o._userTemplateLimit = function() {
+  return this.opts['max_user_templates'] || 0;
+};
+
+o._acquireRedis = function(callback) {
+  this.redis_pool.acquire(this.db_signatures, callback);
+};
+
+o._releaseRedis = function(client) {
+  this.redis_pool.release(this.db_signatures, client);
+};
+
+/**
+ * Internal function to communicate with redis
+ *
+ * @param redisFunc - the redis function to execute
+ * @param redisArgs - the arguments for the redis function in an array
+ * @param callback - function to pass results too.
+ */
+o._redisCmd = function(redisFunc, redisArgs, callback) {
+  var redisClient;
+  var that = this;
+  var db = that.db_signatures;
+
+  Step(
+    function getRedisClient() {
+      that.redis_pool.acquire(db, this);
+    },
+    function executeQuery(err, data) {
+      if ( err ) throw err;
+      redisClient = data;
+      redisArgs.push(this);
+      redisClient[redisFunc.toUpperCase()].apply(redisClient, redisArgs);
+    },
+    function releaseRedisClient(err, data) {
+      if ( ! _.isUndefined(redisClient) ) that.redis_pool.release(db, redisClient);
+      callback(err, data);
+    }
+  );
+};
+
+// @param callback function(err, obtained)
+o._obtainTemplateLock = function(owner, tpl_id, callback) {
+  var usr_tpl_lck_key = _.template(this.key_usr_tpl_lck, {owner:owner});
+  var that = this;
+  var gotLock = false;
+  Step (
+    function obtainLock() {
+      var ctime = Date.now();
+      that._redisCmd('HSETNX', [usr_tpl_lck_key, tpl_id, ctime], this);
+    },
+    function checkLock(err, locked) {
+      if ( err ) throw err;
+      if ( ! locked ) {
+        // Already locked
+        // TODO: unlock if expired ?
+        throw new Error("Template '" + tpl_id + "' of user '" + owner + "' is locked");
+      }
+      return gotLock = true;
+    },
+    function finish(err) {
+      callback(err, gotLock);
+    }
+  );
+};
+
+// @param callback function(err, deleted)
+o._releaseTemplateLock = function(owner, tpl_id, callback) {
+  var usr_tpl_lck_key = _.template(this.key_usr_tpl_lck, {owner:owner});
+  this._redisCmd('HDEL', [usr_tpl_lck_key, tpl_id], callback);
+};
+
+o._reValidIdentifier = /^[a-zA-Z][0-9a-zA-Z_]*$/;
+o._checkInvalidTemplate = function(template) {
+  if ( template.version != '0.0.1' ) {
+    return new Error("Unsupported template version " + template.version);
+  }
+  var tplname = template.name;
+  if ( ! tplname ) {
+    return new Error("Missing template name");
+  }
+  if ( ! tplname.match(this._reValidIdentifier) ) {
+    return new Error("Invalid characters in template name '" + tplname + "'");
+  }
+
+  var phold = template.placeholders;
+  for (var k in phold) {
+    if ( ! k.match(this._reValidIdentifier) ) {
+      return new Error("Invalid characters in placeholder name '" + k + "'");
+    }
+    if ( ! phold[k].hasOwnProperty('default') ) {
+      return new Error("Missing default for placeholder '" + k + "'");
+    }
+    if ( ! phold[k].hasOwnProperty('type') ) {
+      return new Error("Missing type for placeholder '" + k + "'");
+    }
+  };
+
+  // Check certificate validity
+  var cert = this.getTemplateCertificate(template);
+  var err = this.signed_maps.checkInvalidCertificate(cert);
+  if ( err ) return err;
+
+  // TODO: run more checks over template format ?
+};
+
+//--------------- PUBLIC API -------------------------------------
+
+// Extract a signature certificate from a template
+//
+// The certificate will be ready to be passed to
+// SignedMaps.addCertificate or SignedMaps.authorizedByCert
+//
+o.getTemplateCertificate = function(template) {
+  var cert = {
+    version: '0.0.1',
+    template_id: template.name,
+    auth: template.auth
+  };
+  return cert;
+};
+
+// Add a template
+//
+// NOTE: locks user+template_name or fails
+//
+// @param owner cartodb username of the template owner
+//
+// @param template layergroup template, see
+//        http://github.com/CartoDB/Windshaft-cartodb/wiki/Template-maps#template-format
+//
+// @param callback function(err, tpl_id) 
+//        Return template identifier (only valid for given user)
+//
+o.addTemplate = function(owner, template, callback) {
+  var invalidError = this._checkInvalidTemplate(template);
+  if ( invalidError ) {
+    callback(invalidError);
+    return;
+  }
+  var tplname = template.name;
+
+  // Procedure:
+  //
+  // - Check against limit
+  // 0. Obtain a lock for user+template_name, fail if impossible
+  // 1. Check no other template exists with the same name
+  // 2. Install certificate extracted from template, extending
+  //    it to contain a name to properly salt things out.
+  // 3. Modify the template object to reference certificate by id
+  // 4. Install template
+  // 5. Release lock
+  //
+  //
+
+  var usr_tpl_key = _.template(this.key_usr_tpl, {owner:owner});
+  var gotLock = false;
+  var that = this;
+  var limit = that._userTemplateLimit();
+  Step(
+    function checkLimit() {
+      if ( ! limit ) return 0;
+      that._redisCmd('HLEN', [ usr_tpl_key ], this);
+    },
+    // try to obtain a lock
+    function obtainLock(err, len) {
+      if ( err ) throw err;
+      if ( limit && len >= limit ) {
+        throw new Error("User '" + owner + "' reached limit on number of templates (" + len + "/" + limit + ")");
+      }
+      that._obtainTemplateLock(owner, tplname, this);
+    },
+    function getExistingTemplate(err, locked) {
+      if ( err ) throw err;
+      if ( ! locked ) {
+        // Already locked
+        throw new Error("Template '" + tplname + "' of user '" + owner + "' is locked");
+      }
+      gotLock = true;
+      that._redisCmd('HEXISTS', [ usr_tpl_key, tplname ], this);
+    },
+    function installCertificate(err, exists) {
+      if ( err ) throw err;
+      if ( exists ) {
+        throw new Error("Template '" + tplname + "' of user '" + owner + "' already exists"); 
+      }
+      var cert = that.getTemplateCertificate(template);
+      that.signed_maps.addCertificate(owner, cert, this);
+    },
+    function installTemplate(err, crt_id) {
+      if ( err ) throw err;
+      template.auth_id = crt_id;
+      var tpl_val = JSON.stringify(template);
+      that._redisCmd('HSET', [ usr_tpl_key, tplname, tpl_val ], this);
+    },
+    function releaseLock(err, newfield) {
+      if ( ! err && ! newfield ) {
+        console.log("ERROR: addTemplate overridden existing template '"
+          + tplname + "' of '" + owner
+          + "' -- HSET returned " + overridden + ": someone added it without locking ?");
+        // TODO: how to recover this ?!
+      }
+
+      if ( err && ! gotLock ) throw err;
+
+      // release the lock
+      var next = this;
+      that._releaseTemplateLock(owner, tplname, function(e, d) {
+        if ( e ) {
+          console.log("Error removing lock on template '" + tplname
+            + "' of user '" + owner + "': " + e);
+        } else if ( ! d ) {
+          console.log("ERROR: lock on template '" + tplname
+            + "' of user '" + owner + "' externally removed during insert!");
+        }
+        next(err);
+      });
+    },
+    function finish(err) {
+      callback(err, tplname);
+    }
+  );
+};
+
+// Delete a template
+//
+// NOTE: locks user+template_name or fails
+//
+// Also deletes associated authentication certificate, which
+// in turn deletes all instance signatures
+//
+// @param owner cartodb username of the template owner
+//
+// @param tpl_id template identifier as returned
+//        by addTemplate or listTemplates
+//
+// @param callback function(err)
+//
+o.delTemplate = function(owner, tpl_id, callback) {
+  var usr_tpl_key = _.template(this.key_usr_tpl, {owner:owner});
+  var gotLock = false;
+  var that = this;
+  Step(
+    // try to obtain a lock
+    function obtainLock() {
+      that._obtainTemplateLock(owner, tpl_id, this);
+    },
+    function getExistingTemplate(err, locked) {
+      if ( err ) throw err;
+      if ( ! locked ) {
+        // Already locked
+        throw new Error("Template '" + tpl_id + "' of user '" + owner + "' is locked");
+      }
+      gotLock = true;
+      that._redisCmd('HGET', [ usr_tpl_key, tpl_id ], this);
+    },
+    function delCertificate(err, tplval) {
+      if ( err ) throw err;
+      if ( ! tplval ) {
+        throw new Error("Template '" + tpl_id + "' of user '" + owner + "' does not exist"); 
+      }
+      var tpl = JSON.parse(tplval);
+      if ( ! tpl.auth_id ) {
+        // not sure this is an error, in case we'll ever
+        // allow unsigned templates...
+        console.log("ERROR: installed template '" + tpl_id
+            + "' of user '" + owner + "' has no auth_id reference: "); console.dir(tpl);
+        return null;
+      }
+      var next = this;
+      that.signed_maps.delCertificate(owner, tpl.auth_id, function(err) {
+        if ( err ) {
+          var msg = "ERROR: could not delete certificate '"
+                  + tpl.auth_id + "' associated with template '"
+                  + tpl_id + "' of user '" + owner + "': " + err;
+          // I'm actually not sure we want this event to be fatal
+          // (avoiding a deletion of the template itself) 
+          next(new Error(msg));
+        } else {
+          next();
+        }
+      });
+    },
+    function delTemplate(err) {
+      if ( err ) throw err;
+      that._redisCmd('HDEL', [ usr_tpl_key, tpl_id ], this);
+    },
+    function releaseLock(err, deleted) {
+      if ( ! err && ! deleted ) {
+          console.log("ERROR: template '" + tpl_id
+            + "' of user '" + owner + "' externally removed during delete!");
+      }
+
+      if ( ! gotLock ) {
+        if ( err ) throw err;
+        return null;
+      }
+
+      // release the lock
+      var next = this;
+      that._releaseTemplateLock(owner, tpl_id, function(e, d) {
+        if ( e ) {
+          console.log("Error removing lock on template '" + tpl_id
+            + "' of user '" + owner + "': " + e);
+        } else if ( ! d ) {
+          console.log("ERROR: lock on template '" + tpl_id
+            + "' of user '" + owner + "' externally removed during delete!");
+        }
+        next(err);
+      });
+    },
+    function finish(err) {
+      callback(err);
+    }
+  );
+};
+
+// Update a template
+//
+// NOTE: locks user+template_name or fails
+//
+// Also deletes and re-creates associated authentication certificate,
+// which in turn deletes all instance signatures
+//
+// @param owner cartodb username of the template owner
+//
+// @param tpl_id template identifier as returned by addTemplate
+//
+// @param template layergroup template, see
+//        http://github.com/CartoDB/Windshaft-cartodb/wiki/Template-maps#template-format
+//
+// @param callback function(err)
+//        
+o.updTemplate = function(owner, tpl_id, template, callback) {
+
+  var invalidError = this._checkInvalidTemplate(template);
+  if ( invalidError ) {
+    callback(invalidError);
+    return;
+  }
+
+  var tplname = template.name;
+
+  if ( tpl_id != tplname ) {
+    callback(new Error("Cannot update name of a map template ('" + tpl_id + "' != '" + tplname + "')"));
+    return;
+  }
+
+  var usr_tpl_key = _.template(this.key_usr_tpl, {owner:owner});
+  var gotLock = false;
+  var that = this;
+  Step(
+    // try to obtain a lock
+    function obtainLock() {
+      that._obtainTemplateLock(owner, tpl_id, this);
+    },
+    function getExistingTemplate(err, locked) {
+      if ( err ) throw err;
+      if ( ! locked ) {
+        // Already locked
+        throw new Error("Template '" + tpl_id + "' of user '" + owner + "' is locked");
+      }
+      gotLock = true;
+      that._redisCmd('HGET', [ usr_tpl_key, tpl_id ], this);
+    },
+    function delOldCertificate(err, tplval) {
+      if ( err ) throw err;
+      if ( ! tplval ) {
+        throw new Error("Template '" + tpl_id + "' of user '"
+                         + owner +"' does not exist");
+      }
+      var tpl = JSON.parse(tplval);
+      if ( ! tpl.auth_id ) {
+        // not sure this is an error, in case we'll ever
+        // allow unsigned templates...
+        console.log("ERROR: installed template '" + tpl_id
+            + "' of user '" + owner + "' has no auth_id reference: "); console.dir(tpl);
+        return null;
+      }
+      var next = this;
+      that.signed_maps.delCertificate(owner, tpl.auth_id, function(err) {
+        if ( err ) {
+          var msg = "ERROR: could not delete certificate '"
+                  + tpl.auth_id + "' associated with template '"
+                  + tpl_id + "' of user '" + owner + "': " + err;
+          // I'm actually not sure we want this event to be fatal
+          // (avoiding a deletion of the template itself) 
+          next(new Error(msg));
+        } else {
+          next();
+        }
+      });
+    },
+    function installNewCertificate(err) {
+      if ( err ) throw err;
+      var cert = that.getTemplateCertificate(template);
+      that.signed_maps.addCertificate(owner, cert, this);
+    },
+    function updTemplate(err, crt_id) {
+      if ( err ) throw err;
+      template.auth_id = crt_id;
+      var tpl_val = JSON.stringify(template);
+      that._redisCmd('HSET', [ usr_tpl_key, tplname, tpl_val ], this);
+    },
+    function releaseLock(err, newfield) {
+      if ( ! err && newfield ) {
+          console.log("ERROR: template '" + tpl_id
+            + "' of user '" + owner + "' externally removed during update!");
+      }
+
+      if ( ! gotLock ) {
+        if ( err ) throw err;
+        return null;
+      }
+
+      // release the lock
+      var next = this;
+      that._releaseTemplateLock(owner, tpl_id, function(e, d) {
+        if ( e ) {
+          console.log("Error removing lock on template '" + tpl_id
+            + "' of user '" + owner + "': " + e);
+        } else if ( ! d ) {
+          console.log("ERROR: lock on template '" + tpl_id
+            + "' of user '" + owner + "' externally removed during update!");
+        }
+        next(err);
+      });
+    },
+    function finish(err) {
+      callback(err);
+    }
+  );
+};
+
+// List user templates
+//
+// @param owner cartodb username of the templates owner
+//
+// @param callback function(err, tpl_id_list)
+//        Returns a list of template identifiers
+// 
+o.listTemplates = function(owner, callback) {
+  var usr_tpl_key = _.template(this.key_usr_tpl, {owner:owner});
+  this._redisCmd('HKEYS', [ usr_tpl_key ], callback);
+};
+
+// Get a templates
+//
+// @param owner cartodb username of the template owner
+//
+// @param tpl_id template identifier as returned
+//        by addTemplate or listTemplates
+//
+// @param callback function(err, template)
+//        Return full template definition
+//
+o.getTemplate = function(owner, tpl_id, callback) {
+  var usr_tpl_key = _.template(this.key_usr_tpl, {owner:owner});
+  var that = this;
+  Step(
+    function getTemplate() {
+      that._redisCmd('HGET', [ usr_tpl_key, tpl_id ], this);
+    },
+    function parseTemplate(err, tpl_val) {
+      if ( err ) throw err;
+      var tpl = JSON.parse(tpl_val);
+      // Should we strip auth_id ?
+      return tpl;
+    },
+    function finish(err, tpl) {
+      callback(err, tpl);
+    }
+  );
+};
+
+// Perform placeholder substitutions on a template
+//
+// @param template a template object (will not be modified)
+//
+// @param params an object containing named subsitution parameters
+//        Only the ones found in the template's placeholders object
+//        will be used, with missing ones taking default values.
+//
+// @returns a layergroup configuration 
+//
+// @throws Error on malformed template or parameter
+//
+o._reNumber = /^([-+]?[\d\.]?\d+([eE][+-]?\d+)?)$/;
+o._reCSSColorName = /^[a-zA-Z]+$/;
+o._reCSSColorVal = /^#[0-9a-fA-F]{3,6}$/;
+o._replaceVars = function(str, params) {
+  //return _.template(str, params); // lazy way, possibly dangerous 
+  // Construct regular expressions for each param
+  if ( ! params._re ) {
+    params._re = {};
+    for (var k in params) {
+      params._re[k] = RegExp("<%=\\s*" + k + "\\s*%>", "g");
+    }
+  }
+  for (var k in params) str = str.replace(params._re[k], params[k]);
+  return str;
+};
+o.instance = function(template, params) {
+  var all_params = {};
+  var phold = template.placeholders;
+  for (var k in phold) {
+    var val = params.hasOwnProperty(k) ? params[k] : phold[k].default;
+    var type = phold[k].type;
+    // properly escape
+    if ( type === 'sql_literal' ) {
+      // duplicate any single-quote
+      val = val.replace(/'/g, "''");
+    }
+    else if ( type === 'sql_ident' ) {
+      // duplicate any double-quote
+      val = val.replace(/"/g, '""');
+    }
+    else if ( type === 'number' ) {
+      // check it's a number
+      if ( typeof(val) !== 'number' && ! val.match(this._reNumber) ) {
+        throw new Error("Invalid number value for template parameter '"
+          + k + "': " + val);
+      }
+    }
+    else if ( type === 'css_color' ) {
+      // check it only contains letters or
+      // starts with # and only contains hexdigits
+      if ( ! val.match(this._reCSSColorName) && ! val.match(this._reCSSColorVal) ) {
+        throw new Error("Invalid css_color value for template parameter '"
+          + k + "': " + val);
+      }
+    }
+    else {
+      // NOTE: should be checked at template create/update time
+      throw new Error("Invalid placeholder type '" + type + "'");
+    }
+    all_params[k] = val;
+  }
+
+  // NOTE: we're deep-cloning the layergroup here
+  var layergroup = JSON.parse(JSON.stringify(template.layergroup));
+  for (var i=0; i<layergroup.layers.length; ++i) {
+    var lyropt = layergroup.layers[i].options;
+    if ( lyropt.cartocss ) lyropt.cartocss = this._replaceVars(lyropt.cartocss, all_params);
+    if ( lyropt.sql) lyropt.sql = this._replaceVars(lyropt.sql, all_params);
+    // Anything else ?
+  }
+  return layergroup;
+};
+
+// Return a fingerPrint of the object
+o.fingerPrint = function(template) {
+  return crypto.createHash('md5')
+    .update(JSON.stringify(template))
+    .digest('hex')
+  ;
+};
+
+module.exports = TemplateMaps;

npm-shrinkwrap.json

@@ -0,0 +1,479 @@
+{
+  "name": "windshaft-cartodb",
+  "version": "1.8.0",
+  "dependencies": {
+    "node-varnish": {
+      "version": "0.2.0",
+      "from": "http://github.com/Vizzuality/node-varnish/tarball/v0.2.0"
+    },
+    "underscore": {
+      "version": "1.3.3"
+    },
+    "windshaft": {
+      "version": "0.18.2",
+      "from": "http://github.com/CartoDB/Windshaft/tarball/0.18.2",
+      "dependencies": {
+        "grainstore": {
+          "version": "0.18.0",
+          "dependencies": {
+            "mapnik-reference": {
+              "version": "5.0.7"
+            },
+            "millstone": {
+              "version": "0.6.11",
+              "dependencies": {
+                "underscore": {
+                  "version": "1.5.2"
+                },
+                "request": {
+                  "version": "2.26.0",
+                  "dependencies": {
+                    "qs": {
+                      "version": "0.6.6"
+                    },
+                    "json-stringify-safe": {
+                      "version": "5.0.0"
+                    },
+                    "forever-agent": {
+                      "version": "0.5.2"
+                    },
+                    "tunnel-agent": {
+                      "version": "0.3.0"
+                    },
+                    "http-signature": {
+                      "version": "0.10.0",
+                      "dependencies": {
+                        "assert-plus": {
+                          "version": "0.1.2"
+                        },
+                        "asn1": {
+                          "version": "0.1.11"
+                        },
+                        "ctype": {
+                          "version": "0.5.2"
+                        }
+                      }
+                    },
+                    "hawk": {
+                      "version": "1.0.0",
+                      "dependencies": {
+                        "hoek": {
+                          "version": "0.9.1"
+                        },
+                        "boom": {
+                          "version": "0.4.2"
+                        },
+                        "cryptiles": {
+                          "version": "0.2.2"
+                        },
+                        "sntp": {
+                          "version": "0.2.4"
+                        }
+                      }
+                    },
+                    "aws-sign": {
+                      "version": "0.3.0"
+                    },
+                    "oauth-sign": {
+                      "version": "0.3.0"
+                    },
+                    "cookie-jar": {
+                      "version": "0.3.0"
+                    },
+                    "node-uuid": {
+                      "version": "1.4.1"
+                    },
+                    "form-data": {
+                      "version": "0.1.2",
+                      "dependencies": {
+                        "combined-stream": {
+                          "version": "0.0.4",
+                          "dependencies": {
+                            "delayed-stream": {
+                              "version": "0.0.5"
+                            }
+                          }
+                        },
+                        "async": {
+                          "version": "0.2.10"
+                        }
+                      }
+                    }
+                  }
+                },
+                "srs": {
+                  "version": "0.3.10"
+                },
+                "zipfile": {
+                  "version": "0.4.3"
+                },
+                "sqlite3": {
+                  "version": "2.2.0",
+                  "dependencies": {
+                    "node-pre-gyp": {
+                      "version": "0.2.6",
+                      "dependencies": {
+                        "nopt": {
+                          "version": "2.1.2",
+                          "dependencies": {
+                            "abbrev": {
+                              "version": "1.0.4"
+                            }
+                          }
+                        },
+                        "npmlog": {
+                          "version": "0.0.6",
+                          "dependencies": {
+                            "ansi": {
+                              "version": "0.2.1"
+                            }
+                          }
+                        },
+                        "semver": {
+                          "version": "2.1.0"
+                        },
+                        "tar": {
+                          "version": "0.1.19",
+                          "dependencies": {
+                            "inherits": {
+                              "version": "2.0.1"
+                            },
+                            "block-stream": {
+                              "version": "0.0.7"
+                            },
+                            "fstream": {
+                              "version": "0.1.25",
+                              "dependencies": {
+                                "graceful-fs": {
+                                  "version": "2.0.2"
+                                }
+                              }
+                            }
+                          }
+                        },
+                        "tar-pack": {
+                          "version": "2.0.0",
+                          "dependencies": {
+                            "uid-number": {
+                              "version": "0.0.3"
+                            },
+                            "once": {
+                              "version": "1.1.1"
+                            },
+                            "debug": {
+                              "version": "0.7.4"
+                            },
+                            "fstream": {
+                              "version": "0.1.25",
+                              "dependencies": {
+                                "graceful-fs": {
+                                  "version": "2.0.2"
+                                },
+                                "inherits": {
+                                  "version": "2.0.1"
+                                }
+                              }
+                            },
+                            "fstream-ignore": {
+                              "version": "0.0.7",
+                              "dependencies": {
+                                "minimatch": {
+                                  "version": "0.2.14",
+                                  "dependencies": {
+                                    "sigmund": {
+                                      "version": "1.0.0"
+                                    }
+                                  }
+                                },
+                                "inherits": {
+                                  "version": "2.0.1"
+                                }
+                              }
+                            },
+                            "readable-stream": {
+                              "version": "1.0.25-1",
+                              "dependencies": {
+                                "string_decoder": {
+                                  "version": "0.10.25-1"
+                                }
+                              }
+                            },
+                            "graceful-fs": {
+                              "version": "1.2.3"
+                            }
+                          }
+                        },
+                        "aws-sdk": {
+                          "version": "2.0.0-rc9",
+                          "dependencies": {
+                            "xml2js": {
+                              "version": "0.2.4",
+                              "dependencies": {
+                                "sax": {
+                                  "version": "0.6.0"
+                                }
+                              }
+                            },
+                            "xmlbuilder": {
+                              "version": "0.4.2"
+                            }
+                          }
+                        },
+                        "rc": {
+                          "version": "0.3.3",
+                          "dependencies": {
+                            "optimist": {
+                              "version": "0.3.7",
+                              "dependencies": {
+                                "wordwrap": {
+                                  "version": "0.0.2"
+                                }
+                              }
+                            },
+                            "deep-extend": {
+                              "version": "0.2.8"
+                            },
+                            "ini": {
+                              "version": "1.1.0"
+                            }
+                          }
+                        },
+                        "rimraf": {
+                          "version": "2.2.6"
+                        }
+                      }
+                    }
+                  }
+                },
+                "mime": {
+                  "version": "1.2.11"
+                },
+                "mkdirp": {
+                  "version": "0.3.5"
+                },
+                "optimist": {
+                  "version": "0.6.1",
+                  "dependencies": {
+                    "wordwrap": {
+                      "version": "0.0.2"
+                    },
+                    "minimist": {
+                      "version": "0.0.7"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        },
+        "generic-pool": {
+          "version": "2.0.4"
+        },
+        "express": {
+          "version": "2.5.11",
+          "dependencies": {
+            "connect": {
+              "version": "1.9.2",
+              "dependencies": {
+                "formidable": {
+                  "version": "1.0.14"
+                }
+              }
+            },
+            "mime": {
+              "version": "1.2.4"
+            },
+            "qs": {
+              "version": "0.4.2"
+            },
+            "mkdirp": {
+              "version": "0.3.0"
+            }
+          }
+        },
+        "tilelive": {
+          "version": "4.4.3",
+          "dependencies": {
+            "optimist": {
+              "version": "0.3.7",
+              "dependencies": {
+                "wordwrap": {
+                  "version": "0.0.2"
+                }
+              }
+            },
+            "sphericalmercator": {
+              "version": "1.0.2"
+            }
+          }
+        },
+        "tilelive-mapnik": {
+          "version": "0.6.5",
+          "dependencies": {
+            "eio": {
+              "version": "0.2.2"
+            },
+            "mime": {
+              "version": "1.2.11"
+            },
+            "sphericalmercator": {
+              "version": "1.0.2"
+            }
+          }
+        },
+        "lru-cache": {
+          "version": "2.3.1"
+        },
+        "carto": {
+          "version": "0.9.5-cdb2",
+          "from": "http://github.com/CartoDB/carto/tarball/0.9.5-cdb2",
+          "dependencies": {
+            "underscore": {
+              "version": "1.4.4"
+            },
+            "mapnik-reference": {
+              "version": "5.0.7"
+            },
+            "xml2js": {
+              "version": "0.2.8",
+              "dependencies": {
+                "sax": {
+                  "version": "0.5.8"
+                }
+              }
+            },
+            "optimist": {
+              "version": "0.6.1",
+              "dependencies": {
+                "wordwrap": {
+                  "version": "0.0.2"
+                },
+                "minimist": {
+                  "version": "0.0.7"
+                }
+              }
+            }
+          }
+        },
+        "underscore.string": {
+          "version": "1.1.6",
+          "dependencies": {
+            "underscore": {
+              "version": "1.1.7"
+            }
+          }
+        },
+        "pg": {
+          "version": "2.6.2",
+          "dependencies": {
+            "generic-pool": {
+              "version": "2.0.3"
+            },
+            "buffer-writer": {
+              "version": "1.0.0"
+            }
+          }
+        },
+        "torque.js": {
+          "version": "2.2.00"
+        },
+        "node-statsd": {
+          "version": "0.0.7"
+        }
+      }
+    },
+    "step": {
+      "version": "0.0.5"
+    },
+    "request": {
+      "version": "2.9.202"
+    },
+    "cartodb-redis": {
+      "version": "0.3.0"
+    },
+    "redis-mpool": {
+      "version": "0.0.3",
+      "dependencies": {
+        "generic-pool": {
+          "version": "2.0.4"
+        },
+        "hiredis": {
+          "version": "0.1.16",
+          "dependencies": {
+            "bindings": {
+              "version": "1.1.1"
+            }
+          }
+        }
+      }
+    },
+    "mapnik": {
+      "version": "0.7.26"
+    },
+    "lzma": {
+      "version": "1.2.3"
+    },
+    "redis": {
+      "version": "0.8.6"
+    },
+    "strftime": {
+      "version": "0.6.2"
+    },
+    "semver": {
+      "version": "1.1.4"
+    },
+    "mocha": {
+      "version": "1.14.0",
+      "dependencies": {
+        "commander": {
+          "version": "2.0.0"
+        },
+        "growl": {
+          "version": "1.7.0"
+        },
+        "jade": {
+          "version": "0.26.3",
+          "dependencies": {
+            "commander": {
+              "version": "0.6.1"
+            },
+            "mkdirp": {
+              "version": "0.3.0"
+            }
+          }
+        },
+        "diff": {
+          "version": "1.0.7"
+        },
+        "debug": {
+          "version": "0.7.4"
+        },
+        "mkdirp": {
+          "version": "0.3.5"
+        },
+        "glob": {
+          "version": "3.2.3",
+          "dependencies": {
+            "minimatch": {
+              "version": "0.2.14",
+              "dependencies": {
+                "lru-cache": {
+                  "version": "2.5.0"
+                },
+                "sigmund": {
+                  "version": "1.0.0"
+                }
+              }
+            },
+            "graceful-fs": {
+              "version": "2.0.1"
+            },
+            "inherits": {
+              "version": "2.0.1"
+            }
+          }
+        }
+      }
+    }
+  }
+}

package.json

@@ -1,37 +1,42 @@
 {
     "private": true,
     "name": "windshaft-cartodb",
-    "version": "0.1.0",
+    "version": "1.8.0",
     "description": "A map tile server for CartoDB",
-    "url": "https://github.com/Vizzuality/Windshaft-cartodb",
+    "keywords": [
+      "cartodb"
+    ],
+    "url": "https://github.com/CartoDB/Windshaft-cartodb",
     "licenses": [{
         "type": "BSD",
-        "url": "https://github.com/Vizzuality/Windshaft-cartodb/blob/master/LICENCE"
+        "url": "https://github.com/CartoDB/Windshaft-cartodb/blob/master/LICENCE"
     }],
-    "repositories": [{
+    "repository": {
         "type": "git",
-        "url": "git://github.com/Vizzuality/Windshaft-cartodb.git"
-    }],
-    "author": {
-      "name": "Simon Tokumine, Javi Santana, Vizzuality",
-      "url": "http://vizzuality.com",
-      "email": "simon@vizzuality.com"
+        "url": "git://github.com/CartoDB/Windshaft-cartodb.git"
     },
+    "author": "Vizzuality <contact@vizzuality.com> (http://vizzuality.com)",
+    "contributors": [
+      "Simon Tokumine <simon@vizzuality.com>",
+      "Javi Santana <jsantana@vizzuality.com>",
+      "Sandro Santilli <strk@vizzuality.com>"
+    ],
     "dependencies": {
-        "connect": "1.8.7",
-        "cluster": "0.6.4",
-        "node-varnish": "0.1.1",
-        "underscore"  : "1.1.x",
-        "grainstore"  : "~0.6.2",
-        "windshaft"  : "~0.4.16",
+        "node-varnish": "http://github.com/Vizzuality/node-varnish/tarball/v0.2.0",
+        "underscore"  : "~1.3.3",
+        "windshaft"  : "http://github.com/CartoDB/Windshaft/tarball/0.18.2",
         "step": "0.0.x",
-        "generic-pool": "1.0.x",
-        "redis": "0.7.2",
-        "hiredis": "~0.1.12",
-        "request": "2.9.202"
+        "request": "2.9.202",
+        "cartodb-redis": "~0.3.0",
+        "redis-mpool": "~0.0.2",
+        "mapnik": "~0.7.22",
+        "lzma": "~1.2.3"
     },
     "devDependencies": {
-        "mocha": "1.2.1"
+        "mocha": "1.14.0",
+        "redis": "~0.8.3",
+        "strftime": "~0.6.0",
+        "semver": "~1.1.0"
     },
     "scripts": {
         "test": "make check"

run_tests.sh

@@ -1,11 +1,33 @@
 #!/bin/sh
 
-# Must match config.redis_pool.port in test/support/config.js
-REDIS_PORT=6333
+OPT_CREATE_REDIS=yes # create the redis test environment
+OPT_CREATE_PGSQL=yes # create the PostgreSQL test environment
+OPT_DROP_REDIS=yes   # drop the redis test environment
+OPT_DROP_PGSQL=yes   # drop the PostgreSQL test environment
+
+cd $(dirname $0)
+BASEDIR=$(pwd)
+cd -
+
+REDIS_PORT=`node -e "console.log(require('${BASEDIR}/config/environments/test.js').redis.port)"`
+export REDIS_PORT
 
 cleanup() {
-	echo "Cleaning up"
-	kill ${PID_REDIS}
+  if test x"$OPT_DROP_REDIS" = xyes; then
+    if test x"$PID_REDIS" = x; then
+      PID_REDIS=$(cat ${BASEDIR}/redis.pid)
+      if test x"$PID_REDIS" = x; then
+        echo "Could not find a test redis pid to kill it"
+        return;
+      fi
+    fi
+    echo "Killing test redis pid ${PID_REDIS}"
+    kill ${PID_REDIS}
+  fi
+  if test x"$OPT_DROP_PGSQL" = xyes; then
+    # TODO: drop postgresql ?
+    echo "Dropping PostgreSQL test database isn't implemented yet"
+  fi
 }
 
 cleanup_and_exit() {
@@ -22,21 +44,76 @@ die() {
 
 trap 'cleanup_and_exit' 1 2 3 5 9 13
 
-echo "Starting redis on port ${REDIS_PORT}"
-echo "port ${REDIS_PORT}" | redis-server - > test.log &
-PID_REDIS=$!
+while [ -n "$1" ]; do
+        # This is kept for backward compatibility
+        if test "$1" = "--nodrop"; then
+                OPT_DROP_REDIS=no
+                OPT_DROP_PGSQL=no
+                shift
+                continue
+        elif test "$1" = "--nodrop-pg"; then
+                OPT_DROP_PGSQL=no
+                shift
+                continue
+        elif test "$1" = "--nodrop-redis"; then
+                OPT_DROP_REDIS=no
+                shift
+                continue
+        elif test "$1" = "--nocreate-pg"; then
+                OPT_CREATE_PGSQL=no
+                shift
+                continue
+        elif test "$1" = "--nocreate-redis"; then
+                OPT_CREATE_REDIS=no
+                shift
+                continue
+        # This is kept for backward compatibility
+        elif test "$1" = "--nocreate"; then
+                OPT_CREATE_REDIS=no
+                OPT_CREATE_PGSQL=no
+                shift
+                continue
+        else
+                break
+        fi
+done
 
-echo "Preparing the database"
-cd test; sh prepare_db.sh >> test.log || die "database preparation failure (see test.log)"; cd -;
+if [ -z "$1" ]; then
+        echo "Usage: $0 [<options>] <test> [<test>]" >&2
+        echo "Options:" >&2
+        echo " --nocreate   do not create the test environment on start" >&2
+        echo " --nodrop     do not drop the test environment on exit" >&2
+        exit 1
+fi
+
+TESTS=$@
+
+if test x"$OPT_CREATE_REDIS" = xyes; then
+  echo "Starting redis on port ${REDIS_PORT}"
+  echo "port ${REDIS_PORT}" | redis-server - > ${BASEDIR}/test.log &
+  PID_REDIS=$!
+  echo ${PID_REDIS} > ${BASEDIR}/redis.pid
+fi
+
+PREPARE_DB_OPTS=
+if test x"$OPT_CREATE_PGSQL" != xyes; then
+  PREPARE_DB_OPTS="$PREPARE_DB_OPTS --skip-pg"
+fi
+if test x"$OPT_CREATE_REDIS" != xyes; then
+  PREPARE_DB_OPTS="$PREPARE_DB_OPTS --skip-redis"
+fi
+
+echo "Preparing the environment"
+cd ${BASEDIR}/test/support
+sh prepare_db.sh ${PREPARE_DB_OPTS} || die "database preparation failure"
+cd -
 
 PATH=node_modules/.bin/:$PATH
 
 echo "Running tests"
-mocha -u tdd \
-  test/unit/cartodb/redis_pool.test.js \
-  test/unit/cartodb/req2params.test.js \
-  test/acceptance/cache_validator.js \
-  test/acceptance/server.js
-
+mocha -t 10000 -u tdd ${MOCHA_OPTS} ${TESTS}
+ret=$?
 
 cleanup
+
+exit $ret

test/acceptance/cache_validator.js

@@ -1,31 +1,8 @@
 var assert      = require('../support/assert');
-var net = require('net');
 require(__dirname + '/../support/test_helper');
 var CacheValidator = require(__dirname + '/../../lib/cartodb/cache_validator');
-var tests = module.exports = {};
 
-function VarnishEmu(on_cmd_recieved, test_callback) {
-    var self = this;
-    var welcome_msg = 'hi, im a varnish emu, right?';
-
-    self.commands_recieved = [];
-
-    var server = net.createServer(function (socket) {
-      var command = '';
-      socket.write("200 " + welcome_msg.length + "\n");
-      socket.write(welcome_msg);
-      socket.on('data', function(data) {        
-        self.commands_recieved.push(data);
-        on_cmd_recieved && on_cmd_recieved(self.commands_recieved);
-        socket.write('200 0\n');
-      });
-    });
-    server.listen(1337, "127.0.0.1");
-    
-    server.on('listening', function(){
-      test_callback();
-    });    
-}
+var VarnishEmu = require('../support/VarnishEmu');
 
 suite('cache_validator', function() {
 

test/fixtures/test_multilayer_bbox.grid.json

@@ -0,0 +1 @@
+{"grid":["                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"],"keys":["","1"],"data":{"1":{"cartodb_id":1}}}

test/fixtures/test_table_0_0_0_multilayer1.layer0.grid.json

@@ -0,0 +1 @@
+{"grid":["                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                      !!!                       ","                                     !!!!!!                     ","                                    !!!!!!!                     ","                                    !!!!!!!!                    ","                                    !!!!!!!!                    ","                                    !!!!!!!!                    ","                                    !!!!!!!                     ","                                     !!!!!!                     ","                                      !!!                       ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                "],"keys":["","2"],"data":{"2":{"cartodb_id":2}}}

test/fixtures/test_table_0_0_0_multilayer1.layer1.grid.json

@@ -0,0 +1 @@
+{"grid":["                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                       !!                                       ","                      !!!                                       ","                       !!                                       ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                "],"keys":["","2"],"data":{"2":{"cartodb_id":4}}}

test/prepare_db.sh

@@ -1,36 +0,0 @@
-#!/bin/sh
-
-# this script prepare database and redis instance to run accpetance test
-#
-# NOTE: assumes existance of a "template_postgis"
-# NOTE2: use PG* environment variables to control who and where
-#
-# NOTE3: a side effect of the db preparation is the persistent creation
-#        of two database roles which will be valid for the whole cluster
-#        TODO: fix that
-#
-
-die() {
-        msg=$1
-        echo "${msg}" >&2
-        exit 1
-}
-
-TEST_DB="cartodb_test_user_1_db"
-REDIS_PORT=6333
-
-echo "preparing postgres..."
-dropdb "${TEST_DB}"
-createdb -Ttemplate_postgis -EUTF8 "${TEST_DB}" || die "Could not create test database"
-psql "${TEST_DB}" < ./sql/windshaft.test.sql
-psql "${TEST_DB}" < ./sql/gadm4.sql
-
-echo "preparing redis..."
-echo "HSET rails:users:vizzuality id 1" | redis-cli -p ${REDIS_PORT} -n 5
-echo 'HSET rails:users:vizzuality database_name "'"${TEST_DB}"'"' | redis-cli -p ${REDIS_PORT} -n 5
-echo "HSET rails:users:vizzuality map_key 1234" | redis-cli -p ${REDIS_PORT} -n 5
-echo "SADD rails:users:vizzuality:map_key 1235" | redis-cli -p ${REDIS_PORT} -n 5
-echo 'HSET rails:'"${TEST_DB}"':my_table infowindow "this, that, the other"' | redis-cli -p ${REDIS_PORT} -n 0
-echo 'HSET rails:'"${TEST_DB}"':test_table_private_1 privacy "0"' | redis-cli -p ${REDIS_PORT} -n 0
-
-echo "Finished preparing data. Run tests with expresso."

test/support/SQLAPIEmu.js

@@ -0,0 +1,80 @@
+var http        = require('http');
+var url         = require('url');
+var _ = require('underscore');
+
+var o = function(port, cb) {
+
+  this.queries = [];
+  var that = this;
+  this.requests = [];
+
+  this.sqlapi_server = http.createServer(function(req,res) {
+      //console.log("server got request with method " + req.method);
+      var query;
+
+      that.requests.push(req);
+
+      if ( req.method == 'GET' ) {
+        query = url.parse(req.url, true).query;
+        that.handleQuery(query, res);
+      }
+      else if ( req.method == 'POST') {
+        var data = '';
+        req.on('data', function(chunk) {
+          //console.log("GOT Chunk  " + chunk);
+          data += chunk;
+        });
+        req.on('end', function() {
+          //console.log("Data is: "); console.dir(data);
+          query = JSON.parse(data);
+          //console.log("handleQuery is " + that.handleQuery);
+          that.handleQuery(query, res);
+        });
+      }
+      else {
+        that.handleQuery('SQLAPIEmu does not support method' + req.method, res);
+      }
+   }).listen(port, cb);
+};
+
+o.prototype.handleQuery = function(query, res) {
+    this.queries.push(query);
+    if ( query.q.match('SQLAPIERROR') ) {
+      res.statusCode = 400;
+      res.write(JSON.stringify({'error':'Some error occurred'}));
+    } else if ( query.q.match('EPOCH.* as max') ) {
+      // This is the structure of the known query sent by tiler
+      var row = {
+        'max': 1234567890.123
+      };
+      res.write(JSON.stringify({rows: [ row ]}));
+    } else {
+      if ( query.q.match('_private_') && query.api_key === undefined) {
+        res.statusCode = 403;
+        res.write(JSON.stringify({'error':'forbidden: ' + JSON.stringify(query)}));
+      } else {
+        var qs = JSON.stringify(query);
+        var row = {
+          // This is the structure of the known query sent by tiler
+          'cdb_querytables': '{' + qs + '}',
+          'max': qs
+        };
+        var out_obj = {rows: [ row ]};
+        var out = JSON.stringify(out_obj);
+        res.write(out);
+      }
+    }
+    res.end();
+  };
+
+
+o.prototype.close = function(cb) {
+  this.sqlapi_server.close(cb);
+};
+
+o.prototype.getLastRequest = function() {
+  return this.requests.pop();
+};
+
+module.exports = o;
+

test/support/VarnishEmu.js

@@ -0,0 +1,25 @@
+var net = require('net');
+
+module.exports = function(on_cmd_recieved, test_callback) {
+    var self = this;
+    var welcome_msg = 'hi, im a varnish emu, right?';
+
+    self.commands_recieved = [];
+
+    var server = net.createServer(function (socket) {
+      var command = '';
+      socket.write("200 " + welcome_msg.length + "\n");
+      socket.write(welcome_msg);
+      socket.on('data', function(data) {
+        self.commands_recieved.push(data);
+        on_cmd_recieved && on_cmd_recieved(self.commands_recieved);
+        socket.write('200 0\n');
+      });
+    });
+    server.listen(1337, "127.0.0.1");
+
+    server.on('listening', function(){
+      test_callback();
+    });
+};
+

test/support/assert.js

@@ -8,29 +8,93 @@ var exec = require('child_process').exec;
 
 var assert = module.exports = exports = require('assert');
 
-assert.imageEqualsFile = function(buffer, file_b, callback) {
+// @param tolerance number of tolerated grid cell differences
+assert.utfgridEqualsFile = function(buffer, file_b, tolerance, callback) {
+    fs.writeFileSync('/tmp/grid.json', buffer, 'binary'); // <-- to debug/update
+    var expected_json = JSON.parse(fs.readFileSync(file_b, 'utf8'));
+
+    var err = null;
+
+    var Celldiff = function(x, y, ev, ov) {
+      this.x = x;
+      this.y = y;
+      this.ev = ev;
+      this.ov = ov;
+    };
+
+    Celldiff.prototype.toString = function() {
+      return '(' + this.x + ',' + this.y + ')["' + this.ev + '" != "' + this.ov + '"]';
+    };
+
+    try {
+      var obtained_json = JSON.parse(buffer);
+
+      // compare grid
+      var obtained_grid = obtained_json.grid;
+      var expected_grid = expected_json.grid;
+      var nrows = obtained_grid.length
+      if (nrows != expected_grid.length) {
+        throw new Error( "Obtained grid rows (" + nrows +
+                    ") != expected grid rows (" + expected_grid.length + ")" );
+      }
+      var celldiff = [];
+      for (var i=0; i<nrows; ++i) {
+        var ocols = obtained_grid[i];
+        var ecols = expected_grid[i];
+        var ncols = ocols.length;
+        if ( ncols != ecols.length ) {
+          throw new Error( "Obtained grid cols (" + ncols +
+                   ") != expected grid cols (" + ecols.length +
+                   ") on row " + i ); 
+        }
+        for (var j=0; j<ncols; ++j) {
+          var ocell = ocols[j];
+          var ecell = ecols[j];
+          if ( ocell !== ecell ) {
+            celldiff.push(new Celldiff(i, j, ecell, ocell));
+          }
+        }
+      }
+
+      if ( celldiff.length > tolerance ) {
+        throw new Error( celldiff.length + " cell differences: " + celldiff );
+      }
+
+      assert.deepEqual(obtained_json.keys, expected_json.keys);
+    } catch (e) { err = e; }
+
+    callback(err);
+};
+
+//
+// @param tol tolerated color distance as a percent over max channel value
+//            by default this is zero. For meaningful values, see
+//            http://www.imagemagick.org/script/command-line-options.php#metric
+//
+assert.imageEqualsFile = function(buffer, file_b, tol, callback) {
     if (!callback) callback = function(err) { if (err) throw err; };
     file_b = path.resolve(file_b);
-    var file_a = '/tmp/' + (Math.random() * 1e16);
+    var file_a = '/tmp/windshaft-test-image-test.png'; // + (Math.random() * 1e16); // TODO: make predictable 
     var err = fs.writeFileSync(file_a, buffer, 'binary');
     if (err) throw err;
 
-    exec('compare -metric PSNR "' + file_a + '" "' +
+    var fuzz = tol + '%';
+    exec('compare -fuzz ' + fuzz + ' -metric AE "' + file_a + '" "' +
             file_b + '" /dev/null', function(err, stdout, stderr) {
         if (err) {
             fs.unlinkSync(file_a);
             callback(err);
         } else {
             stderr = stderr.trim();
-            if (stderr === 'inf') {
-                fs.unlinkSync(file_a);
-                callback(null);
+            var similarity = parseFloat(stderr);
+            if ( similarity > 0 ) {
+              var err = new Error('Images not equal(' + similarity + '): ' +
+                      file_a  + '    ' + file_b);
+              err.similarity = similarity;
+              callback(err);
             } else {
-                var similarity = parseFloat(stderr);
-                var err = new Error('Images not equal(' + similarity + '): ' +
-                        file_a  + '    ' + file_b);
-                err.similarity = similarity;
-                callback(err);
+              fs.unlinkSync(file_a);
+              callback(null);
             }
         }
     });

test/support/config.js

@@ -1,5 +1,8 @@
 var _ = require('underscore');
 
+
+require(__dirname + '/test_helper');
+
 module.exports = function(opts) {
 
     var config = {
@@ -7,7 +10,7 @@ module.exports = function(opts) {
             max: 10, 
             idleTimeoutMillis: 1, 
             reapIntervalMillis: 1,
-            port: 6333 // TODO: read from test env ?
+            port: global.environment.redis.port
         }
     }
 

test/support/prepare_db.sh

@@ -0,0 +1,111 @@
+#!/bin/sh
+
+# this script prepare database and redis instance to run accpetance test
+#
+# NOTE: assumes existance of a "template_postgis"
+# NOTE2: use PG* environment variables to control who and where
+#
+# NOTE3: a side effect of the db preparation is the persistent creation
+#        of two database roles which will be valid for the whole cluster
+#        TODO: fix that
+#
+
+PREPARE_REDIS=yes
+PREPARE_PGSQL=yes
+
+while [ -n "$1" ]; do
+  if test "$1" = "--skip-pg"; then
+    PREPARE_PGSQL=no
+    shift; continue
+  elif test "$1" = "--skip-redis"; then
+    PREPARE_REDIS=no
+    shift; continue
+  fi
+done
+
+die() {
+        msg=$1
+        echo "${msg}" >&2
+        exit 1
+}
+
+# This is where postgresql connection parameters are read from
+TESTENV=../../config/environments/test.js
+if [ \! -r ${TESTENV} ]; then
+  echo "Cannot read ${TESTENV}" >&2
+  exit 1
+fi
+
+TESTUSERID=1
+
+TESTUSER=`node -e "console.log(require('${TESTENV}').postgres_auth_user || '')"`
+if test -z "$TESTUSER"; then
+  echo "Missing postgres_auth_user from ${TESTENV}" >&2
+  exit 1
+fi
+TESTUSER=`echo ${TESTUSER} | sed "s/<%= user_id %>/${TESTUSERID}/"`
+
+TESTPASS=`node -e "console.log(require('${TESTENV}').postgres_auth_pass || 'test')"`
+# TODO: should postgres_auth_pass be optional ?
+if test -z "$TESTPASS"; then
+  echo "Missing postgres_auth_pass from ${TESTENV}" >&2
+  exit 1
+fi
+TESTPASS=`echo ${TESTPASS} | sed "s/<%= user_id %>/${TESTUSERID}/"`
+
+TEST_DB="${TESTUSER}_db"
+
+# NOTE: will be set by caller trough environment
+if test -z "$REDIS_PORT"; then REDIS_PORT=6333; fi
+
+PUBLICUSER=`node -e "console.log(require('${TESTENV}').postgres.user || 'xxx')"`
+PUBLICPASS=`node -e "console.log(require('${TESTENV}').postgres.password || 'xxx')"`
+echo "PUBLICUSER: ${PUBLICUSER}"
+echo "PUBLICPASS: ${PUBLICPASS}"
+echo "TESTUSER: ${TESTUSER}"
+echo "TESTPASS: ${TESTPASS}"
+
+if test x"$PREPARE_PGSQL" = xyes; then
+
+  echo "preparing postgres..."
+  dropdb "${TEST_DB}"
+  createdb -Ttemplate_postgis -EUTF8 "${TEST_DB}" || die "Could not create test database"
+
+  cat sql/windshaft.test.sql sql/gadm4.sql |
+    sed "s/:PUBLICUSER/${PUBLICUSER}/" | 
+    sed "s/:PUBLICPASS/${PUBLICPASS}/" | 
+    sed "s/:TESTUSER/${TESTUSER}/" | 
+    sed "s/:TESTPASS/${TESTPASS}/" | 
+    psql ${TEST_DB}
+
+fi
+
+if test x"$PREPARE_REDIS" = xyes; then
+
+  echo "preparing redis..."
+
+  cat <<EOF | redis-cli -p ${REDIS_PORT} -n 5
+HMSET rails:users:localhost id ${TESTUSERID} \
+                            database_name '${TEST_DB}' \
+                            map_key 1234
+SADD rails:users:localhost:map_key 1235
+EOF
+
+  # A user configured as with cartodb-2.5.0+ 
+  cat <<EOF | redis-cli -p ${REDIS_PORT} -n 5
+HMSET rails:users:cartodb250user id ${TESTUSERID} \
+                                 database_name "${TEST_DB}" \
+                                 database_host "localhost" \
+                                 database_password "${TESTPASS}" \
+                                 map_key 4321
+EOF
+
+  cat <<EOF | redis-cli -p ${REDIS_PORT} -n 0
+HSET rails:${TEST_DB}:my_table infowindow "this, that, the other"
+HSET rails:${TEST_DB}:test_table_private_1 privacy "0"
+EOF
+
+fi
+
+echo "Finished preparing data. Ready to run tests"
+

test/support/sql/windshaft.test.sql

@@ -15,11 +15,13 @@ SET search_path = public, pg_catalog;
 SET default_tablespace = '';
 SET default_with_oids = false;
 
--- publicuser role
-CREATE USER publicuser;
+-- public user role
+DROP USER IF EXISTS :PUBLICUSER;
+CREATE USER :PUBLICUSER WITH PASSWORD ':PUBLICPASS';
 
 -- db owner role
-CREATE USER test_cartodb_user_1;
+DROP USER IF EXISTS :TESTUSER;
+CREATE USER :TESTUSER WITH PASSWORD ':TESTPASS';
 
 -- first table
 CREATE TABLE test_table (
@@ -51,19 +53,20 @@ SELECT pg_catalog.setval('test_table_cartodb_id_seq', 60, true);
 
 ALTER TABLE test_table ALTER COLUMN cartodb_id SET DEFAULT nextval('test_table_cartodb_id_seq'::regclass);
 
-INSERT INTO test_table VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241');
-INSERT INTO test_table VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241');
-INSERT INTO test_table VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.324', 3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241');
-INSERT INTO test_table VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.329509', 4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241');
-INSERT INTO test_table VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.334931', 5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241');
+INSERT INTO test_table VALUES
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.324', 3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.329509', 4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.334931', 5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241');
 
 ALTER TABLE ONLY test_table ADD CONSTRAINT test_table_pkey PRIMARY KEY (cartodb_id);
 
 CREATE INDEX test_table_the_geom_idx ON test_table USING gist (the_geom);
 CREATE INDEX test_table_the_geom_webmercator_idx ON test_table USING gist (the_geom_webmercator);
 
-GRANT ALL ON TABLE test_table TO test_cartodb_user_1;
-GRANT SELECT ON TABLE test_table TO publicuser;
+GRANT ALL ON TABLE test_table TO :TESTUSER;
+GRANT SELECT ON TABLE test_table TO :PUBLICUSER;
 
 -- second table
 CREATE TABLE test_table_2 (
@@ -95,19 +98,20 @@ SELECT pg_catalog.setval('test_table_2_cartodb_id_seq', 60, true);
 
 ALTER TABLE test_table_2 ALTER COLUMN cartodb_id SET DEFAULT nextval('test_table_2_cartodb_id_seq'::regclass);
 
-INSERT INTO test_table_2 VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241');
-INSERT INTO test_table_2 VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241');
-INSERT INTO test_table_2 VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.324', 3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241');
-INSERT INTO test_table_2 VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.329509', 4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241');
-INSERT INTO test_table_2 VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.334931', 5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241');
+INSERT INTO test_table_2 VALUES
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.324', 3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.329509', 4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.334931', 5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241');
 
 ALTER TABLE ONLY test_table_2 ADD CONSTRAINT test_table_2_pkey PRIMARY KEY (cartodb_id);
 
 CREATE INDEX test_table_2_the_geom_idx ON test_table_2 USING gist (the_geom);
 CREATE INDEX test_table_2_the_geom_webmercator_idx ON test_table_2 USING gist (the_geom_webmercator);
 
-GRANT ALL ON TABLE test_table_2 TO test_cartodb_user_1;
-GRANT SELECT ON TABLE test_table_2 TO publicuser;
+GRANT ALL ON TABLE test_table_2 TO :TESTUSER;
+GRANT SELECT ON TABLE test_table_2 TO :PUBLICUSER;
 
 -- third table
 CREATE TABLE test_table_3 (
@@ -139,19 +143,20 @@ SELECT pg_catalog.setval('test_table_3_cartodb_id_seq', 60, true);
 
 ALTER TABLE test_table_3 ALTER COLUMN cartodb_id SET DEFAULT nextval('test_table_3_cartodb_id_seq'::regclass);
 
-INSERT INTO test_table_3 VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241');
-INSERT INTO test_table_3 VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241');
-INSERT INTO test_table_3 VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.324', 3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241');
-INSERT INTO test_table_3 VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.329509', 4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241');
-INSERT INTO test_table_3 VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.334931', 5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241');
+INSERT INTO test_table_3 VALUES
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.324', 3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.329509', 4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.334931', 5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241');
 
 ALTER TABLE ONLY test_table_3 ADD CONSTRAINT test_table_3_pkey PRIMARY KEY (cartodb_id);
 
 CREATE INDEX test_table_3_the_geom_idx ON test_table_3 USING gist (the_geom);
 CREATE INDEX test_table_3_the_geom_webmercator_idx ON test_table_3 USING gist (the_geom_webmercator);
 
-GRANT ALL ON TABLE test_table_3 TO test_cartodb_user_1;
-GRANT SELECT ON TABLE test_table_3 TO publicuser;
+GRANT ALL ON TABLE test_table_3 TO :TESTUSER;
+GRANT SELECT ON TABLE test_table_3 TO :PUBLICUSER;
 
 -- private table
 CREATE TABLE test_table_private_1 (
@@ -169,5 +174,6 @@ CREATE TABLE test_table_private_1 (
     CONSTRAINT enforce_srid_the_geom CHECK ((st_srid(the_geom) = 4326)),
     CONSTRAINT enforce_srid_the_geom_webmercator CHECK ((st_srid(the_geom_webmercator) = 3857))
 );
+INSERT INTO test_table_private_1 SELECT * from test_table;
 
-GRANT ALL ON TABLE test_table_private_1 TO test_cartodb_user_1;
+GRANT ALL ON TABLE test_table_private_1 TO :TESTUSER;

test/support/test_helper.js

@@ -6,6 +6,7 @@
  */
 
 var _ = require('underscore');
+var LZMA  = require('lzma/lzma_worker.js').LZMA;
 
 // set environment specific variables
 global.settings     = require(__dirname + '/../../config/settings');
@@ -13,4 +14,21 @@ global.environment  = require(__dirname + '/../../config/environments/test');
 _.extend(global.settings, global.environment);
 
 
+// Utility function to compress & encode LZMA
+function lzma_compress_to_base64(payload, mode, callback) {
+  LZMA.compress(payload, mode, 
+    function(ints) {
+      ints = ints.map(function(c) { return String.fromCharCode(c + 128) }).join('')
+      var base64 = new Buffer(ints, 'binary').toString('base64');
+      callback(null, base64);
+    },
+    function(percent) {
+      //console.log("Compressing: " + percent + "%");
+    }
+  );
+}
+
+module.exports = {
+  lzma_compress_to_base64: lzma_compress_to_base64
+}
 

test/unit/cartodb/redis_pool.test.js

@@ -1,61 +0,0 @@
-var assert = require('../../support/assert')
-  , _ = require('underscore')
-  , RedisPool = require('../../../lib/cartodb/redis_pool')
-  , tests = module.exports = {};
-
-suite('redis_pool', function() {
-
-    // configure redis pool instance to use in tests
-    var test_opts = require('../../support/config').redis_pool;
-    
-    var redis_pool = new RedisPool(test_opts);
-
-    test('RedisPool object exists', function(done){
-      assert.ok(RedisPool);
-      done();
-    });
-    
-    test('RedisPool can create new redis_pool objects with default settings', function(done){
-      var redis_pool = new RedisPool();
-      done();
-    });
-    
-    test('RedisPool can create new redis_pool objects with specific settings', function(done){
-      var redis_pool = new RedisPool(_.extend({host:'127.0.0.1', port: '6379'}, test_opts));
-      done();
-    });
-    
-    
-    test('pool object has an acquire function', function(done){
-      var found=false;
-      var functions = _.functions(redis_pool);
-      for (var i=0; i<functions.length; ++i) {
-          if ( functions[i] == 'acquire' ) { found=true; break; }
-      }
-      assert.ok(found);
-      done();
-    });
-    
-    test('calling aquire returns a redis client object that can get/set', function(done){
-      redis_pool.acquire(0, function(err, client){
-        client.set("key","value");
-        client.get("key", function(err,data){      
-          assert.equal(data, "value");      
-          redis_pool.release(0, client); // needed to exit tests
-          done();
-        })
-      });    
-    });
-    
-    test('calling aquire on another DB returns a redis client object that can get/set', function(done){
-      redis_pool.acquire(2, function(err, client){
-        client.set("key","value");
-        client.get("key", function(err,data){      
-          assert.equal(data, "value");      
-          redis_pool.release(2, client); // needed to exit tests
-          done();
-        })
-      });      
-    });
-
-});

test/unit/cartodb/req2params.test.js

@@ -14,53 +14,73 @@ suite('req2params', function() {
     });
 
     test('cleans up request', function(done){
-      opts.req2params({headers: { host:'h1' }, query: {dbuser:'hacker',dbname:'secret'}}, function(err, req) {
-          if ( err ) { console.log(err); throw new Error(err); }
+      opts.req2params({headers: { host:'localhost' }, query: {dbuser:'hacker',dbname:'secret'}}, function(err, req) {
+          if ( err ) { done(err); return; }
           assert.ok(_.isObject(req.query), 'request has query');
           assert.ok(!req.query.hasOwnProperty('dbuser'), 'dbuser was removed from query');
           assert.ok(req.hasOwnProperty('params'), 'request has params');
           assert.ok(req.params.hasOwnProperty('interactivity'), 'request params have interactivity');
-          assert.ok(_.isNull(req.params.dbname), 'could forge dbname');
-          assert.ok(!req.hasOwnProperty('dbuser'), 'could inject dbuser ('+req.params.dbuser+')');
+          assert.equal(req.params.dbname, 'test_cartodb_user_1_db', 'could forge dbname: '+ req.params.dbname);
+          assert.ok(req.params.dbuser === 'testpublicuser', 'could inject dbuser ('+req.params.dbuser+')');
           done();
       });
     });
 
     test('sets dbname from redis metadata', function(done){
-      opts.req2params({headers: { host:'vizzuality' }, query: {} }, function(err, req) {
-          if ( err ) { console.log(err); throw new Error(err); }
+      opts.req2params({headers: { host:'localhost' }, query: {} }, function(err, req) {
+          if ( err ) { done(err); return; }
           //console.dir(req);
           assert.ok(_.isObject(req.query), 'request has query');
           assert.ok(!req.query.hasOwnProperty('dbuser'), 'dbuser was removed from query');
           assert.ok(req.hasOwnProperty('params'), 'request has params');
           assert.ok(req.params.hasOwnProperty('interactivity'), 'request params have interactivity');
-          // database_name for user "vizzuality" (see test/support/prepare_db.sh)
-          assert.equal(req.params.dbname, 'cartodb_test_user_1_db');
+          // database_name for user "localhost" (see test/support/prepare_db.sh)
+          assert.equal(req.params.dbname, 'test_cartodb_user_1_db');
           // unauthenticated request gets no dbuser
-          assert.ok(!req.params.hasOwnProperty('dbuser'), 'could inject dbuser ('+req.params.dbuser+')');
+          assert.ok(req.params.dbuser === 'testpublicuser', 'could inject dbuser ('+req.params.dbuser+')');
           done();
       });
     });
 
     test('sets also dbuser for authenticated requests', function(done){
-      opts.req2params({headers: { host:'vizzuality' }, query: {map_key: '1234'} }, function(err, req) {
-          if ( err ) { console.log(err); throw new Error(err); }
+      opts.req2params({headers: { host:'localhost' }, query: {map_key: '1234'} }, function(err, req) {
+          if ( err ) { done(err); return; }
           //console.dir(req);
           assert.ok(_.isObject(req.query), 'request has query');
           assert.ok(!req.query.hasOwnProperty('dbuser'), 'dbuser was removed from query');
           assert.ok(req.hasOwnProperty('params'), 'request has params');
           assert.ok(req.params.hasOwnProperty('interactivity'), 'request params have interactivity');
-          // database_name for user "vizzuality" (see test/support/prepare_db.sh)
-          assert.equal(req.params.dbname, 'cartodb_test_user_1_db');
-          // id for user "vizzuality" (see test/support/prepare_db.sh)
-          assert.equal(req.dbuser, 'test_cartodb_user_1');
+          // database_name for user "localhost" (see test/support/prepare_db.sh)
+          assert.equal(req.params.dbname, 'test_cartodb_user_1_db');
+          // id for user "localhost" (see test/support/prepare_db.sh)
+          assert.equal(req.params.dbuser, 'test_cartodb_user_1');
  
-          opts.req2params({headers: { host:'vizzuality' }, query: {map_key: '1235'} }, function(err, req) {
+          opts.req2params({headers: { host:'localhost' }, query: {map_key: '1235'} }, function(err, req) {
               // wrong key resets params to no user
-              assert.ok(!req.hasOwnProperty('dbuser'), 'could inject dbuser ('+req.params.dbuser+')');
+              assert.ok(req.params.dbuser === 'testpublicuser', 'could inject dbuser ('+req.params.dbuser+')');
               done();
           });
       });
     });
+
+    test('it should extend params with decoded lzma', function(done) {
+      var qo = {
+        style: 'test',
+        style_version: '2.1.0',
+        cache_buster: 5
+      };
+      test_helper.lzma_compress_to_base64(JSON.stringify(qo), 1, function(err, data) {
+        opts.req2params({ headers: { host:'localhost' }, query: { non_included: 'toberemoved', api_key: 'test', style: 'override', lzma: data }}, function(err, req) {
+          if ( err ) { done(err); return; }
+          var query = req.params
+          assert.equal(qo.style, query.style)
+          assert.equal(qo.style_version, query.style_version)
+          assert.equal(qo.cache_buster, query.cache_buster)
+          assert.equal('test', query.api_key)
+          assert.equal(undefined, query.non_included)
+          done();
+        });
+      });
+    });
     
 });

test/unit/cartodb/signed_maps.test.js

@@ -0,0 +1,109 @@
+var assert = require('assert')
+  //, _ = require('underscore')
+  , RedisPool = require('redis-mpool')
+  , SignedMaps = require('../../../lib/cartodb/signed_maps.js')
+  , test_helper = require('../../support/test_helper')
+  , Step = require('step')
+  , tests = module.exports = {};
+
+suite('signed_maps', function() {
+
+    // configure redis pool instance to use in tests
+    var redis_pool = RedisPool(global.environment.redis);
+    
+    test('can sign map with open and token-based auth', function(done) {
+      var smap = new SignedMaps(redis_pool);
+      assert.ok(smap);
+      var sig = 'sig1';
+      var map = 'map1';
+      var tok = 'tok1';
+      var crt = {
+        version:'0.0.1',
+        layergroup_id:map,
+        auth: {}
+      };
+      var crt1_id; // by token
+      var crt2_id; // open
+      Step(
+        function() {
+          smap.isAuthorized(sig,map,tok,this);
+        },
+        function checkAuthFailure1(err, authorized) {
+          if ( err ) throw err;
+          assert.ok(!authorized, "unexpectedly authorized");
+          crt.auth.method = 'token';
+          crt.auth.valid_tokens = [tok];
+          smap.addSignature(sig, map, crt, this)
+        },
+        function getCert1(err, id) {
+          if ( err ) throw err;
+          assert.ok(id, "undefined signature id");
+          crt1_id = id; // keep note of it
+//console.log("Certificate 1 is " + crt1_id);
+          smap.isAuthorized(sig,map,'',this);
+        },
+        function checkAuthFailure2(err, authorized) {
+          if ( err ) throw err;
+          assert.ok(!authorized, "unexpectedly authorized");
+          smap.isAuthorized(sig,map,tok,this);
+        },
+        function checkAuthSuccess1(err, authorized) {
+          if ( err ) throw err;
+          assert.ok(authorized, "unauthorized :(");
+          crt.auth.method = 'open';
+          delete crt.auth.valid_tokens;
+          smap.addSignature(sig, map, crt, this)
+        },
+        function getCert2(err, id) {
+          if ( err ) throw err;
+          assert.ok(id, "undefined signature id");
+          crt2_id = id; // keep note of it
+//console.log("Certificate 2 is " + crt2_id);
+          smap.isAuthorized(sig,map,'arbitrary',this);
+        },
+        function checkAuthSuccess2_delCert2(err, authorized) {
+          if ( err ) throw err;
+          assert.ok(authorized, "unauthorized :(");
+          var next = this;
+          smap.delCertificate(sig, crt2_id, function(e) {
+            if (e) next(e);
+            else smap.isAuthorized(sig,map,'arbitrary',next);
+          });
+        },
+        function checkAuthFailure3_delCert2(err, authorized) {
+          if ( err ) throw err;
+          assert.ok(!authorized, "unexpectedly authorized");
+          smap.delCertificate(sig, crt1_id, this);
+        },
+        function finish(err) {
+          done(err);
+        }
+      );
+    });
+
+    test('can validate certificates', function(done) {
+      var smap = new SignedMaps(redis_pool);
+      assert.ok(smap);
+      Step(
+        function invalidVersion() {
+          var cert = { version: '-1' };
+          var err = smap.checkInvalidCertificate(cert);
+          assert.ok(err);
+          assert.equal(err.message, "Unsupported certificate version -1");
+          return null;
+        },
+        function invalidTokenAuth() {
+          var cert = { version: '0.0.1', auth: { method:'token', valid_token:[] } };
+          var err = smap.checkInvalidCertificate(cert);
+          assert.ok(err);
+          assert.equal(err.message, "Invalid 'token' authentication: missing valid_tokens");
+          return null;
+        },
+        function finish(err) {
+          done(err);
+        }
+      );
+    });
+
+    
+});

test/unit/cartodb/template_maps.test.js

@@ -0,0 +1,506 @@
+var assert = require('assert')
+  //, _ = require('underscore')
+  , RedisPool = require('redis-mpool')
+  , SignedMaps = require('../../../lib/cartodb/signed_maps.js')
+  , TemplateMaps = require('../../../lib/cartodb/template_maps.js')
+  , test_helper = require('../../support/test_helper')
+  , Step = require('step')
+  , tests = module.exports = {};
+
+suite('template_maps', function() {
+
+  // configure redis pool instance to use in tests
+  var redis_pool = RedisPool(global.environment.redis);
+  var signed_maps = new SignedMaps(redis_pool);
+    
+  test('does not accept template with unsupported version', function(done) {
+    var tmap = new TemplateMaps(redis_pool, signed_maps);
+    assert.ok(tmap);
+    var tpl = { version:'6.6.6',
+      name:'k', auth: {}, layergroup: {} };
+    Step(
+      function() {
+        tmap.addTemplate('me', tpl, this);
+      },
+      function checkFailed(err) {
+        assert.ok(err);
+        assert.ok(err.message.match(/unsupported.*version/i), err);
+        return null;
+      },
+      function finish(err) {
+        done(err);
+      }
+    );
+  });
+
+  test('does not accept template with missing name', function(done) {
+    var tmap = new TemplateMaps(redis_pool, signed_maps);
+    assert.ok(tmap);
+    var tpl = { version:'0.0.1',
+      auth: {}, layergroup: {} };
+    Step(
+      function() {
+        tmap.addTemplate('me', tpl, this);
+      },
+      function checkFailed(err) {
+        assert.ok(err);
+        assert.ok(err.message.match(/missing.*name/i), err);
+        return null;
+      },
+      function finish(err) {
+        done(err);
+      }
+    );
+  });
+
+  test('does not accept template with invalid name', function(done) {
+    var tmap = new TemplateMaps(redis_pool, signed_maps);
+    assert.ok(tmap);
+    var tpl = { version:'0.0.1',
+      auth: {}, layergroup: {} };
+    var invalidnames = [ "ab|", "a b", "a@b", "1ab", "_x", "", " x", "x " ];
+    var testNext = function() {
+      if ( ! invalidnames.length ) { done(); return; }
+      var n = invalidnames.pop();
+      tpl.name = n;
+      tmap.addTemplate('me', tpl, function(err) {
+        if ( ! err ) {
+          done(new Error("Unexpected success with invalid name '" + n + "'"));
+        }
+        else if ( ! err.message.match(/template.*name/i) ) {
+          done(new Error("Unexpected error message with invalid name '" + n
+            + "': " + err));
+        }
+        else {
+          testNext();
+        }
+      });
+    };
+    testNext();
+  });
+
+  test('does not accept template with invalid placeholder name', function(done) {
+    var tmap = new TemplateMaps(redis_pool, signed_maps);
+    assert.ok(tmap);
+    var tpl = { version:'0.0.1',
+      name: "valid", placeholders: {},
+      auth: {}, layergroup: {} };
+    var invalidnames = [ "ab|", "a b", "a@b", "1ab", "_x", "", " x", "x " ];
+    var testNext = function() {
+      if ( ! invalidnames.length ) { done(); return; }
+      var n = invalidnames.pop();
+      tpl.placeholders = {};
+      tpl.placeholders[n] = { type:'number', default:1 };
+      tmap.addTemplate('me', tpl, function(err) {
+        if ( ! err ) {
+          done(new Error("Unexpected success with invalid name '" + n + "'"));
+        }
+        else if ( ! err.message.match(/invalid.*name/i) ) {
+          done(new Error("Unexpected error message with invalid name '" + n
+            + "': " + err));
+        }
+        else {
+          testNext();
+        }
+      });
+    };
+    testNext();
+  });
+
+  test('does not accept template with missing placeholder default', function(done) {
+    var tmap = new TemplateMaps(redis_pool, signed_maps);
+    assert.ok(tmap);
+    var tpl = { version:'0.0.1',
+      name: "valid", placeholders: { v: {} },
+      auth: {}, layergroup: {} };
+    tmap.addTemplate('me', tpl, function(err) {
+        if ( ! err ) {
+          done(new Error("Unexpected success with missing placeholder default"));
+        }
+        else if ( ! err.message.match(/missing default/i) ) {
+          done(new Error("Unexpected error message with missing placeholder default: "
+            + err));
+        }
+        else {
+          done();
+        }
+      });
+  });
+
+  test('does not accept template with missing placeholder type', function(done) {
+    var tmap = new TemplateMaps(redis_pool, signed_maps);
+    assert.ok(tmap);
+    var tpl = { version:'0.0.1',
+      name: "valid", placeholders: { v: { default:1 } },
+      auth: {}, layergroup: {} };
+    tmap.addTemplate('me', tpl, function(err) {
+        if ( ! err ) {
+          done(new Error("Unexpected success with missing placeholder type"));
+        }
+        else if ( ! err.message.match(/missing type/i) ) {
+          done(new Error("Unexpected error message with missing placeholder default: "
+            + err));
+        }
+        else {
+          done();
+        }
+      });
+  });
+
+  // See http://github.com/CartoDB/Windshaft-cartodb/issues/128
+  test('does not accept template with invalid token auth (undefined tokens)',
+  function(done) {
+    var tmap = new TemplateMaps(redis_pool, signed_maps);
+    assert.ok(tmap);
+    var tpl = { version:'0.0.1',
+      name: "invalid_auth1", placeholders: { },
+      auth: { method: 'token' }, layergroup: {} };
+    tmap.addTemplate('me', tpl, function(err) {
+        if ( ! err ) {
+          done(new Error("Unexpected success with invalid token auth (undefined tokens)"));
+        }
+        else if ( ! err.message.match(/invalid 'token' authentication/i) ) {
+          done(new Error("Unexpected error message with invalid token auth (undefined tokens): "
+            + err));
+        }
+        else {
+          done();
+        }
+      });
+  });
+
+  test('add, get and delete a valid template', function(done) {
+    var tmap = new TemplateMaps(redis_pool, signed_maps);
+    assert.ok(tmap);
+    var expected_failure = false;
+    var tpl_id;
+    var tpl = { version:'0.0.1',
+      name: 'first', auth: {}, layergroup: {} };
+    Step(
+      function() {
+        tmap.addTemplate('me', tpl, this);
+      },
+      function addOmonimousTemplate(err, id) {
+        if ( err ) throw err;
+        tpl_id = id;
+        assert.equal(tpl_id, 'first');
+        expected_failure = true;
+        // should fail, as it already exists
+        tmap.addTemplate('me', tpl, this);
+      },
+      function getTemplate(err) {
+        if ( ! expected_failure && err ) throw err;
+        assert.ok(err);
+        assert.ok(err.message.match(/already exists/i), err);
+        tmap.getTemplate('me', tpl_id, this);
+      },
+      function delTemplate(err, got_tpl) {
+        if ( err ) throw err;
+        assert.deepEqual(got_tpl, tpl);
+        tmap.delTemplate('me', tpl_id, this);
+      },
+      function finish(err) {
+        done(err);
+      }
+    );
+  });
+
+  test('add multiple templates, list them', function(done) {
+    var tmap = new TemplateMaps(redis_pool, signed_maps);
+    assert.ok(tmap);
+    var expected_failure = false;
+    var tpl1 = { version:'0.0.1', name: 'first', auth: {}, layergroup: {} };
+    var tpl1_id;
+    var tpl2 = { version:'0.0.1', name: 'second', auth: {}, layergroup: {} };
+    var tpl2_id;
+    Step(
+      function addTemplate1() {
+        tmap.addTemplate('me', tpl1, this);
+      },
+      function addTemplate2(err, id) {
+        if ( err ) throw err;
+        tpl1_id = id;
+        tmap.addTemplate('me', tpl2, this);
+      },
+      function listTemplates(err, id) {
+        if ( err ) throw err;
+        tpl2_id = id;
+        tmap.listTemplates('me', this);
+      },
+      function checkTemplates(err, ids) {
+        if ( err ) throw err;
+        assert.equal(ids.length, 2);
+        assert.ok(ids.indexOf(tpl1_id) != -1, ids.join(','));
+        assert.ok(ids.indexOf(tpl2_id) != -1, ids.join(','));
+        return null;
+      },
+      function delTemplate1(err) {
+        if ( tpl1_id ) {
+          var next = this;
+          tmap.delTemplate('me', tpl1_id, function(e) {
+            if ( err || e ) next(new Error(err + '; ' + e));
+            else next();
+          });
+        } else {
+          if ( err ) throw err;
+          return null;
+        }
+      },
+      function delTemplate2(err) {
+        if ( tpl2_id ) {
+          var next = this;
+          tmap.delTemplate('me', tpl2_id, function(e) {
+            if ( err || e ) next(new Error(err + '; ' + e));
+            else next();
+          });
+        } else {
+          if ( err ) throw err;
+          return null;
+        }
+      },
+      function finish(err) {
+        done(err);
+      }
+    );
+  });
+
+  test('update templates', function(done) {
+    var tmap = new TemplateMaps(redis_pool, signed_maps);
+    assert.ok(tmap);
+    var expected_failure = false;
+    var owner = 'me';
+    var tpl = { version:'0.0.1',
+      name: 'first',
+      auth: { method: 'open' },
+      layergroup: {}
+    };
+    var tpl_id;
+    Step(
+      function addTemplate() {
+        tmap.addTemplate(owner, tpl, this);
+      },
+      // Updating template name should fail
+      function updateTemplateName(err, id) {
+        if ( err ) throw err;
+        tpl_id = id;
+        expected_failure = true;
+        tpl.name = 'second';
+        tmap.updTemplate(owner, tpl_id, tpl, this);
+      },
+      function updateTemplateAuth(err) {
+        if ( err && ! expected_failure) throw err;
+        expected_failure = false;
+        assert.ok(err);
+        tpl.name = 'first';
+        tpl.auth.method = 'token';
+        tpl.auth.valid_tokens = [ 'tok1' ];
+        tmap.updTemplate(owner, tpl_id, tpl, this);
+      },
+      function updateTemplateWithInvalid(err) {
+        if ( err ) throw err;
+        tpl.version = '999.999.999';
+        expected_failure = true;
+        tmap.updTemplate(owner, tpl_id, tpl, this);
+      },
+      function updateUnexistentTemplate(err) {
+        if ( err && ! expected_failure) throw err;
+        expected_failure = false;
+        assert.ok(err);
+        assert.ok(err.message.match(/unsupported.*version/i), err);
+        tpl.version = '0.0.1';
+        expected_failure = true;
+        tmap.updTemplate(owner, 'unexistent', tpl, this);
+      },
+      function delTemplate(err) {
+        if ( err && ! expected_failure) throw err;
+        expected_failure = false;
+        assert.ok(err);
+        assert.ok(err.message.match(/cannot update name/i), err);
+        tmap.delTemplate(owner, tpl_id, this);
+      },
+      function finish(err) {
+        done(err);
+      }
+    );
+  });
+
+  test('instanciate templates', function() {
+    var tmap = new TemplateMaps(redis_pool, signed_maps);
+    assert.ok(tmap);
+
+    var tpl1 =  {
+        version: '0.0.1',
+        name: 'acceptance1',
+        auth: { method: 'open' },
+        placeholders: {
+          fill: { type: "css_color", default: "red" },
+          color: { type: "css_color", default: "#a0fF9A" },
+          name: { type: "sql_literal", default: "test" },
+          zoom: { type: "number", default: "0" },
+          test_number: { type: "number", default: 23 },
+        },
+        layergroup: {
+          version: '1.0.0',
+          global_cartocss_version: '2.0.2',
+          layers: [
+             { options: {
+                 sql: "select '<%=name %>' || id, g from t",
+                 cartocss: '#layer { marker-fill:<%= fill %>; marker-width: <%=test_number  %>; }'
+               } },
+             { options: {
+                 sql: "select fun('<%=     name%>') g from x",
+                 cartocss: '#layer { line-color:<%= color %>; marker-fill:<%= color %>; }'
+               } },
+             { options: {
+                 sql: "select g from x",
+                 cartocss: '#layer[zoom=<%=zoom%>] { }'
+               } }
+          ]
+        }
+    };
+
+    var inst = tmap.instance(tpl1, {});
+
+    var lyr = inst.layers[0].options;
+    assert.equal(lyr.sql, "select 'test' || id, g from t");
+    assert.equal(lyr.cartocss, '#layer { marker-fill:red; marker-width: 23; }');
+
+    lyr = inst.layers[1].options;
+    assert.equal(lyr.sql, "select fun('test') g from x");
+    assert.equal(lyr.cartocss, '#layer { line-color:#a0fF9A; marker-fill:#a0fF9A; }');
+
+    inst = tmap.instance(tpl1, {color:'yellow', name:"it's dangerous"});
+
+    lyr = inst.layers[0].options;
+    assert.equal(lyr.sql, "select 'it''s dangerous' || id, g from t");
+    assert.equal(lyr.cartocss, '#layer { marker-fill:red; marker-width: 23; }');
+
+    lyr = inst.layers[1].options;
+    assert.equal(lyr.sql, "select fun('it''s dangerous') g from x");
+    assert.equal(lyr.cartocss, '#layer { line-color:yellow; marker-fill:yellow; }');
+
+    // Invalid css_color
+    var err = null;
+    try { inst = tmap.instance(tpl1, {color:'##ff00ff'}); }
+    catch (e) { err = e; }
+    assert.ok(err);
+    assert.ok(err.message.match(/invalid css_color/i), err);
+
+    // Invalid css_color 2 (too few digits)
+    var err = null;
+    try { inst = tmap.instance(tpl1, {color:'#ff'}); }
+    catch (e) { err = e; }
+    assert.ok(err);
+    assert.ok(err.message.match(/invalid css_color/i), err);
+
+    // Invalid css_color 3 (too many digits)
+    var err = null;
+    try { inst = tmap.instance(tpl1, {color:'#1234567'}); }
+    catch (e) { err = e; }
+    assert.ok(err);
+    assert.ok(err.message.match(/invalid css_color/i), err);
+
+    // Invalid number
+    var err = null;
+    try { inst = tmap.instance(tpl1, {zoom:'#'}); }
+    catch (e) { err = e; }
+    assert.ok(err);
+    assert.ok(err.message.match(/invalid number/i), err);
+
+    // Invalid number 2
+    var err = null;
+    try { inst = tmap.instance(tpl1, {zoom:'23e'}); }
+    catch (e) { err = e; }
+    assert.ok(err);
+    assert.ok(err.message.match(/invalid number/i), err);
+
+    // Valid number
+    var err = null;
+    try { inst = tmap.instance(tpl1, {zoom:'-.23e10'}); }
+    catch (e) { err = e; }
+    assert.ok(!err);
+  });
+
+  // Can set a limit on the number of user templates
+  test('can limit number of user templates', function(done) {
+    var tmap = new TemplateMaps(redis_pool, signed_maps, {
+      max_user_templates: 2
+    });
+    assert.ok(tmap);
+    var tpl = { version:'0.0.1', auth: {}, layergroup: {} };
+    var expectErr = false;
+    var idMe = [];
+    var idYou = [];
+    Step(
+      function oneForMe() {
+        tpl.name = 'oneForMe';
+        tmap.addTemplate('me', tpl, this);
+      },
+      function twoForMe(err, id) {
+        if ( err ) throw err;
+        assert.ok(id);
+        idMe.push(id);
+        tpl.name = 'twoForMe';
+        tmap.addTemplate('me', tpl, this);
+      },
+      function threeForMe(err, id) {
+        if ( err ) throw err;
+        assert.ok(id);
+        idMe.push(id);
+        tpl.name = 'threeForMe';
+        expectErr = true;
+        tmap.addTemplate('me', tpl, this);
+      },
+      function errForMe(err, id) {
+        if ( err && ! expectErr ) throw err;
+        expectErr = false;
+        assert.ok(err);
+        assert.ok(err.message.match(/limit.*template/), err);
+        return null;
+      },
+      function delOneMe(err) {
+        if ( err ) throw err;
+        tmap.delTemplate('me', idMe.shift(), this);
+      },
+      function threeForMeRetry(err) {
+        if ( err ) throw err;
+        tpl.name = 'threeForMe';
+        tmap.addTemplate('me', tpl, this);
+      },
+      function oneForYou(err, id) {
+        if ( err ) throw err;
+        assert.ok(id);
+        idMe.push(id);
+        tpl.name = 'oneForYou';
+        tmap.addTemplate('you', tpl, this);
+      },
+      function twoForYou(err, id) {
+        if ( err ) throw err;
+        assert.ok(id);
+        idYou.push(id);
+        tpl.name = 'twoForYou';
+        tmap.addTemplate('you', tpl, this);
+      },
+      function threeForYou(err, id) {
+        if ( err ) throw err;
+        assert.ok(id);
+        idYou.push(id);
+        tpl.name = 'threeForYou';
+        expectErr = true;
+        tmap.addTemplate('you', tpl, this);
+      },
+      function errForYou(err, id) {
+        if ( err && ! expectErr ) throw err;
+        expectErr = false;
+        assert.ok(err);
+        assert.ok(err.message.match(/limit.*template/), err);
+        return null;
+      },
+      function finish(err) {
+        // TODO: delete all templates
+        done(err);
+      }
+    );
+  });
+    
+});

tools/convert_database_styles

@@ -0,0 +1,81 @@
+#!/usr/bin/env node
+
+var path = require('path');
+
+var grainstore = require('../node_modules/windshaft/node_modules/grainstore');
+var mapnik = require('mapnik');
+var redis = require('redis');
+
+function usage(me, exitcode) {
+  console.log("Usage: " + me + " <database_name> <table_name> [<target_mapnik_version>]");
+  process.exit(exitcode);
+}
+
+var node_path = process.argv.shift(); 
+var script_path = process.argv.shift(); 
+var me = path.basename(script_path);
+
+var database_name = process.argv.shift()
+var table_name = process.argv.shift()
+var MAPNIK_VERSION = process.argv.shift()
+
+
+if ( ! MAPNIK_VERSION ) {
+  MAPNIK_VERSION = mapnik.versions.mapnik;
+}
+
+if ( ! database_name || ! table_name) {
+  usage(me, 1);
+}
+
+var REDIS_PORT = 6379; // TODO: make a command line parameter
+
+var dbnum = 0;
+
+var mml_store = new grainstore.MMLStore({port:REDIS_PORT}, {mapnik_version:MAPNIK_VERSION});
+
+var failures = [];
+
+var client = redis.createClient(REDIS_PORT, 'localhost');
+client.on('connect', function() {
+  client.select(dbnum);
+  client.keys('map_style|' + database_name + '|' + table_name, function(err, matches) {
+
+    processNext = function() {
+      if ( ! matches.length ) process.exit(failures.length);
+      var k = matches.shift();
+
+      if ( /map_style\|.*\|.*\|/.test(k) ) {
+        //console.warn("Key " + k + " is EXTENDED, skipping");
+        processNext();
+      } 
+
+      var out = 'map_style|' + database_name + '|' + table_name + ': ';
+
+      var mml_builder = mml_store.mml_builder({
+        dbname:database_name, 
+        table:table_name},
+        function(err, payload) {
+          if ( err ) { 
+            console.warn(out + err.message); 
+            failures.push(k); processNext(); 
+          }
+          else {
+            mml_builder.resetStyle(function(err, data) {
+              if ( err ) { 
+                console.warn(out + err.message); 
+                failures.push(k); 
+              }
+              else console.log(out + 'OK');
+              processNext();
+            }, true);
+          }
+      });
+
+    };
+
+    processNext();
+
+  });
+
+});

tools/create_multilayer

@@ -0,0 +1,40 @@
+#!/bin/sh
+
+verbose=no
+tiler_url=http://dev.localhost.lan:8181/tiles/layergroup
+
+# This is for direct windshaft connection
+#tiler_url=http://dev.localhost.lan:8083/database/cartodb_dev_user_1_db/layergroup
+
+while test -n "$1"; do
+  if test "$1" = "-v"; then
+    verbose=yes
+  elif test -z "$cfg"; then
+    cfg="$1"
+  else
+    tiler_url="$1"
+  fi
+  shift
+done
+
+if test -z "$cfg"; then
+  echo "Usage: $0 [-v] <config_file> [<tiler_url>]" >&2
+  echo "Default <tiler_url> is ${tiler_url}" >&2
+  exit 1
+fi
+
+cmd="curl -skH Content-Type:application/json --data-binary @- ${tiler_url}"
+if test x${verbose} = xyes; then
+  cmd="${cmd} -v"
+fi
+res=`cat ${cfg} | tr '\n' ' ' | ${cmd}`
+if test $? -gt 0; then
+  echo "curl command failed: ${cmd}"
+fi
+
+if test x${verbose} = xyes; then
+  echo "${res}"
+fi
+
+tok=`echo "$res" | sed 's/.*"layergroupid":"\([^"]*\)".*/\1/'`
+echo $tok

tools/create_template

@@ -0,0 +1,49 @@
+#!/bin/sh
+
+verbose=no
+tiler_url=http://dev.localhost.lan:8181/tiles/template
+apikey=${CDB_APIKEY}
+
+while test -n "$1"; do
+  if test "$1" = "-v"; then
+    verbose=yes
+  elif test "$1" = "-k"; then
+    shift
+    apikey="$1"
+  elif test "$1" = "-u"; then
+    shift
+    tiler_url="$1"
+  elif test -z "$cfg"; then
+    cfg="$1"
+  else
+    echo "Unused parameter $1" >&2
+  fi
+  shift
+done
+
+if test -z "$cfg"; then
+  echo "Usage: $0 [-v] [-k <api_key>] [-u <tiler_url>] <template_config>" >&2
+  echo "Default <tiler_url> is ${tiler_url}" >&2
+  echo "Default <api_key> is read from CDB_APIKEY env variable" >&2
+  exit 1
+fi
+
+cmd="curl -skH Content-Type:application/json --data-binary @- ${tiler_url}?api_key=${apikey}"
+if test x${verbose} = xyes; then
+  cmd="${cmd} -v"
+fi
+res=`cat ${cfg} | tr '\n' ' ' | ${cmd}`
+if test $? -gt 0; then
+  echo "curl command failed: ${cmd}"
+fi
+
+if test x${verbose} = xyes; then
+  echo "${res}"
+fi
+
+# Successful response contains no space
+echo "$res" | grep " " && { echo $res && exit 1; }
+
+tok=`echo "$res" | sed 's/.*"template_id":"\([^"]*\)".*/\1/'`
+echo $tok
+

tools/delete_template

@@ -0,0 +1,45 @@
+#!/bin/sh
+
+verbose=no
+tiler_url=http://dev.localhost.lan:8181/tiles/template
+apikey=${CDB_APIKEY}
+
+while test -n "$1"; do
+  if test "$1" = "-v"; then
+    verbose=yes
+  elif test "$1" = "-k"; then
+    shift
+    apikey="$1"
+  elif test "$1" = "-u"; then
+    shift
+    tiler_url="$1"
+  elif test -z "$tpl"; then
+    tpl="$1"
+  else
+    echo "Unused parameter $1" >&2
+  fi
+  shift
+done
+
+if test -z "$tpl"; then
+  echo "Usage: $0 [-v] [-k <api_key>] [-u <tiler_url>] <template_id>" >&2
+  echo "Default <tiler_url> is ${tiler_url}" >&2
+  echo "Default <api_key> is read from CDB_APIKEY env variable" >&2
+  exit 1
+fi
+
+cmd="curl -X DELETE -skH Content-Type:application/json ${tiler_url}/${tpl}?api_key=${apikey}"
+if test x${verbose} = xyes; then
+  cmd="${cmd} -v"
+fi
+res=`${cmd}`
+if test $? -gt 0; then
+  echo "curl command failed: ${cmd}"
+fi
+
+if test x${verbose} = xyes; then
+  echo "${res}"
+fi
+
+tok=`echo "$res" | sed 's/.*"template_id":"\([^"]*\)".*/\1/'`
+echo $tok

tools/examples/mapconfig_simple.js

@@ -0,0 +1,11 @@
+{"version":"1.0.1",
+	"layers":[{
+    "type":"cartodb",
+	  "options":{
+      "sql":"select 1 as id, ST_SetSRID(ST_MakePoint(0,0),3857) as the_geom_webmercator",
+	    "cartocss":"#style{ marker-width: 12;}",
+	    "cartocss_version":"2.1.1",
+      "Interactivity":"id"
+    }
+  }]
+}

tools/examples/template_simple.js

@@ -0,0 +1,17 @@
+{
+ "version":"0.0.1",
+ "name":"simple",
+ "placeholders":{},
+ "auth":{ "method":"open" },
+ "layergroup":{
+   "version":"1.0.1",
+    "layers":[{
+      "type":"cartodb",
+      "options":{
+        "sql":"select ST_SetSRID(ST_MakePoint(0,0),3857) as the_geom_webmercator",
+        "cartocss":"#s{ marker-width: 12;}",
+        "cartocss_version":"2.1.1"
+      }
+    }]
+  }
+}

tools/flush_cache

@@ -0,0 +1,59 @@
+#!/usr/bin/env node
+
+var path = require('path');
+var request    = require('request');
+
+function usage(me, exitcode) {
+  console.log("Usage: " + me + " [--env <environment>] <username> <tablename>");
+  process.exit(exitcode);
+}
+
+var node_path = process.argv.shift(); 
+var script_path = process.argv.shift(); 
+var basedir = path.dirname(script_path);
+var me = path.basename(script_path);
+
+var ENV = 'development.js';
+var username, table;
+var arg;
+while ( arg = process.argv.shift() ) {
+  if ( arg == '--env' ) {
+    ENV = process.argv.shift();
+  }
+  else if ( ! username ) {
+    username = arg;
+  }
+  else if ( ! table ) {
+    table = arg;
+  }
+  else {
+    console.warn("Unused parameter " + arg);
+  }
+}
+
+if ( ! table ) {
+  usage(me, 1);
+}
+
+global.environment = require('../config/environments/' + ENV);
+
+// _after_ setting global.environment
+var serverOptions = require('../lib/cartodb/server_options');
+
+var host = global.environment.host;
+var port = global.environment.port;
+var re = ''+serverOptions.re_userFromHost;
+var hostname = re.replace(/^\/\^/, '')
+                 .replace(/\/$$/, '')
+                 .replace(/\\/g,'')
+                 .replace(/\([^)]*\)/,username)
+              ;
+//console.log("re: " + re);
+//console.log("hostname: " + hostname);
+
+var url = 'http://' + host + ':' + port + '/tiles/' + table + '/flush_cache';
+request.del({ url: url, headers: { host: hostname } },
+function(err, res, body) {
+  if ( err ) throw err;
+  console.log(res.body);
+});

tools/instanciate_template

@@ -0,0 +1,53 @@
+#!/bin/sh
+
+verbose=no
+tiler_url=http://dev.localhost.lan:8181/tiles/template
+apikey=${CDB_APIKEY}
+
+while test -n "$1"; do
+  if test "$1" = "-v"; then
+    verbose=yes
+  elif test "$1" = "-k"; then
+    shift
+    apikey="$1"
+  elif test "$1" = "-u"; then
+    shift
+    tiler_url="$1"
+  elif test -z "$tpl"; then
+    tpl="$1"
+  elif test -z "$cfg"; then
+    cfg="$1"
+  else
+    echo "Unused parameter $1" >&2
+  fi
+  shift
+done
+
+if test -z "$tpl"; then
+  echo "Usage: $0 [-v] [-k <api_key>] [-u <tiler_url>] <template_id> [<template_params>]" >&2
+  echo "Default <tiler_url> is ${tiler_url}" >&2
+  echo "Default <api_key> is read from CDB_APIKEY env variable" >&2
+  exit 1
+fi
+
+if test -z "$cfg"; then
+  cfg="/dev/null"
+fi
+
+tiler_url="${tiler_url}/${tpl}"
+
+cmd="curl -X POST -skH Content-Type:application/json --data-binary @- ${tiler_url}?api_key=${apikey}"
+if test x${verbose} = xyes; then
+  cmd="${cmd} -v"
+fi
+res=`cat ${cfg} | tr '\n' ' ' | ${cmd}`
+if test $? -gt 0; then
+  echo "curl command failed: ${cmd}"
+fi
+
+if test x${verbose} = xyes; then
+  echo "${res}"
+fi
+
+tok=`echo "$res" | sed 's/.*"layergroupid":"\([^"]*\)".*/\1/'`
+echo $tok

tools/list_templates

@@ -0,0 +1,45 @@
+#!/bin/sh
+
+verbose=no
+tiler_url=http://dev.localhost.lan:8181/tiles/template
+apikey=${CDB_APIKEY}
+
+while test -n "$1"; do
+  if test "$1" = "-v"; then
+    verbose=yes
+  elif test "$1" = "-k"; then
+    shift
+    apikey="$1"
+  elif test "$1" = "-u"; then
+    shift
+    tiler_url="$1"
+  elif test "$1" = "-h" -o "$1" = "-?"; then
+    echo "Usage: $0 [-v] [-k <api_key>] [-u <tiler_url>]" >&2
+    echo "Default <tiler_url> is ${tiler_url}" >&2
+    echo "Default <api_key> is read from CDB_APIKEY env variable" >&2
+    exit 0
+  else
+    echo "Unused parameter $1" >&2
+  fi
+  shift
+done
+
+
+cmd="curl -X GET -sk ${tiler_url}?api_key=${apikey}"
+if test x${verbose} = xyes; then
+  cmd="${cmd} -v"
+fi
+res=`${cmd}`
+if test $? -gt 0; then
+  echo "curl command failed: ${cmd}"
+fi
+
+if test x${verbose} = xyes; then
+  echo "${res}"
+fi
+
+node <<EOF
+var parsed = JSON.parse('$res');
+console.dir(parsed);
+EOF
+

tools/munin/Makefile

@@ -0,0 +1,19 @@
+MUNIN_PLUGINS_DIR=/etc/munin/plugins
+MUNIN_PLUGINS_CONFIG_DIR=/etc/munin/plugin-conf.d
+PWD=$(shell pwd)
+
+all: windshaft.conf
+
+windshaft.conf: windshaft.conf.in
+	sed 's#@PWD@#$(PWD)#' < $< > $@
+
+install-munin-plugin-conf: windshaft.conf 
+	install -m 644 $< $(MUNIN_PLUGINS_CONFIG_DIR)/windshaft.conf
+
+install-munin-plugin: windshaft
+	install -m 755 $< $(MUNIN_PLUGINS_DIR)/windshaft
+
+install: install-munin-plugin install-munin-plugin-conf
+
+clean:
+	rm -f windshaft.conf

tools/munin/windshaft

@@ -0,0 +1,87 @@
+#!/bin/sh
+
+envnik=$(basename "${TILER_ENVIRONMENT}" .js)
+
+if test "$1" = "config"; then
+echo "graph_title fd usage (${envnik})"
+cat <<"EOM"
+graph_vlabel number of file descriptors
+graph_category windshaft
+graph_scale no
+procs.label Number of tiler processes
+pgsql.label PostgreSQL connections (max)
+redis.label Redis connections (max)
+http.label Incoming http requests (max)
+caches.label Number of renderer caches (max)
+nfd.label Total file descriptors (max)
+EOM
+exit 0
+fi
+
+if test x"$1" != x; then
+  TILER_ENVIRONMENT=$(cd $(dirname $0); pwd)/../../config/environments/${1}.js
+fi
+
+if test -z "$TILER_ENVIRONMENT"; then
+  echo "Usage: $0 [<environment>]" >&2
+  echo "   or: [TILER_ENVIRONMENT=<environment>] $0" >&2
+  exit 1
+fi
+
+http_port=$(echo "console.log(require('${TILER_ENVIRONMENT}').port)" | node) || exit 1
+pgsql_port=$(echo "console.log(require('${TILER_ENVIRONMENT}').postgres.port)" | node) || exit 1
+redis_port=$(echo "console.log(require('${TILER_ENVIRONMENT}').redis.port)" | node) || exit 1
+
+pids=$(lsof -i :${http_port} | grep LISTEN | awk '{print $2}')
+nworkers=$(echo "${pids}" | wc -l)
+pids=$(echo "${pids}" | paste -sd ' ')
+
+if test -z "${pids}"; then
+  echo "No processes found listening on tcp port '${http_port}'" >&2
+  exit 1
+fi
+
+tmpreport="/tmp/checkfd.$$.txt"
+
+lsof -Pp $(echo "${pids}" | tr ' ' ',') > "${tmpreport}"
+
+maxdb=0
+maxredis=0
+maxhttp=0
+maxtot=0
+maxcache=0
+
+for pid in ${pids}; do
+
+  cnt=$(grep "${pid}" "${tmpreport}" | grep ":${pgsql_port} " | wc -l);
+  if test $cnt -gt $maxdb; then maxdb=$cnt; fi
+  
+  cnt=$(grep "${pid}" "${tmpreport}" | grep ":${redis_port} " | wc -l);
+  if test $cnt -gt $maxredis; then maxredis=$cnt; fi
+
+  cnt=$(grep "${pid}" "${tmpreport}" | grep ":${http_port}-" | grep -v "LISTEN" | wc -l);
+  if test $cnt -gt $maxhttp; then maxhttp=$cnt; fi
+
+  cnt=$(grep "${pid}" "${tmpreport}" | wc -l);
+  if test $cnt -gt $maxtot; then maxtot=$cnt; fi
+
+  log=$(grep "${pid}" "${tmpreport}" | grep -w 1w | awk '{print $9}')
+  if test -e "${log}"; then
+    kill -USR2 "${pid}"
+    cnt=$(tac ${log} | sed -n -e '/ItemKey/p;/^RenderCache/q' | wc -l)
+    if test $cnt -gt $maxcache; then maxcache=$cnt; fi
+  else
+    # report the error...
+    maxcache=-1
+  fi
+
+done
+
+echo "procs.value ${nworkers}"
+echo "pgsql.value ${maxdb}"
+echo "redis.value ${maxredis}"
+echo "http.value ${maxhttp}"
+echo "caches.value ${maxcache}"
+echo "nfd.value ${maxtot}"
+
+rm -f "${tmpreport}"

tools/munin/windshaft.conf.in

@@ -0,0 +1,5 @@
+# Configuration file for munin plugin
+
+[windshaft]
+user root
+env.TILER_ENVIRONMENT @PWD@/../../config/environments/production.js

tools/performance/stresstest_templates.sh

@@ -0,0 +1,54 @@
+#!/bin/sh
+
+verbose=no
+tiler_url=http://dev.localhost.lan:8181/tiles/template
+apikey=${CDB_APIKEY}
+
+while test -n "$1"; do
+  if test "$1" = "-k"; then
+    shift
+    apikey="$1"
+  elif test "$1" = "-u"; then
+    shift
+    tiler_url="$1"
+  elif test -z "$tpl"; then
+    tpl="$1"
+  else
+    echo "Unused parameter $1" >&2
+  fi
+  shift
+done
+
+if test -z "$tpl"; then
+  echo "Usage: $0 [-v] [-k <api_key>] [-u <tiler_url>] <template_config>" >&2
+  echo "Default <tiler_url> is ${tiler_url}" >&2
+  echo "Default <api_key> is read from CDB_APIKEY env variable" >&2
+  exit 1
+fi
+
+basedir=$(cd $(dirname $0); cd ..; pwd)
+export CDB_APIKEY=${apikey}
+max=3000000
+i=0
+while test "$i" -le "$max"; do
+  tpln=`cat ${tpl} | sed "s/\"name\":\"\(.*\)\"/\"name\":\"\1${i}\"/"`
+  tpl_id=`echo ${tpln} | ${basedir}/create_template -u ${tiler_url} /dev/stdin`
+  if test $? -ne 0; then
+    echo $tpl_id >&2
+    break
+  fi
+  tpl_id=`echo ${tpln} | ${basedir}/update_template -u ${tiler_url} ${tpl_id} /dev/stdin`
+  if test $? -ne 0; then
+    echo $tpl_id >&2
+    break
+  fi
+  out=`${basedir}/delete_template -u ${tiler_url} ${tpl_id}`
+  if test $? -ne 0; then
+    echo $out >&2
+    break
+  fi
+  i=$((i+1))
+  if test `expr $i % 100` -eq 0; then
+    echo -n "."
+  fi
+done

tools/reset_styles

@@ -1,35 +1,105 @@
 #!/usr/bin/env node
 
-// Reset redis-stored XML styles so that they are regenerated
-// from CartoCSS on next tile request
+/*
 
-var redis = require('redis')
+This scripts drops all extended map_style keys in redis and regenerates
+the XML caches in all the base ones to target the configured mapnik_version.
 
-var REDIS_PORT = 6379; // TODO: make a parameter
+Optionally (with --convert) it also re-writes the CartoCSS if needed
+to target the configured mapnik_version.
+
+It is recommended to make a backup of the redis database before using
+this script.
+
+*/
+
+var path = require('path');
+
+// Reset all styles in the store
+var grainstore = require('../node_modules/windshaft/node_modules/grainstore/lib/grainstore');
+var mapnik = require('mapnik');
+var redis = require('redis');
+
+function usage(me, exitcode) {
+  console.log("Usage: " + me + " [--convert] <environment>");
+  process.exit(exitcode);
+}
+
+var doConvert = false;
+
+var node_path = process.argv.shift(); 
+var script_path = process.argv.shift(); 
+var me = path.basename(script_path);
+var ENV;
+var arg;
+while ( arg = process.argv.shift() ) {
+  if ( arg == '--convert' ) {
+    doConvert = true;
+  }
+  else if ( ! ENV ) {
+    ENV = arg;
+  }
+  else {
+    usage(me, 1);
+  }
+}
+
+if ( ! ENV ) usage(me, 1);
+
+global.environment = require('../config/environments/' + ENV);
+var serverOptions = require('../lib/cartodb/server_options'); // _after_ setting global.environment
+
+var MAPNIK_VERSION = global.environment.mapnik_version || mapnik.versions.mapnik;
+
+console.log( (doConvert ? "Converting" : "Resetting" ) + ' all styles to target ' + MAPNIK_VERSION);
 
 var dbnum = 0;
 
-var client = redis.createClient(REDIS_PORT, 'localhost');
+var mml_store = new grainstore.MMLStore(serverOptions.redis, serverOptions.grainstore);
+
+var failures = [];
+
+var client = redis.createClient(serverOptions.redis.port, serverOptions.redis.host);
 client.on('connect', function() {
   client.select(dbnum);
   client.keys('map_style|*', function(err, matches) {
 
     processNext = function() {
-      if ( ! matches.length ) process.exit(0);
+      if ( ! matches.length ) process.exit(failures.length);
       var k = matches.shift();
-      console.log("Resetting XML in key: " + k);
-      client.get(k, function(err, val) {
-        if ( err ) throw err;
-        val = JSON.parse(val);
-        delete val.xml;
-        client.set(k, JSON.stringify(val), function() {
-          console.log("done with style " + k);
+
+      if ( /map_style\|.*\|.*\|/.test(k) ) {
+        // See https://github.com/Vizzuality/Windshaft-cartodb/issues/58
+        //console.warn("Key " + k + " is EXTENDED, dropping");
+        client.del(k, function(err) {
+          if ( err ) console.warn("Error dropping key " + k);
           processNext();
         });
+        return;
+      } 
+
+      var params = RegExp(/map_style\|(.*)\|(.*)/).exec(k);
+      var db = params[1];
+      var tab = params[2];
+      var out = 'map_style|' + db + '|' + tab + ': ';
+
+      var mml_builder = mml_store.mml_builder({dbname:db, table:tab},
+                                              function(err, payload) {
+
+          if ( err ) { console.warn(out + err.message); failures.push(k); processNext(); }
+          else {
+            mml_builder.resetStyle(function(err, data) {
+              if ( err ) { console.warn(out + err.message); failures.push(k); }
+              else console.log(out + 'OK' + ( doConvert ? ' (converted)' : '' ));
+              processNext();
+            }, doConvert);
+          }
       });
-    }
+
+    };
 
     processNext();
 
   });
+
 });

tools/show_style

@@ -0,0 +1,147 @@
+#!/usr/bin/env node
+
+var path = require('path');
+var redis = require('redis');
+var Step = require('step');
+
+
+function usage(me, exitcode) {
+  console.log("Usage: " + me + " [--env <environment>] <username> [<tablename>|~<token>]");
+  process.exit(exitcode);
+}
+
+var node_path = process.argv.shift(); 
+var script_path = process.argv.shift(); 
+var basedir = path.dirname(script_path);
+var me = path.basename(script_path);
+
+var ENV = 'development.js';
+var username, token;
+var arg;
+while ( arg = process.argv.shift() ) {
+  if ( arg == '--env' ) {
+    ENV = process.argv.shift();
+  }
+  else if ( ! username ) {
+    username = arg;
+  }
+  else if ( ! token ) {
+    token = arg;
+  }
+  else {
+    console.warn("Unused parameter " + arg);
+  }
+}
+
+if ( ! username ) usage(me, 1);
+
+console.log("Using environment " + ENV);
+
+global.environment = require('../config/environments/' + ENV);
+var serverOptions = require('../lib/cartodb/server_options'); // _after_ setting global.environment
+
+var client;
+var dbname;
+Step(
+  function getClient() {
+    client = redis.createClient(serverOptions.redis.port, serverOptions.redis.host);
+    client.on('connect', this);
+  },
+  function getUserMeta(err) {
+    if ( err ) throw err;
+    client.select(5);
+    client.hgetall('rails:users:' + username, this);
+  },
+  function readDB(err, data) {
+    if ( err ) throw err;
+    if ( ! data )
+      throw new Error('Username ' + username + ' unknown by redis on port '
+        + serverOptions.redis.port + ' (try CARTODB/script/restore_redis?)');
+    //console.log("Data:"); console.dir(data);
+    dbname = data['database_name'];
+    console.log("Database name for user " + username + ": " + dbname);
+    client.select(0);
+    return null;
+  },
+  function showTokens(err) {
+    if ( err ) throw err;
+    if ( token ) return null;
+    var next = this;
+    Step(
+      function getTokens() {
+        client.keys('map_style|' + dbname + '|*', this);
+      },
+      function showTokens(err, data) {
+        if (err) throw err;
+        if ( data ) console.log(data.join('\n'));
+        return null;
+      },
+      function showTokensFinish(err) {
+        next(err);
+      }
+    );
+  },
+  function showStyle(err) {
+    if ( err ) throw err;
+    if ( ! token ) return null;
+    var next = this;
+    Step(
+      function getStyle() {
+        client.get('map_style|' + dbname + '|' + token, this);
+      },
+      function showStyle(err, data) {
+        if ( err ) throw err;
+        if ( ! data ) {
+          throw new Error(token + ': no such map style known by redis on port '
+            + serverOptions.redis.port);
+        }
+//console.log("data: " + data);
+        var x=JSON.parse(data);
+        printMapnikStyle(x, this);
+      },
+      function showStyleFinish(err) {
+        next(err);
+      }
+    );
+  },
+  function finish(err) {
+    if ( err ) {
+      console.error(err.message)
+      process.exit(1);
+    }
+    process.exit(0);
+  }
+);
+
+function printMapnikStyle(x, callback) {
+  console.log('style: ' + x.style);
+  console.log('version: ' + x.version);
+  var grainstore = require(basedir + '/../node_modules/windshaft/node_modules/grainstore/lib/grainstore');
+  var mml_store = new grainstore.MMLStore(serverOptions.redis, serverOptions.grainstore);
+  var builderconfig = {dbname:dbname};
+  if ( token.match(/^~/) ) {
+    builderconfig.token = token.substring(1);
+  } else {
+    builderconfig.table = token;
+  }
+  var mml_builder;
+  Step(
+    function getBuilder() {
+      mml_builder = mml_store.mml_builder(builderconfig, this);
+    },
+    function getXML(err, builder) {
+      if ( err ) throw err;
+      mml_builder.toXML(this);
+    },
+    function showXML(err, xml) {
+      if ( err ) throw err;
+      console.log('- XML - ');
+      console.log(xml);
+      return null;
+    },
+    function finish(err) {
+      callback(err);
+    }
+  );
+}
+

tools/update_template

@@ -0,0 +1,47 @@
+#!/bin/sh
+
+verbose=no
+tiler_url=http://dev.localhost.lan:8181/tiles/template
+apikey=${CDB_APIKEY}
+
+while test -n "$1"; do
+  if test "$1" = "-v"; then
+    verbose=yes
+  elif test "$1" = "-k"; then
+    shift
+    apikey="$1"
+  elif test "$1" = "-u"; then
+    shift
+    tiler_url="$1"
+  elif test -z "$tpl"; then
+    tpl="$1"
+  elif test -z "$cfg"; then
+    cfg="$1"
+  else
+    echo "Unused parameter $1" >&2
+  fi
+  shift
+done
+
+if test -z "$cfg"; then
+  echo "Usage: $0 [-v] [-k <api_key>] [-u <tiler_url>] <template_id> <template_config>" >&2
+  echo "Default <tiler_url> is ${tiler_url}" >&2
+  echo "Default <api_key> is read from CDB_APIKEY env variable" >&2
+  exit 1
+fi
+
+cmd="curl -X PUT -skH Content-Type:application/json --data-binary @- ${tiler_url}/${tpl}?api_key=${apikey}"
+if test x${verbose} = xyes; then
+  cmd="${cmd} -v"
+fi
+res=`cat ${cfg} | tr '\n' ' ' | ${cmd}`
+if test $? -gt 0; then
+  echo "curl command failed: ${cmd}"
+fi
+
+if test x${verbose} = xyes; then
+  echo "${res}"
+fi
+
+tok=`echo "$res" | sed 's/.*"template_id":"\([^"]*\)".*/\1/'`
+echo $tok