Release 1.14.1

Closes #87.
Includes testcase

.gitignore

@@ -1,2 +1,6 @@
-node_modules
-.idea
+node_modules*
+config/environments/*.js
+.idea
+tools/munin/windshaft.conf
+logs/
+pids/

.travis.yml

@@ -0,0 +1,14 @@
+before_install:
+  - sudo apt-add-repository --yes ppa:mapnik/v2.1.0
+  - sudo apt-add-repository --yes ppa:ubuntugis/ppa
+  - sudo apt-get update -q
+  - sudo apt-get install -q libmapnik libmapnik-dev postgresql-9.1-postgis libsigc++-dev
+  - createdb template_postgis
+  - psql -c "CREATE EXTENSION postgis" template_postgis
+
+env:
+  - NPROCS=1 JOBS=1
+
+language: node_js
+node_js:
+  - "0.8"

HOWTO_RELEASE

@@ -0,0 +1,11 @@
+1. Ensure proper version in package.json 
+2. Ensure NEWS section exists for the new version, review it, add release date
+3. Drop npm-shrinkwrap.json
+4. Run npm install
+5. Test (make check or npm test), fix if broken before proceeding
+6. Run npm shrinkwrap
+7. Commit package.json, npm-shrinwrap.json, NEWS
+8. Tag Major.Minor.Patch
+9. Announce
+10. Stub NEWS/package for next version
+

Makefile

@@ -1,8 +1,32 @@
+srcdir=$(shell pwd)
+
 all:
 	npm install
 
 clean:
 	rm -rf node_modules/*
 
-check:
-	./run_tests.sh
+config/environments/test.js: config/environments/test.js.example Makefile
+	./configure --environment=test
+
+check-local: config/environments/test.js
+	./run_tests.sh ${RUNTESTFLAGS} \
+    test/unit/cartodb/redis_pool.test.js \
+    test/unit/cartodb/req2params.test.js \
+    test/acceptance/cache_validator.js \
+    test/acceptance/server.js \
+    test/acceptance/multilayer.js
+
+check-submodules:
+	PATH="$$PATH:$(srcdir)/node_modules/.bin/"; \
+	for sub in windshaft grainstore node-varnish mapnik; do \
+	  if test -e node_modules/$${sub}; then \
+	      echo "Testing submodule $${sub}"; \
+	      make -C node_modules/$${sub} check || exit 1; \
+	  fi; \
+	done
+
+check-full: check-local check-submodules
+
+check: check-local
+

NEWS.md

@@ -0,0 +1,177 @@
+1.4.1 -- 2013-11-08
+-------------------
+
+* Fix support for exponential notation in CartoCSS filter values (#87)
+
+1.4.0 -- 2013-10-31
+-------------------
+
+* Add Support for Mapnik-2.2.0 (#78)
+
+1.3.6 -- 2013-10-11
+-------------------
+
+* Restore support for node-0.8.9 accidentally dropped by 1.3.5
+  NOTE: needs removing node_modules/windshaft and re-running npm install
+
+1.3.5 -- 2013-10-03
+-------------------
+
+* Fixing apostrophes in CartoCSS
+* Fix "sql/table must contain zoom variable" error when using
+  "[ zoom > 3]" CartoCSS snippets (note the space)
+* Fix backward compatibility handling of sqlapi.host configuration (#82)
+* Fix error for invalid text-name in CartoCSS (#81)
+* Do not let anonymous requests use authorized renderer caches 
+
+1.3.4
+------
+
+NOTE: configuration sqlapi.host renamed to sqlapi.domain
+      (support for "sqlapi.host" is retained for backward compatibility)
+
+* Improve empty CartoCSS error message
+* Improve invalid mapnik-geometry-type CSS error message
+* Fix race condition in localization of network resources
+
+1.3.3
+------
+* Set Last-Modified header to allow for 304 responses
+* Add profiling support (needs useProfiler in env config file)
+* Fix double-checking for layergroups with no interactivity
+* Log full layergroup config at creation time (#76)
+
+1.3.2
+------
+* Set default layergroup TTL to 2 hours
+* Serve multilayer tiles and grid with persistent cache control
+
+1.3.1
+------
+* Fix deadlock on new style creation
+* Fix database authentication with multi-table layergroups
+* Add tile and grid fetching checks at layergroup creation time
+* Fix SQL error reporting to NOT split on newline
+* Fix support for CartoCSS attachments
+
+1.3.0
+------
+* Change stats format for multilayer map token request, see
+  http://github.com/Vizzuality/Windshaft-cartodb/wiki/Redis-stats-format
+
+1.2.1
+------
+* Fix multilayer post from firefox
+* Fix multilayer cartocss layer name handling
+
+1.2.0
+------
+* Multilayer API changes
+  * Layers passed by index in grid fetching url
+  * Interactivity only specified in layergroup config
+  * Embed cache_buster within token 
+  * Use ISO format for last_modified timestamp
+* Expected LZMA encoding changed to base64
+
+1.1.10
+------
+* Fix regression with default interactivity parameter (#74)
+* More verbose logging for SQL api connection errors
+* Write stats for multilayer map token request
+
+1.1.9
+-----
+* Handle SQL API errors by requesting no Varnish cache
+* Fix X-Cache-Channel for multilayer (by token) responses
+* Add last_modified field to layergroup creation response (#72)
+* Deprecate signal handler for USR1, add handler for USR2 (#71)
+* Fix support for ampersend characters in CartoCSS
+* Add support for LZMA compressed GET parameters
+* Add support for creating layergroups via GET
+
+1.1.8
+-----
+* Require Windshaft-0.9.1, to reduce harmfulness of cache_buster param
+
+1.1.7 (DD//MM//YY)
+-----
+* Do not let /etc/services confuse FD checker (munin plugin)
+* Multilayer support (#72)
+* Expose renderer settings in the environment config files
+
+1.1.6 (19//02//13)
+-----
+* Require windshaft 0.8.5, fixing some stability issues
+  and providing cache info on request
+* Require grainstore 0.10.9, fixing an issue with multi-geom markers
+* Enhance run_tests.sh to allow running single tests and skipping preparation
+* Fix async throws in getGeometryType, getInfoWindow and getMapMetadata
+* Survive connection refusals from redis
+* Add maxConnection environment configuration, default to 128
+
+1.1.5 (DD//MM//YY)
+-----
+* Fix bogus cached return of utf grid for fully contained tiles (#67)
+
+1.1.4 (DD//MM//YY)
+-----
+* Reduce default extent to allow for consistent proj4 round-tripping 
+* Enhance reset_styles script to use full configuration (#62)
+* Have reset_styles script also drop extended keys (#58)
+* Fix example postgis parameter for simplifying input geoms (#63)
+* Add row_limit to example config (#64)
+
+1.1.3 (30//11//12)
+-----
+* Fix reset_styles script to really skip extended keys
+* CartoCSS versioning
+ * Mapnik-version dependent default styles
+ * Enhance 2.0 -> 2.1 transforms:
+  * styles with conditional markers
+  * scale arrow markers by 50%
+
+1.1.2 (DD//MM//YY)
+-----
+* CartoCSS versioning
+ * Fix use of "style_version" with GET (inline styles)
+ * Enhance 2.0 -> 2.1 transforms:
+  * styles with no semicolon
+  * markers shift due to geometry clipping
+
+1.1.1 (DD//MM//YY)
+-----
+* Add support for persistent client cache headers
+* Fix crash on unknown user (#55)
+* Add /version entry point
+* CartoCSS versioning
+ * Include style_version in GET /style response
+ * Support style_version and style_convert parameters in POST /style request
+ * Support style_version in GET /:z/:x/:y request
+
+1.1.0 (30/10/12)
+=======
+* Add /version entry point
+* CartoCSS versioning
+ * Include version in GET /style response
+ * Support version and convert parameters in POST /style request
+ * Autodetect target mapnik version and let config override it
+ * Add tools/reset_styles script to batch-reset (and optionally convert) styles
+* Configurable logging format (#4)
+* Detailed error on missing user metadata 
+* Properly handle unauthenticated requests for metadata
+* Accept "api_key" in addition to "map_key",
+  both in query_string and POST body (#38)
+* Add ./configure script
+* Allow listening on host IP
+* Replaced environment configs by .example ones
+* Fixed some issues with cluster2
+
+1.0.0 (03/10/12)
+-----
+* Migrated to node 0.8.x.
+
+0.9.0 (25/09/12)
+-----
+* External resources in CartoCSS
+* Added X-Cache-Channel header in all the tiler GET requests
+* Small fixes

README.md

@@ -1,21 +1,47 @@
 Windshaft-CartoDB
 ==================
 
-NOTE: requires node-0.4.x
+[![Build Status](https://travis-ci.org/CartoDB/Windshaft-cartodb.png)](http://travis-ci.org/CartoDB/Windshaft-cartodb)
 
 This is the CartoDB map tiler. It extends Windshaft with some extra
 functionality and custom filters for authentication
 
 * reads dbname from subdomain and cartodb redis for pretty tile urls
-* configures windshaft to publish cartodb_id as the interactivity layer
+* configures windshaft to publish ``cartodb_id`` as the interactivity layer
 * gets the default geometry type from the cartodb redis store
 * allows tiles to be styled individually
 * provides a link to varnish high speed cache
 * provides a infowindow endpoint for windshaft
-* provides a map_metadata endpoint for windshaft
+* provides a ``map_metadata`` endpoint for windshaft
 
-Install
--------
+Requirements
+------------
+
+ [core]
+ - node-0.8.x+
+ - PostgreSQL-8.3+
+ - PostGIS-1.5.0+
+ - Redis 2.2.0+ (http://www.redis.io)
+ - Mapnik 2.0 or 2.1 
+
+ [for cache control]
+ - CartoDB-SQL-API 1.0.0+
+ - CartoDB 0.9.5+ (for ``CDB_QueryTables``)
+ - Varnish (https://www.varnish-cache.org)
+
+Configure
+---------
+
+Create the config/environments/<env>.js files (there are .example files
+to start from). You can optionally use the ./configure script for this,
+see ```./configure --help``` to see available options.
+
+Look at lib/cartodb/server_options.js for more on config
+
+Build/install
+-------------
+
+To fetch and build all node-based dependencies, run:
 
 ```
 git clone
@@ -28,22 +54,18 @@ happen to have startup errors you may need to force rebuilding those
 modules. At any time just wipe out the node_modules/ directory and run
 ```npm install``` again.
 
-Configure
----------
-
-Edit config/environments/<env>.js files
-
-Look at lib/cartodb/server_options for more on config
 
 Run
 ---
 
 ```
-node app.js [development | staging | production]
+node app.js <env> 
 ```
 
-Note that caches are kept in redis. If you're not seeing what
-you expect there may be out-of-sync records in there.
+Where <env> is the name of a configuration file under config/environments/.
+
+Note that caches are kept in redis. If you're not seeing what you expect
+there may be out-of-sync records in there.
 Take a look: http://redis.io/commands
 
 
@@ -58,8 +80,16 @@ Args:
 
 * sql - plain SQL arguments
 * interactivity - specify the column to use in UTFGrid
-* cache_buster - if needed you can add a cachebuster to make sure you're
-  rendering new
+* cache_buster - Specify an identifier for the internal tile cache.
+                 Requesting tiles with the same cache_buster value may
+                 result in being served a cached version of the tile
+                 (even when requesting a tile for the first time, as tiles
+                 can be prepared in advance)
+* cache_policy - Set to "persist" to have the server send an Cache-Control
+                 header requesting caching devices to keep the response
+                 cached as much as possible. This is best used with a
+                 timestamp value in cache_buster for manual control of
+                 updates.
 * geom_type - override the cartodb default
 * style - override the default map style with Carto
 
@@ -71,6 +101,8 @@ Args:
 Args:
 
 * style - the style in CartoCSS you want to set
+* style_version - the version of the style for POST
+* style_convert - request conversion to target version (both POST and GET)
 
 
 **INFOWINDOW**

app.js

@@ -10,9 +10,9 @@
 
 // sanity check
 var ENV = process.argv[2]
-if (ENV != 'development' && ENV != 'production'){
+if (ENV != 'development' && ENV != 'production' && ENV != 'staging' ){
     console.error("\nnode app.js [environment]");
-    console.error("environments: [development, production]\n");
+    console.error("environments: [development, production, staging]\n");
     process.exit(1);
 }
 
@@ -33,5 +33,25 @@ var Windshaft = require('windshaft');
 var serverOptions = require('./lib/cartodb/server_options');
 
 ws = CartodbWindshaft(serverOptions);
-ws.listen(global.environment.port);
-console.log("Windshaft tileserver started on port " + global.environment.port);
+
+// Maximum number of connections for one process
+// 128 is a good number if you have up to 1024 filedescriptors
+// 4 is good if you have max 32 filedescriptors
+// 1 is good if you have max 16 filedescriptors
+ws.maxConnections = global.environment.maxConnections || 128;
+
+ws.listen(global.environment.port, global.environment.host);
+
+ws.on('listening', function() {
+  console.log("Windshaft tileserver started on " + global.environment.host + ':' + global.environment.port);
+});
+
+// DEPRECATED, use SIGUSR2
+process.on('SIGUSR1', function() {
+  console.log('WARNING: handling of SIGUSR1 by Windshaft-CartoDB is deprecated, please send SIGUSR2 instead');
+  ws.dumpCacheStats();
+});
+
+process.on('SIGUSR2', function() {
+  ws.dumpCacheStats();
+});

cluster.js

@@ -1,45 +0,0 @@
-/*
- * Windshaft-CartoDB
- * ===============
- *
- * ./app.js [environment]
- *
- * environments: [development, production]
- */
-
-var cluster = require('cluster');
-
-// sanity check
-var ENV = process.argv[2]
-if (ENV != 'development' && ENV != 'production' && ENV != 'staging'){
-    console.error("\nnode app.js [environment]");
-    console.error("environments: [development, production, staging]\n");
-    process.exit(1);
-}
-
-var _ = require('underscore')
-    , Step       = require('step')
-	, CartodbWindshaft = require('./lib/cartodb/cartodb_windshaft');
-    
-
-// set environment specific variables
-global.settings     = require(__dirname + '/config/settings');
-global.environment  = require(__dirname + '/config/environments/' + ENV);
-_.extend(global.settings, global.environment);
-
-// Include cart_data.js only _after_ the "global" variable is set
-// See https://github.com/Vizzuality/Windshaft-cartodb/issues/28
-var cartoData = require('./lib/cartodb/carto_data');
-
-var Windshaft = require('windshaft');
-var serverOptions = require('./lib/cartodb/server_options');
-
-ws = CartodbWindshaft(serverOptions);
-cluster(ws)
-  .use(cluster.logger('logs'))
-  .use(cluster.stats())
-  .use(cluster.pidfiles('pids'))
-  .set('workers', 1)
-  .listen(global.environment.port, global.environment.host);
-
-console.log("Windshaft tileserver started on port " + global.environment.port);

config/environments/development.js

@@ -1,43 +0,0 @@
-var config = {
-     environment: 'development'
-    ,port: 8181
-    ,host: '127.0.0.1'
-    ,enable_cors: true
-    ,cache_enabled: false
-    ,postgres_auth_user: 'development_cartodb_user_<%= user_id %>'
-    ,postgres: {
-        type: "postgis",
-        user: "publicuser",
-        host: '127.0.0.1',
-        port: 5432,
-        extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188",
-        /* experimental
-        geometry_field: "the_geom",
-        extent: "-180,-90,180,90",
-        srid: 4326,
-        */
-        simplify: true
-    }
-    ,millstone: {
-        cache_basedir: '/tmp/cdb-tiler-dev/millstone-dev'
-    }
-    ,redis: {
-        host: '127.0.0.1',
-        port: 6379,
-        idleTimeoutMillis: 1,
-        reapIntervalMillis: 1
-    }
-    ,sqlapi: {
-        protocol: 'http',
-        host: 'localhost.lan',
-        port: 8080,
-        version: 'v1'
-    }
-    ,varnish: {
-        host: 'localhost',
-        port: 6082,
-        ttl: 86400
-    }
-};
-
-module.exports = config;

config/environments/development.js.example

@@ -0,0 +1,73 @@
+var config = {
+     environment: 'development'
+    ,port: 8181
+    ,host: '127.0.0.1'
+    // Maximum number of connections for one process
+    // 128 is a good value with a limit of 1024 open file descriptors
+    ,maxConnections:128
+    // idle socket timeout, in miliseconds 
+    ,socket_timeout: 600000
+    ,enable_cors: true
+    ,cache_enabled: false
+    ,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
+    ,postgres_auth_user: 'development_cartodb_user_<%= user_id %>'
+    ,postgres: {
+        // Parameters to pass to datasource plugin of mapnik
+        // See http://github.com/mapnik/mapnik/wiki/PostGIS
+        type: "postgis",
+        user: "publicuser",
+        host: '127.0.0.1',
+        port: 5432,
+        extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
+        /* experimental
+        geometry_field: "the_geom",
+        extent: "-180,-90,180,90",
+        srid: 4326,
+        */
+        row_limit: 65535,
+        simplify_geometries: true,
+        max_size: 500
+    }
+    ,mapnik_version: undefined
+    ,renderer: {
+      // Milliseconds since last access before renderer cache item expires
+      cache_ttl: 60000,
+      metatile: 4,
+      bufferSize: 64
+    }
+    ,millstone: {
+        // Needs to be writable by server user
+        cache_basedir: '/tmp/cdb-tiler-dev/millstone-dev'
+    }
+    ,redis: {
+        host: '127.0.0.1',
+        port: 6379,
+        // Max number of connections in each pool.
+        // Users will be put on a queue when the limit is hit.
+        // Set to maxConnection to have no possible queues.
+        // There are currently 3 pools involved in serving
+        // windshaft-cartodb requests so multiply this number
+        // by 3 to know how many possible connections will be
+        // kept open by the server. The default is 50.
+        max: 50,
+        idleTimeoutMillis: 1, // idle time before dropping connection
+        reapIntervalMillis: 1 // time between cleanups
+    }
+    ,sqlapi: {
+        protocol: 'http',
+        domain: 'localhost.lan',
+        port: 8080,
+        version: 'v1'
+    }
+    ,varnish: {
+        host: 'localhost',
+        port: 6082,
+        ttl: 86400
+    }
+    // If useProfiler is true every response will be served with an
+    // X-Tiler-Profile header containing elapsed timing for various 
+    // steps taken for producing the response.
+    ,useProfiler:true
+};
+
+module.exports = config;

config/environments/production.js

@@ -1,35 +0,0 @@
-var config = {
-    environment: 'production'
-    ,port: 8181
-    ,host: '127.0.0.1'
-    ,enable_cors: true
-    ,cache_enabled: true
-    ,postgres_auth_user: 'cartodb_user_<%= user_id %>'
-    ,postgres: {
-        user: "publicuser",
-        host: '127.0.0.1',
-        port: 6432,
-        extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188",
-        simplify: true
-    }
-    ,millstone: {
-        cache_basedir: '/home/ubuntu/tile_assets/'
-    }
-    ,redis: {
-        host: '127.0.0.1',
-        port: 6379
-    }
-    ,sqlapi: {
-        protocol: 'https',
-        host: 'cartodb.com',
-        port: 8080,
-        version: 'v2'
-    }
-    ,varnish: {
-        host: 'localhost',
-        port: 6082,
-        ttl: 86400
-    }
-};
-
-module.exports = config;

config/environments/production.js.example

@@ -0,0 +1,67 @@
+var config = {
+    environment: 'production'
+    ,port: 8181
+    ,host: '127.0.0.1'
+    // Maximum number of connections for one process
+    // 128 is a good value with a limit of 1024 open file descriptors
+    ,maxConnections:128
+    // idle socket timeout, in miliseconds
+    ,socket_timeout: 600000 
+    ,enable_cors: true
+    ,cache_enabled: true
+    ,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
+    ,postgres_auth_user: 'cartodb_user_<%= user_id %>'
+    ,postgres: {
+        // Parameters to pass to datasource plugin of mapnik
+        // See http://github.com/mapnik/mapnik/wiki/PostGIS
+        user: "publicuser",
+        host: '127.0.0.1',
+        port: 6432,
+        extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
+        row_limit: 65535,
+        simplify_geometries: true,
+        max_size: 500
+    }
+    ,mapnik_version: undefined
+    ,renderer: {
+      // Milliseconds since last access before renderer cache item expires
+      cache_ttl: 60000,
+      metatile: 4,
+      bufferSize: 64
+    }
+    ,millstone: {
+        // Needs to be writable by server user
+        cache_basedir: '/home/ubuntu/tile_assets/'
+    }
+    ,redis: {
+        host: '127.0.0.1',
+        port: 6379,
+        // Max number of connections in each pool.
+        // Users will be put on a queue when the limit is hit.
+        // Set to maxConnection to have no possible queues.
+        // There are currently 3 pools involved in serving
+        // windshaft-cartodb requests so multiply this number
+        // by 3 to know how many possible connections will be
+        // kept open by the server. The default is 50.
+        max: 50,
+        idleTimeoutMillis: 30000, // idle time before dropping connection
+        reapIntervalMillis: 1000 // time between cleanups
+    }
+    ,sqlapi: {
+        protocol: 'https',
+        domain: 'cartodb.com',
+        port: 8080,
+        version: 'v2'
+    }
+    ,varnish: {
+        host: 'localhost',
+        port: 6082,
+        ttl: 86400
+    }
+    // If useProfiler is true every response will be served with an
+    // X-Tiler-Profile header containing elapsed timing for various 
+    // steps taken for producing the response.
+    ,useProfiler:false
+};
+
+module.exports = config;

config/environments/staging.js

@@ -1,35 +0,0 @@
-var config = {
-    environment: 'production'
-    ,port: 8181
-    ,host: '127.0.0.1'
-    ,enable_cors: true
-    ,cache_enabled: true
-    ,postgres_auth_user: 'cartodb_staging_user_<%= user_id %>'
-    ,postgres: {
-        user: "publicuser",
-        host: '127.0.0.1',
-        port: 6432,
-        extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188",
-        simplify: true
-    }
-    ,millstone: {
-        cache_basedir: '/home/ubuntu/tile_assets/'
-    }
-    ,redis: {
-        host: '127.0.0.1',
-        port: 6379
-    }
-    ,sqlapi: {
-        protocol: 'https',
-        host: 'cartodb.com',
-        port: 8080,
-        version: 'v2'
-    }
-    ,varnish: {
-        host: 'localhost',
-        port: 6082,
-        ttl: 86400
-    }
-};
-
-module.exports = config;

config/environments/staging.js.example

@@ -0,0 +1,67 @@
+var config = {
+    environment: 'production'
+    ,port: 8181
+    ,host: '127.0.0.1'
+    // Maximum number of connections for one process
+    // 128 is a good value with a limit of 1024 open file descriptors
+    ,maxConnections:128
+    // idle socket timeout, in miliseconds
+    ,socket_timeout: 600000 
+    ,enable_cors: true
+    ,cache_enabled: true
+    ,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms (:res[X-Tiler-Profiler]) -> :res[Content-Type]'
+    ,postgres_auth_user: 'cartodb_staging_user_<%= user_id %>'
+    ,postgres: {
+        // Parameters to pass to datasource plugin of mapnik
+        // See http://github.com/mapnik/mapnik/wiki/PostGIS
+        user: "publicuser",
+        host: '127.0.0.1',
+        port: 6432,
+        extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
+        row_limit: 65535,
+        simplify_geometries: true,
+        max_size: 500
+    }
+    ,mapnik_version: undefined
+    ,renderer: {
+      // Milliseconds since last access before renderer cache item expires
+      cache_ttl: 60000,
+      metatile: 4,
+      bufferSize: 64
+    }
+    ,millstone: {
+        // Needs to be writable by server user
+        cache_basedir: '/home/ubuntu/tile_assets/'
+    }
+    ,redis: {
+        host: '127.0.0.1',
+        port: 6379,
+        // Max number of connections in each pool.
+        // Users will be put on a queue when the limit is hit.
+        // Set to maxConnection to have no possible queues.
+        // There are currently 3 pools involved in serving
+        // windshaft-cartodb requests so multiply this number
+        // by 3 to know how many possible connections will be
+        // kept open by the server. The default is 50.
+        max: 50,
+        idleTimeoutMillis: 30000, // idle time before dropping connection
+        reapIntervalMillis: 1000 // time between cleanups
+    }
+    ,sqlapi: {
+        protocol: 'https',
+        domain: 'cartodb.com',
+        port: 8080,
+        version: 'v2'
+    }
+    ,varnish: {
+        host: 'localhost',
+        port: 6082,
+        ttl: 86400
+    }
+    // If useProfiler is true every response will be served with an
+    // X-Tiler-Profile header containing elapsed timing for various 
+    // steps taken for producing the response.
+    ,useProfiler:true
+};
+
+module.exports = config;

config/environments/test.js

@@ -1,38 +0,0 @@
-var config = {
-     environment: 'test'
-    ,port: 8888
-    ,host: '127.0.0.1'
-    ,enable_cors: true
-    ,cache_enabled: false
-    ,postgres_auth_user: 'test_cartodb_user_<%= user_id %>'
-    ,postgres: {
-        user: "publicuser",
-        host: '127.0.0.1',
-        port: 5432,
-        srid: 4326,
-        extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188",
-        simplify: true
-    }
-    ,millstone: {
-        cache_basedir: '/tmp/cdb-tiler-test/millstone'
-    }
-    ,redis: {
-        host: '127.0.0.1',
-        port: 6333,
-        idleTimeoutMillis: 1,
-        reapIntervalMillis: 1
-    }
-    ,sqlapi: {
-        protocol: 'http',
-        host: 'localhost.lan',
-        port: 8080,
-        version: 'v1'
-    }
-    ,varnish: {
-        host: '',
-        port: null,
-        ttl: 86400
-    }
-};
-
-module.exports = config;

config/environments/test.js.example

@@ -0,0 +1,69 @@
+var config = {
+     environment: 'test'
+    ,port: 8888
+    ,host: '127.0.0.1'
+    // Maximum number of connections for one process
+    // 128 is a good value with a limit of 1024 open file descriptors
+    ,maxConnections:128
+    // idle socket timeout, in miliseconds
+    ,socket_timeout: 600000 
+    ,enable_cors: true
+    ,cache_enabled: false
+    ,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
+    ,postgres_auth_user: 'test_cartodb_user_<%= user_id %>'
+    ,postgres: {
+        // Parameters to pass to datasource plugin of mapnik
+        // See http://github.com/mapnik/mapnik/wiki/PostGIS
+        user: "publicuser",
+        host: '127.0.0.1',
+        port: 5432,
+        extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
+        row_limit: 65535,
+        simplify_geometries: true,
+        max_size: 500
+    }
+    ,mapnik_version: '2.0.2'
+    ,renderer: {
+      // Milliseconds since last access before renderer cache item expires
+      cache_ttl: 60000,
+      metatile: 4,
+      bufferSize: 64
+    }
+    ,millstone: {
+        // Needs to be writable by server user
+        cache_basedir: '/tmp/cdb-tiler-test/millstone'
+    }
+    ,redis: {
+        host: '127.0.0.1',
+        port: 6333,
+        // Max number of connections in each pool.
+        // Users will be put on a queue when the limit is hit.
+        // Set to maxConnection to have no possible queues.
+        // There are currently 3 pools involved in serving
+        // windshaft-cartodb requests so multiply this number
+        // by 3 to know how many possible connections will be
+        // kept open by the server. The default is 50.
+        max: 50,
+        idleTimeoutMillis: 1, // idle time before dropping connection
+        reapIntervalMillis: 1 // time between cleanups
+    }
+    ,sqlapi: {
+        protocol: 'http',
+        domain: '',
+        // This port will be used by "make check" for testing purposes
+        // It must be available
+        port: 1080,
+        version: 'v1'
+    }
+    ,varnish: {
+        host: '',
+        port: null,
+        ttl: 86400
+    }
+    // If useProfiler is true every response will be served with an
+    // X-Tiler-Profile header containing elapsed timing for various 
+    // steps taken for producing the response.
+    ,useProfiler:false
+};
+
+module.exports = config;

configure

@@ -0,0 +1,78 @@
+#!/bin/sh
+
+#
+# This script creates config/environments/*.js files using
+# config/environments/*.js.example files as input and performing
+# settings substitutions.
+#
+# It relies on a known format of the .js.example files which haven't
+# been made easier to parse to still let humans copy them manually and
+# do further editing or leave them as such to get the same setup as before
+# the introduction of this script.
+#
+# The script is a work in progress. Available switches are printed
+# by invoking with the --help switch. More switches will be added
+# as the need/request for them arises.
+#
+# --strk(2012-07-23)
+#
+
+PGPORT=5432
+SQLAPI_PORT=8080
+MAPNIK_VERSION=
+ENVIRONMENT=development
+
+usage() {
+  echo "Usage: $0 [OPTION]"
+  echo 
+  echo "Configuration:"
+  echo "  --help                        display this help and exit"
+  echo "  --with-pgport=NUM             access PostgreSQL server on TCP port NUM [$PGPORT]"
+  echo "  --with-sqlapi-port=NUM        access SQL-API server on TCP port NUM [$SQLAPI_PORT]"
+  echo "  --with-mapnik-version=STRING  set mapnik version string [$MAPNIK_VERSION]"
+  echo "  --environment=STRING          set output environment name [$ENVIRONMENT]"
+}
+
+while test -n "$1"; do
+  case "$1" in
+    --help|-h)
+      usage
+      exit 0
+      ;;
+    --with-pgport=*)
+      PGPORT=`echo "$1" | cut -d= -f2`
+      ;;
+    --with-sqlapi-port=*)
+      SQLAPI_PORT=`echo "$1" | cut -d= -f2`
+      ;;
+    --with-mapnik-version=*)
+      MAPNIK_VERSION=`echo "$1" | cut -d= -f2`
+      ;;
+    --environment=*)
+      ENVIRONMENT=`echo "$1" | cut -d= -f2`
+      ;;
+    *)
+      echo "Unknown option '$1'" >&2
+      usage >&2
+      exit 1
+  esac
+  shift
+done
+
+echo "PGPORT: $PGPORT"
+echo "SQLAPI_PORT: $SQLAPI_PORT"
+echo "MAPNIK_VERSION: $MAPNIK_VERSION"
+echo "ENVIRONMENT: $ENVIRONMENT"
+
+# TODO: allow specifying configuration settings !
+for f in config/environments/${ENVIRONMENT}.js.example; do
+  o=`dirname "$f"`/`basename "$f" .example`
+  echo "Writing $o"
+
+  # See http://austinmatzko.com/2008/04/26/sed-multi-line-search-and-replace/
+  sed -n "1h;1!H;\${;g;s/\(,postgres: {[^}]*port: *'\?\)[^',]*\('\?,\)/\1$PGPORT\2/;p;}" < "$f" \
+  | sed "s/mapnik_version:.*/mapnik_version: '$MAPNIK_VERSION'/" \
+  | sed -n "1h;1!H;\${;g;s/\(,sqlapi: {[^}]*port: *'\?\)[^',]*\('\?,\)/\1$SQLAPI_PORT\2/;p;}" \
+  > "$o"
+
+done

lib/cartodb/cache_validator.js

@@ -1,8 +1,5 @@
 var _             = require('underscore'),
     Varnish       = require('node-varnish'),
-    request       = require('request'),
-    crypto      = require('crypto'),
-    channelCache  = {},
     varnish_queue = null;
 
 function init(host, port) {
@@ -18,72 +15,7 @@ function invalidate_db(dbname, table) {
     }
 }
 
-function generateCacheChannel(req, callback){
-    var cacheChannel = "";
-
-    // use key to call sql api with sql request if present, else just return dbname and table name
-    // base key
-    var tableNames = req.params.table;
-    var dbName     = req.params.dbname;
-    var username   = req.headers.host.split('.')[0];
-
-    // replace tableNames with the results of the explain if present
-    if (_.isString(req.params.sql) && req.params.sql != ''){
-        // initialise MD5 key of sql for cache lookups
-        var sql_md5 = generateMD5(req.params.sql);
-        var api = global.environment.sqlapi;
-        var qs  = {};
-
-        // use cache if present
-        if (!_.isNull(channelCache[sql_md5]) && !_.isUndefined(channelCache[sql_md5])) {
-            callback(channelCache[sql_md5]);
-        } else{
-            // strip out windshaft/mapnik inserted sql if present
-            var sql = req.params.sql.match(/^\((.*)\)\sas\scdbq$/);
-            sql = (sql != null) ? sql[1] : req.params.sql;
-
-            // build up api string
-            var sqlapi = api.protocol + '://' + username + '.' + api.host + ':' + api.port + '/api/' + api.version + '/sql'
-
-            // add query to querystring
-            qs.q = 'SELECT CDB_QueryTables($windshaft$' + sql + '$windshaft$)';
-
-            // add api_key if present in tile request (means table is private)
-            if (_.isString(req.params.map_key) && req.params.map_key != ''){
-                qs.api_key = req.params.map_key;
-            }
-
-            // call sql api
-            request.get({url:sqlapi, qs:qs, json:true}, function(err, response, body){
-                if (!err && response.statusCode == 200) {
-                    tableNames = body.rows[0].cdb_querytables.split(/^\{(.*)\}$/)[1];
-                } else {
-                    //oops, no SQL API. Just cache using fallback 'table' key
-                    tableNames = 'table';
-                }
-                cacheChannel = buildCacheChannel(dbName,tableNames);
-                channelCache[sql_md5] = cacheChannel; // store for caching
-                callback(cacheChannel);
-            });
-        }
-    } else {
-        cacheChannel = buildCacheChannel(dbName,tableNames);
-        callback(cacheChannel);
-    }
-}
-
-function buildCacheChannel(dbName, tableNames){
-    return dbName + ':' + tableNames;
-}
-
-function generateMD5(data){
-    var hash = crypto.createHash('md5');
-    hash.update(data);
-    return hash.digest('hex');
-}
-
 module.exports = {
     init: init,
-    invalidate_db: invalidate_db,
-    generateCacheChannel: generateCacheChannel
+    invalidate_db: invalidate_db
 }

lib/cartodb/carto_data.js

@@ -7,6 +7,8 @@
  *       and geometry type from the redis stores of cartodb
  */
 
+var strftime    = require('strftime');
+
 var   RedisPool = require("./redis_pool")
     , _ = require('underscore')
     , Step = require('step');
@@ -19,7 +21,9 @@ module.exports = function() {
         user_metadata_db: 5,
         table_metadata_db: 0,
         user_key:  "rails:users:<%= username %>",
-        table_key: "rails:<%= database_name %>:<%= table_name %>"
+        table_key: "rails:<%= database_name %>:<%= table_name %>",
+        global_mapview_key: "user:<%= username %>:mapviews:global",
+        tagged_mapview_key: "user:<%= username %>:mapviews:stat_tag:<%= stat_tag %>"
     };
 
 
@@ -27,16 +31,53 @@ module.exports = function() {
      * Get the database name for this particular subdomain/username
      *
      * @param req - standard express req object. importantly contains host information
-     * @param callback
+     * @param callback - gets called with args(err, dbname) 
      */
     me.getDatabase = function(req, callback) {
         // strip subdomain from header host
         var username = req.headers.host.split('.')[0]
         var redisKey = _.template(this.user_key, {username: username});
 
-        this.retrieve(this.user_metadata_db, redisKey, 'database_name', callback);
+        this.retrieve(this.user_metadata_db, redisKey, 'database_name', function(err, dbname) {
+          if ( err ) callback(err, null);
+          else if ( dbname === null ) {
+            callback(new Error("missing " + username + "'s dbname in redis (try CARTODB/script/restore_redis)"), null);
+          }
+          else callback(err, dbname);
+        });
     };
 
+    me.userFromHostname = function(hostname) {
+        return hostname.split('.')[0];
+    }
+
+    /**
+     * Increment mapview count for a user
+     *
+     * @param username 
+     * @param stat_tag
+     * @param callback will be called with the new value
+     */
+    me.incMapviewCount = function(username, stat_tag, callback) {
+        var that = this;
+        var now = strftime("%Y%m%d", new Date());
+        var redisKey;
+        Step (
+          function incrementGlobal() {
+            redisKey = _.template(that.global_mapview_key, {username: username});
+            that.redisCmd(me.user_metadata_db, 'ZINCRBY', [redisKey, 1, now], this);
+          },
+          function incrementTag(err, val) {
+            if ( err ) throw err;
+            if ( _.isUndefined(stat_tag) ) return 1;
+            redisKey = _.template(that.tagged_mapview_key, {username: username, stat_tag: stat_tag});
+            that.redisCmd(me.user_metadata_db, 'ZINCRBY', [redisKey, 1, now], this);
+          },
+          function finish(err, val) {
+            if ( callback ) callback(err);
+          }
+        );
+    };
 
 
     /**
@@ -50,7 +91,13 @@ module.exports = function() {
         var username = req.headers.host.split('.')[0];
         var redisKey = _.template(this.user_key, {username: username});
 
-        this.retrieve(this.user_metadata_db, redisKey, 'id', callback);
+        this.retrieve(this.user_metadata_db, redisKey, 'id', function(err, dbname) {
+          if ( err ) callback(err, null);
+          else if ( dbname === null ) {
+            callback(new Error("missing " + username + "'s dbuser in redis (try CARTODB/script/restore_redis)"), null);
+          }
+          else callback(err, dbname);
+        });
     };
 
     /**
@@ -65,7 +112,13 @@ module.exports = function() {
         var redisKey = "rails:users:" + username; 
         this.retrieve(this.user_metadata_db, redisKey, "map_key", function(err, val) {
             var valid = 0;
-            if ( val && val == req.query.map_key ) valid = 1;
+            if ( val ) {
+              if ( val == req.query.map_key ) valid = 1;
+              else if ( val == req.query.api_key ) valid = 1;
+              // check also in request body 
+              else if ( req.body && req.body.map_key && val == req.body.map_key ) valid = 1;
+              else if ( req.body && req.body.api_key && val == req.body.api_key ) valid = 1;
+            }
             callback(err, valid);
         });
     };
@@ -88,9 +141,9 @@ module.exports = function() {
                 if (check_result === 1) {
                     // authorized by key, login as db owner 
                     that.getId(req, function(err, user_id) {
-                        if (err) throw new Error(err);
+                        if (err) throw err;
                         var dbuser = _.template(global.settings.postgres_auth_user, {user_id: user_id});
-                        _.extend(req, {dbuser:dbuser});
+                        _.extend(req.params, {dbuser:dbuser});
                         callback(err, true); 
                     });
                 } else {
@@ -108,7 +161,6 @@ module.exports = function() {
                 that.retrieve(that.table_metadata_db, redisKey, 'privacy', this);
             },
             function(err, data){
-                if (err) throw err;
                 callback(err, data);
              }
         );
@@ -134,7 +186,6 @@ module.exports = function() {
                 that.retrieve(that.table_metadata_db, redisKey, 'the_geom_type', this);
             },
             function(err, data){
-                if (err) throw err;
                 callback(err, data);
             }
         );
@@ -154,7 +205,6 @@ module.exports = function() {
                 that.retrieve(that.table_metadata_db, redisKey, 'infowindow', this);
             },
             function(err, data){
-                if (err) throw err;
                 callback(err, data);
             }
         );
@@ -175,13 +225,14 @@ module.exports = function() {
                 that.retrieve(that.table_metadata_db, redisKey, 'map_metadata', this);
             },
             function(err, data){
-                if (err) throw err;
                 callback(err, data);
             }
         );
     };
 
     // Redis Hash lookup
+    // @param callback will be invoked with args (err, reply)
+    //                 note that reply is null when the key is missing
     me.retrieve = function(db, redisKey, hashKey, callback) {
         this.redisCmd(db,'HGET',[redisKey, hashKey], callback);
     };
@@ -191,6 +242,11 @@ module.exports = function() {
         this.redisCmd(db,'SISMEMBER',[setKey, member], callback);
     };
 
+    // Redis INCREMENT 
+    me.increment = function(db, key, callback) {
+        this.redisCmd(db,'INCR', key, callback);
+    };
+
     /**
      * Use Redis
      *
@@ -207,13 +263,13 @@ module.exports = function() {
                 redis_pool.acquire(db, this);
             },
             function executeQuery(err, data) {
+                if ( err ) throw err;
                 redisClient = data;
                 redisArgs.push(this);
                 redisClient[redisFunc.toUpperCase()].apply(redisClient, redisArgs);
             },
             function releaseRedisClient(err, data) {
-                if (err) throw err;
-                redis_pool.release(db, redisClient);
+                if ( ! _.isUndefined(redisClient) ) redis_pool.release(db, redisClient);
                 callback(err, data);
             }
         );

lib/cartodb/cartodb_windshaft.js

@@ -6,17 +6,6 @@ var _ = require('underscore')
 
 var CartodbWindshaft = function(serverOptions) {
 
-    // set the cache chanel info to invalidate the cache on the frontend server
-    serverOptions.afterTileRender = function(req, res, tile, headers, callback) {
-        var ttl = global.environment.varnish.ttl || 86400;
-        Cache.generateCacheChannel(req, function(channel){
-            res.header('X-Cache-Channel', channel);
-            res.header('Last-Modified', new Date().toUTCString());
-            res.header('Cache-Control', 'no-cache,max-age='+ttl+',must-revalidate, public');
-            callback(null, tile, headers);
-        });
-    };
-
     if(serverOptions.cache_enabled) {
         console.log("cache invalidation enabled, varnish on ", serverOptions.varnish_host, ' ', serverOptions.varnish_port);
         Cache.init(serverOptions.varnish_host, serverOptions.varnish_port);
@@ -28,7 +17,7 @@ var CartodbWindshaft = function(serverOptions) {
 
     serverOptions.beforeStateChange = function(req, callback) {
         var err = null;
-        if ( ! req.hasOwnProperty('dbuser') ) {
+        if ( ! req.params.hasOwnProperty('dbuser') ) {
           err = new Error("map state cannot be changed by unauthenticated request!");
         }
         callback(err, req);
@@ -37,6 +26,14 @@ var CartodbWindshaft = function(serverOptions) {
     // boot
     var ws = new Windshaft.Server(serverOptions);
 
+    // Override getVersion to include cartodb-specific versions
+    var wsversion = ws.getVersion;
+    ws.getVersion = function() {
+      var version = wsversion();
+      version.windshaft_cartodb = require('../../package.json').version;
+      return version;
+    }
+
     /**
      * Helper to allow access to the layer to be used in the maps infowindow popup.
      */
@@ -48,7 +45,8 @@ var CartodbWindshaft = function(serverOptions) {
             },
             function(err, data){
                 if (err){
-                    res.send({error: err.message}, 500);
+                    ws.sendError(res, {error: err.message}, 500, 'GET INFOWINDOW');
+                    //res.send({error: err.message}, 500);
                 } else {
                     res.send({infowindow: data}, 200);
                 }
@@ -68,7 +66,8 @@ var CartodbWindshaft = function(serverOptions) {
             },
             function(err, data){
                 if (err){
-                    res.send(err.message, 500);
+                    ws.sendError(res, {error: err.message}, 500, 'GET MAP_METADATA');
+                    //res.send(err.message, 500);
                 } else {
                     res.send({map_metadata: data}, 200);
                 }
@@ -83,12 +82,13 @@ var CartodbWindshaft = function(serverOptions) {
     ws.del(serverOptions.base_url + '/flush_cache', function(req, res){
         ws.doCORS(res);
         Step(
-            function(){
-                serverOptions.flushCache(req, Cache, this);
+            function flushCache(){
+                serverOptions.flushCache(req, serverOptions.cache_enabled ? Cache : null, this);
             },
-            function(err, data){
+            function sendResponse(err, data){
                 if (err){
-                    res.send(500);
+                    ws.sendError(res, {error: err.message}, 500, 'DELETE CACHE');
+                    //res.send(500);
                 } else {
                     res.send({status: 'ok'}, 200);
                 }

lib/cartodb/redis_pool.js

@@ -58,8 +58,11 @@ var RedisPool = function(opts){
           client.send_anyway = true;
           client.select(database);  
           client.send_anyway = false;
+          callback(null, client);
         });    
-        return callback(null, client);
+        client.on('error', function (err) {
+          callback(err, null);
+        });
       },
       destroy: function(client) { 
         return client.quit(); 
@@ -74,4 +77,4 @@ var RedisPool = function(opts){
   return me;
 };
 
-module.exports = RedisPool;
+module.exports = RedisPool;

lib/cartodb/server_options.js

@@ -1,22 +1,307 @@
 var   _          = require('underscore')
     , Step       = require('step')
-    , cartoData  = require('./carto_data');
+    , cartoData  = require('./carto_data')
+    , Cache = require('./cache_validator')
+    , mapnik     = require('mapnik')
+    , crypto     = require('crypto')
+    , request    = require('request')
+    , LZMA       = require('lzma/lzma_worker.js').LZMA
+  ;
+
+// This is for backward compatibility with 1.3.3
+if ( _.isUndefined(global.environment.sqlapi.domain) ) {
+  global.environment.sqlapi.domain = global.environment.sqlapi.host;
+}
 
 module.exports = function(){
+
+    var rendererConfig = _.defaults(global.environment.renderer || {}, {
+      cache_ttl: 60000, // milliseconds
+      metatile: 4,
+      bufferSize: 64
+    });
+
     var me = {
         base_url: '/tiles/:table',
+        base_url_notable: '/tiles',
         grainstore: {
           datasource: global.environment.postgres,
-          cachedir: global.environment.millstone.cache_basedir
+          cachedir: global.environment.millstone.cache_basedir,
+          mapnik_version: global.environment.mapnik_version || mapnik.versions.mapnik,
+          default_layergroup_ttl: 7200 // seconds (defaultis 300)
+        },
+        mapnik: {
+          metatile: rendererConfig.metatile,
+          bufferSize: rendererConfig.bufferSize
+        },
+        renderCache: {
+          ttl: rendererConfig.cache_ttl
         },
         redis: global.environment.redis,
         enable_cors: global.environment.enable_cors,
         varnish_host: global.environment.varnish.host,
         varnish_port: global.environment.varnish.port,
         cache_enabled: global.environment.cache_enabled,
-        log_format: '[:date] :req[X-Real-IP] \033[90m:method\033[0m \033[36m:req[Host]:url\033[0m \033[90m:status :response-time ms -> :res[Content-Type]\033[0m'
+        log_format: global.environment.log_format,
+        useProfiler: global.environment.useProfiler
     };
 
+    // Be nice and warn if configured mapnik version
+    // is != instaled mapnik version
+    if ( mapnik.versions.mapnik != me.grainstore.mapnik_version ) {
+      console.warn("WARNING: detected mapnik version ("
+        + mapnik.versions.mapnik + ") != configured mapnik version ("
+        + me.grainstore.mapnik_version + ")");
+    }
+
+/* This whole block is about generating X-Cache-Channel { */
+
+    // TODO: review lifetime of elements of this cache
+    // NOTE: by-token indices should only be dropped when
+    //       the corresponding layegroup is dropped, because
+    //       we have no SQL after layer creation.
+    me.channelCache = {};
+
+    // Run a query through the SQL api
+    me.sqlQuery = function (username, api_key, sql, callback) {
+      var api = global.environment.sqlapi;
+
+      // build up api string
+      var sqlapi = api.protocol + '://' + username + '.' + api.domain + ':' + api.port + '/api/' + api.version + '/sql'
+
+      var qs  = { q: sql }
+
+      // add api_key if given
+      if (_.isString(api_key) && api_key != '') { qs.api_key = api_key; }
+
+      // call sql api
+      request.get({url:sqlapi, qs:qs, json:true}, function(err, res, body){
+          if (err){
+            console.log('ERROR running connecting to SQL API on ' + sqlapi + ': ' + err);
+            callback(err);
+            return;
+          }
+          if (res.statusCode != 200) {
+              var msg = res.body.error ? res.body.error : res.body;
+              callback(new Error(msg));
+              console.log('unexpected response status (' + res.statusCode + ') for sql query: ' + sql + ': ' + msg);
+              return;
+          }
+          callback(null, body.rows);
+      });
+    };
+
+    //
+    // Invoke callback with number of milliseconds since
+    // last update in any of the given tables
+    //
+    me.findLastUpdated = function (username, api_key, tableNames, callback) {
+      var sql = 'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max FROM CDB_TableMetadata m WHERE m.tabname::name = any (\'{'
+        + tableNames.join(',') + '}\')';
+
+      // call sql api
+      me.sqlQuery(username, api_key, sql, function(err, rows){
+          if (err){
+            var msg = err.message ? err.message : err;
+            callback(new Error('could not find last updated timestamp: ' + msg));
+            return;
+          }
+          // when the table has not updated_at means it hasn't been changed so a default last_updated is set
+          var last_updated = 0;
+          if(rows.length !== 0) {
+            last_updated = rows[0].max || 0;
+          }
+          callback(null, last_updated*1000);
+      });
+    };
+
+    me.affectedTables = function (username, api_key, sql, callback) {
+
+      // Replace mapnik tokens
+      sql = sql.replace(RegExp('!bbox!', 'g'), 'ST_MakeEnvelope(0,0,0,0)')
+               .replace(RegExp('!pixel_width!', 'g'), '1')
+               .replace(RegExp('!pixel_height!', 'g'), '1')
+            ;
+
+      // Pass to CDB_QueryTables
+      sql = 'SELECT CDB_QueryTables($windshaft$' + sql + '$windshaft$)';
+
+      // call sql api
+      me.sqlQuery(username, api_key, sql, function(err, rows){
+          if (err){
+            var msg = err.message ? err.message : err;
+            callback(new Error('could not fetch source tables: ' + msg));
+            return;
+          }
+          var qtables = rows[0].cdb_querytables;
+          var tableNames = qtables.split(/^\{(.*)\}$/)[1];
+          tableNames = tableNames.split(',');
+          callback(null, tableNames);
+      });
+    };
+
+    me.buildCacheChannel = function (dbName, tableNames){
+      return dbName + ':' + tableNames.join(',');
+    };
+
+    me.generateMD5 = function(data){
+        var hash = crypto.createHash('md5');
+        hash.update(data);
+        return hash.digest('hex');
+    }
+
+    me.generateCacheChannel = function(req, callback){
+
+      // use key to call sql api with sql request if present, else
+      // just return dbname and table name base key
+      var dbName     = req.params.dbname;
+
+      var cacheKey = [ dbName ];
+      if ( req.params.token ) cacheKey.push(req.params.token);
+      else if ( req.params.sql ) cacheKey.push( me.generateMD5(req.params.sql) );
+      cacheKey = cacheKey.join(':');
+
+      if ( me.channelCache.hasOwnProperty(cacheKey) ) {
+        callback(null, me.channelCache[cacheKey]);
+        return;
+      }
+      else if ( req.params.token ) {
+        // cached cache channel for token-based access should be constructed
+        // at cache creation time
+        callback(new Error('missing channel cache for token ' + req.params.token));
+        return;
+      }
+
+      if ( ! req.params.sql && ! req.params.token ) {
+        var cacheChannel = me.buildCacheChannel(dbName, [req.params.table]);
+        // not worth caching this
+        callback(null, cacheChannel);
+        return;
+      }
+
+      if ( ! req.params.sql ) {
+        callback(new Error("this request doesn't need an X-Cache-Channel generated"));
+        return;
+      }
+
+      var dbName     = req.params.dbname;
+      var username   = req.headers.host.split('.')[0];
+
+      // strip out windshaft/mapnik inserted sql if present
+      var sql = req.params.sql.match(/^\((.*)\)\sas\scdbq$/);
+      sql = (sql != null) ? sql[1] : req.params.sql;
+
+      me.affectedTables(username, req.params.map_key, sql, function(err, tableNames) {
+          if ( err ) { callback(err); return; }
+          var cacheChannel = me.buildCacheChannel(dbName,tableNames);
+          me.channelCache[cacheKey] = cacheChannel; // store for caching
+          callback(null, cacheChannel);
+      });
+    };
+
+    // Set the cache chanel info to invalidate the cache on the frontend server
+    //
+    // @param req The request object.
+    //            The function will have no effect unless req.res exists.
+    //            It is expected that req.params contains 'table' and 'dbname'
+    //
+    // @param cb function(err, channel) will be called when ready.
+    //           the channel parameter will be null if nothing was added
+    //
+    me.addCacheChannel = function(req, cb) {
+        // skip non-GET requests, or requests for which there's no response
+        if ( req.method != 'GET' || ! req.res ) { cb(null, null); return; }
+        var res = req.res;
+        var cache_policy = req.query.cache_policy;
+        if ( req.params.token ) cache_policy = 'persist';
+        if ( cache_policy == 'persist' ) {
+          res.header('Cache-Control', 'public,max-age=31536000'); // 1 year
+        } else {
+          var ttl = global.environment.varnish.ttl || 86400;
+          res.header('Cache-Control', 'no-cache,max-age='+ttl+',must-revalidate, public');
+        }
+
+        // Set Last-Modified header
+        var lastUpdated;
+        if ( req.params.cache_buster ) {
+          // Assuming cache_buster is a timestamp
+          // FIXME: store lastModified in the cache channel instead
+          lastUpdated = new Date(parseInt(req.params.cache_buster));
+        } else {
+          lastUpdated = new Date();
+        }
+        res.header('Last-Modified', lastUpdated.toUTCString());
+
+        me.generateCacheChannel(req, function(err, channel){
+            if ( ! err ) {
+              res.header('X-Cache-Channel', channel);
+              cb(null, channel);
+            } else {
+              console.log('ERROR generating cache channel: ' + ( err.message ? err.message : err ));
+              // TODO: evaluate if we should bubble up the error instead
+              cb(null, 'ERROR');
+            }
+        });
+    };
+
+    me.afterLayergroupCreate = function(req, mapconfig, response, callback) {
+        var token = response.layergroupid;
+
+        var username = cartoData.userFromHostname(req.headers.host);
+
+        var tasksleft = 2; // redis key and affectedTables
+        var errors = [];
+
+        var done = function(err) {
+          if ( err ) {
+            errors.push('' + err);
+          }
+          if ( ! --tasksleft ) {
+            err = errors.length ? new Error(errors.join('\n')) : null;
+            callback(err);
+          }
+        }
+
+        // Don't wait for the mapview count increment to
+        // take place before proceeding. Error will be logged
+        // asyncronously
+        cartoData.incMapviewCount(username, mapconfig.stat_tag, function(err) {
+          if (req.profiler) req.profiler.done('incMapviewCount');
+          if ( err ) console.log("ERROR: failed to increment mapview count for user '" + username + "': " + err);
+          done();
+        });
+
+        var sql = [];
+        _.each(mapconfig.layers, function(lyr) {
+          sql.push(lyr.options.sql);
+        });
+        sql = sql.join(';');
+
+        var dbName = req.params.dbname;
+        var usr    = req.headers.host.split('.')[0];
+        var key    = req.params.map_key;
+
+        var cacheKey = dbName + ':' + token;
+
+        me.affectedTables(usr, key, sql, function(err, tableNames) {
+            if (req.profiler) req.profiler.done('affectedTables');
+
+            if ( err ) { done(err); return; }
+            var cacheChannel = me.buildCacheChannel(dbName,tableNames);
+            me.channelCache[cacheKey] = cacheChannel; // store for caching
+            // find last updated
+            me.findLastUpdated(usr, key, tableNames, function(err, lastUpdated) {
+              if (req.profiler) req.profiler.done('findLastUpdated');
+              if ( err ) { done(err); return; }
+              response.layergroupid = response.layergroupid + ':' + lastUpdated; // use epoch
+              response.last_updated = new Date(lastUpdated).toISOString(); // TODO: use ISO format
+              done(null);
+            });
+        });
+    };
+
+/* X-Cache-Channel generation } */
+
     /**
      * Whitelist input and get database name & default geometry type from
      * subdomain/user metadata held in CartoDB Redis
@@ -25,13 +310,48 @@ module.exports = function(){
      */
     me.req2params = function(req, callback){
 
+        if ( req.query.lzma ) {
+
+          // TODO: check ?
+          //console.log("type of req.query.lzma is " + typeof(req.query.lzma));
+
+          // Decode (from base64)
+          var lzma = (new Buffer(req.query.lzma, 'base64').toString('binary')).split('').map(function(c) { return c.charCodeAt(0) - 128 })
+
+          // Decompress
+          LZMA.decompress(
+            lzma,
+            function(result) {
+              if (req.profiler) req.profiler.done('LZMA decompress');
+              try {
+                delete req.query.lzma
+                _.extend(req.query, JSON.parse(result))
+                me.req2params(req, callback);
+              } catch (err) {
+                callback(new Error('Error parsing lzma as JSON: ' + err));
+              }
+            },
+            function(percent) { // progress
+              //console.log("LZMA decompression " + percent + "%");
+            }
+          );
+          return;
+        }
+
         // Whitelist query parameters and attach format
-        var good_query = ['sql', 'geom_type', 'cache_buster','callback', 'interactivity', 'map_key', 'style'];
+        var good_query = ['sql', 'geom_type', 'cache_buster', 'cache_policy', 'callback', 'interactivity', 'map_key', 'api_key', 'style', 'style_version', 'style_convert', 'config' ];
         var bad_query  = _.difference(_.keys(req.query), good_query);
 
         _.each(bad_query, function(key){ delete req.query[key]; });
         req.params =  _.extend({}, req.params); // shuffle things as request is a strange array/object
 
+        if ( req.params.token ) {
+          //console.log("Request parameters include token " + req.params.token);
+          var tksplit = req.params.token.split(':');
+          req.params.token = tksplit[0];
+          if ( tksplit.length > 1 ) req.params.cache_buster= tksplit[1];
+        }
+
         // bring all query values onto req.params object
         _.extend(req.params, req.query);
 
@@ -39,17 +359,22 @@ module.exports = function(){
         req.params.interactivity = req.params.interactivity || 'cartodb_id';
 
         req.params.processXML = function(req, xml, callback) {
-          var dbuser = req.dbuser ? req.dbuser : global.settings.postgres.user;
-          if ( ! me.rx_dbuser ) me.rx_dbuser = /(<Parameter name="user"><!\[CDATA\[)[^\]]*(]]><\/Parameter>)/;
+          var dbuser = req.params.dbuser || global.settings.postgres.user;
+          if ( ! me.rx_dbuser ) me.rx_dbuser = /(<Parameter name="user"><!\[CDATA\[)[^\]]*(]]><\/Parameter>)/g;
           xml = xml.replace(me.rx_dbuser, "$1" + dbuser + "$2");
           callback(null, xml);
         }
 
+        var that = this;
+
+        if (req.profiler) req.profiler.done('req2params.setup');
+
         Step(
             function getPrivacy(){
                 cartoData.authorize(req, this);
             },
             function gatekeep(err, data){
+        if (req.profiler) req.profiler.done('cartoData.authorize');
                 if(err) throw err;
                 if(data === "0") throw new Error("Sorry, you are unauthorized (permission denied)");
                 return data;
@@ -60,16 +385,23 @@ module.exports = function(){
                 cartoData.getDatabase(req, this);
             },
             function getGeometryType(err, data){
+        if (req.profiler) req.profiler.done('cartoData.getDatabase');
                 if (err) throw err;
                 _.extend(req.params, {dbname:data});
 
                 cartoData.getGeometryType(req, this);
             },
             function finishSetup(err, data){
+        if (req.profiler) req.profiler.done('cartoData.getGeometryType');
+                if ( err ) { callback(err, req); return; }
+
                 if (!_.isNull(data))
                     _.extend(req.params, {geom_type: data});
 
-                callback(err, req);
+                that.addCacheChannel(req, function(err) {
+        if (req.profiler) req.profiler.done('addCacheChannel');
+                  callback(err, req);
+                });
             }
         );
     };
@@ -106,8 +438,8 @@ module.exports = function(){
                 that.req2params(req, this);
             },
             function(err, data){
-                if (err) throw err;
-                cartoData.getMapMetadata(data, callback);
+                if (err) callback(err, null);
+                else cartoData.getMapMetadata(data, callback);
             }
         );
     };
@@ -121,12 +453,20 @@ module.exports = function(){
         var that = this;
 
         Step(
-            function(){
+            function getParams(){
+                // this is mostly to compute req.params.dbname
                 that.req2params(req, this);
             },
-            function(err, data){
-                if (err) throw err;
-                Cache.invalidate_db(req.params.dbname, req.params.table);
+            function flushInternalCache(err){
+                // TODO: implement this, see
+                // http://github.com/Vizzuality/Windshaft-cartodb/issues/73
+                return true;
+            },
+            function flushVarnishCache(err){
+                if (err) { callback(err); return; }
+                if(Cache) {
+                  Cache.invalidate_db(req.params.dbname, req.params.table);
+                }
                 callback(null, true);
             }
         );

npm-shrinkwrap.json

@@ -0,0 +1,316 @@
+{
+  "name": "windshaft-cartodb",
+  "version": "1.4.1",
+  "dependencies": {
+    "node-varnish": {
+      "version": "0.1.1"
+    },
+    "underscore": {
+      "version": "1.3.3"
+    },
+    "windshaft": {
+      "version": "0.14.2",
+      "dependencies": {
+        "grainstore": {
+          "version": "0.14.2",
+          "dependencies": {
+            "carto": {
+              "version": "0.9.5-cdb2",
+              "from": "git://github.com/CartoDB/carto.git#0.9.5-cdb2",
+              "dependencies": {
+                "underscore": {
+                  "version": "1.4.4"
+                },
+                "xml2js": {
+                  "version": "0.2.8",
+                  "dependencies": {
+                    "sax": {
+                      "version": "0.5.5"
+                    }
+                  }
+                },
+                "optimist": {
+                  "version": "0.6.0",
+                  "dependencies": {
+                    "wordwrap": {
+                      "version": "0.0.2"
+                    },
+                    "minimist": {
+                      "version": "0.0.5"
+                    }
+                  }
+                }
+              }
+            },
+            "mapnik-reference": {
+              "version": "5.0.7"
+            },
+            "millstone": {
+              "version": "0.6.8",
+              "dependencies": {
+                "underscore": {
+                  "version": "1.5.2"
+                },
+                "request": {
+                  "version": "2.26.0",
+                  "dependencies": {
+                    "qs": {
+                      "version": "0.6.5"
+                    },
+                    "json-stringify-safe": {
+                      "version": "5.0.0"
+                    },
+                    "forever-agent": {
+                      "version": "0.5.0"
+                    },
+                    "tunnel-agent": {
+                      "version": "0.3.0"
+                    },
+                    "http-signature": {
+                      "version": "0.10.0",
+                      "dependencies": {
+                        "assert-plus": {
+                          "version": "0.1.2"
+                        },
+                        "asn1": {
+                          "version": "0.1.11"
+                        },
+                        "ctype": {
+                          "version": "0.5.2"
+                        }
+                      }
+                    },
+                    "hawk": {
+                      "version": "1.0.0",
+                      "dependencies": {
+                        "hoek": {
+                          "version": "0.9.1"
+                        },
+                        "boom": {
+                          "version": "0.4.2"
+                        },
+                        "cryptiles": {
+                          "version": "0.2.2"
+                        },
+                        "sntp": {
+                          "version": "0.2.4"
+                        }
+                      }
+                    },
+                    "aws-sign": {
+                      "version": "0.3.0"
+                    },
+                    "oauth-sign": {
+                      "version": "0.3.0"
+                    },
+                    "cookie-jar": {
+                      "version": "0.3.0"
+                    },
+                    "node-uuid": {
+                      "version": "1.4.1"
+                    },
+                    "form-data": {
+                      "version": "0.1.2",
+                      "dependencies": {
+                        "combined-stream": {
+                          "version": "0.0.4",
+                          "dependencies": {
+                            "delayed-stream": {
+                              "version": "0.0.5"
+                            }
+                          }
+                        },
+                        "async": {
+                          "version": "0.2.9"
+                        }
+                      }
+                    }
+                  }
+                },
+                "srs": {
+                  "version": "0.3.8"
+                },
+                "zipfile": {
+                  "version": "0.4.2"
+                },
+                "sqlite3": {
+                  "version": "2.1.19",
+                  "dependencies": {
+                    "tar.gz": {
+                      "version": "0.1.1",
+                      "dependencies": {
+                        "fstream": {
+                          "version": "0.1.24",
+                          "dependencies": {
+                            "rimraf": {
+                              "version": "2.2.2"
+                            },
+                            "graceful-fs": {
+                              "version": "2.0.1"
+                            },
+                            "inherits": {
+                              "version": "2.0.1"
+                            }
+                          }
+                        },
+                        "tar": {
+                          "version": "0.1.18",
+                          "dependencies": {
+                            "inherits": {
+                              "version": "2.0.1"
+                            },
+                            "block-stream": {
+                              "version": "0.0.7"
+                            }
+                          }
+                        },
+                        "commander": {
+                          "version": "1.1.1",
+                          "dependencies": {
+                            "keypress": {
+                              "version": "0.1.0"
+                            }
+                          }
+                        }
+                      }
+                    }
+                  }
+                },
+                "mime": {
+                  "version": "1.2.11"
+                },
+                "mkdirp": {
+                  "version": "0.3.5"
+                },
+                "optimist": {
+                  "version": "0.6.0",
+                  "dependencies": {
+                    "wordwrap": {
+                      "version": "0.0.2"
+                    },
+                    "minimist": {
+                      "version": "0.0.5"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        },
+        "express": {
+          "version": "2.5.11",
+          "dependencies": {
+            "connect": {
+              "version": "1.9.2",
+              "dependencies": {
+                "formidable": {
+                  "version": "1.0.14"
+                }
+              }
+            },
+            "mime": {
+              "version": "1.2.4"
+            },
+            "qs": {
+              "version": "0.4.2"
+            },
+            "mkdirp": {
+              "version": "0.3.0"
+            }
+          }
+        },
+        "tilelive": {
+          "version": "4.4.3",
+          "dependencies": {
+            "optimist": {
+              "version": "0.3.7",
+              "dependencies": {
+                "wordwrap": {
+                  "version": "0.0.2"
+                }
+              }
+            },
+            "sphericalmercator": {
+              "version": "1.0.2"
+            }
+          }
+        },
+        "tilelive-mapnik": {
+          "version": "0.5.0",
+          "from": "git://github.com/Vizzuality/tilelive-mapnik.git#5908346",
+          "dependencies": {
+            "eio": {
+              "version": "0.2.2"
+            },
+            "mime": {
+              "version": "1.2.11"
+            },
+            "sphericalmercator": {
+              "version": "1.0.2"
+            }
+          }
+        },
+        "lru-cache": {
+          "version": "2.3.1"
+        }
+      }
+    },
+    "step": {
+      "version": "0.0.5"
+    },
+    "generic-pool": {
+      "version": "2.0.3"
+    },
+    "redis": {
+      "version": "0.8.3"
+    },
+    "hiredis": {
+      "version": "0.1.15",
+      "dependencies": {
+        "bindings": {
+          "version": "1.1.0"
+        }
+      }
+    },
+    "request": {
+      "version": "2.9.202"
+    },
+    "mapnik": {
+      "version": "0.7.22"
+    },
+    "strftime": {
+      "version": "0.6.0"
+    },
+    "lzma": {
+      "version": "1.2.3"
+    },
+    "semver": {
+      "version": "1.1.4"
+    },
+    "mocha": {
+      "version": "1.2.1",
+      "dependencies": {
+        "commander": {
+          "version": "0.6.1"
+        },
+        "growl": {
+          "version": "1.5.1"
+        },
+        "jade": {
+          "version": "0.26.3",
+          "dependencies": {
+            "mkdirp": {
+              "version": "0.3.0"
+            }
+          }
+        },
+        "diff": {
+          "version": "1.0.2"
+        },
+        "debug": {
+          "version": "0.7.2"
+        }
+      }
+    }
+  }
+}

package.json

@@ -1,37 +1,39 @@
 {
     "private": true,
     "name": "windshaft-cartodb",
-    "version": "0.1.0",
+    "version": "1.4.1",
     "description": "A map tile server for CartoDB",
-    "url": "https://github.com/Vizzuality/Windshaft-cartodb",
+    "url": "https://github.com/CartoDB/Windshaft-cartodb",
     "licenses": [{
         "type": "BSD",
-        "url": "https://github.com/Vizzuality/Windshaft-cartodb/blob/master/LICENCE"
+        "url": "https://github.com/CartoDB/Windshaft-cartodb/blob/master/LICENCE"
     }],
-    "repositories": [{
+    "repository": {
         "type": "git",
-        "url": "git://github.com/Vizzuality/Windshaft-cartodb.git"
-    }],
-    "author": {
-      "name": "Simon Tokumine, Javi Santana, Vizzuality",
-      "url": "http://vizzuality.com",
-      "email": "simon@vizzuality.com"
+        "url": "git://github.com/CartoDB/Windshaft-cartodb.git"
     },
+    "author": "Vizzuality <contact@vizzuality.com> (http://vizzuality.com)",
+    "contributors": [
+      "Simon Tokumine <simon@vizzuality.com>",
+      "Javi Santana <jsantana@vizzuality.com>",
+      "Sandro Santilli <strk@vizzuality.com>"
+    ],
     "dependencies": {
-        "connect": "1.8.7",
-        "cluster": "0.6.4",
         "node-varnish": "0.1.1",
-        "underscore"  : "1.1.x",
-        "grainstore"  : "~0.6.2",
-        "windshaft"  : "~0.4.14",
+        "underscore"  : "~1.3.3",
+        "windshaft"  : "~0.14.2",
         "step": "0.0.x",
-        "generic-pool": "1.0.x",
-        "redis": "0.7.2",
-        "hiredis": "~0.1.12",
-        "request": "2.9.202"
+        "generic-pool": "~2.0.3",
+        "redis": "~0.8.3",
+        "hiredis": "~0.1.14",
+        "request": "2.9.202",
+        "mapnik": "~0.7.22",
+        "strftime": "~0.6.0",
+        "lzma": "~1.2.3"
     },
     "devDependencies": {
-        "mocha": "1.2.1"
+        "mocha": "1.2.1",
+        "semver": "~1.1.0"
     },
     "scripts": {
         "test": "make check"

run_tests.sh

@@ -1,11 +1,27 @@
 #!/bin/sh
 
-# Must match config.redis_pool.port in test/support/config.js
-REDIS_PORT=6333
+OPT_CREATE=yes # create the test environment
+OPT_DROP=yes   # drop the test environment
+
+cd $(dirname $0)
+BASEDIR=$(pwd)
+cd -
+
+REDIS_PORT=`node -e "console.log(require('${BASEDIR}/config/environments/test.js').redis.port)"`
+export REDIS_PORT
 
 cleanup() {
-	echo "Cleaning up"
-	kill ${PID_REDIS}
+  if test x"$OPT_DROP" = xyes; then
+    if test x"$PID_REDIS" = x; then
+      PID_REDIS=$(cat ${BASEDIR}/redis.pid)
+      if test x"$PID_REDIS" = x; then
+        echo "Could not find a test redis pid to kill it"
+        return;
+      fi
+    fi
+    echo "Cleaning up"
+    kill ${PID_REDIS}
+  fi
 }
 
 cleanup_and_exit() {
@@ -22,21 +38,46 @@ die() {
 
 trap 'cleanup_and_exit' 1 2 3 5 9 13
 
-echo "Starting redis on port ${REDIS_PORT}"
-echo "port ${REDIS_PORT}" | redis-server - > test.log &
-PID_REDIS=$!
+while [ -n "$1" ]; do
+        if test "$1" = "--nodrop"; then
+                OPT_DROP=no
+                shift
+                continue
+        elif test "$1" = "--nocreate"; then
+                OPT_CREATE=no
+                shift
+                continue
+        else
+                break
+        fi
+done
 
-echo "Preparing the database"
-cd test; sh prepare_db.sh >> test.log || die "database preparation failure (see test.log)"; cd -;
+if [ -z "$1" ]; then
+        echo "Usage: $0 [<options>] <test> [<test>]" >&2
+        echo "Options:" >&2
+        echo " --nocreate   do not create the test environment on start" >&2
+        echo " --nodrop     do not drop the test environment on exit" >&2
+        exit 1
+fi
+
+TESTS=$@
+
+if test x"$OPT_CREATE" = xyes; then
+  echo "Starting redis on port ${REDIS_PORT}"
+  echo "port ${REDIS_PORT}" | redis-server - > ${BASEDIR}/test.log &
+  PID_REDIS=$!
+  echo ${PID_REDIS} > ${BASEDIR}/redis.pid
+
+  echo "Preparing the environment"
+  cd ${BASEDIR}/test/support; sh prepare_db.sh || die "database preparation failure"; cd -
+fi
 
 PATH=node_modules/.bin/:$PATH
 
 echo "Running tests"
-mocha -u tdd \
-  test/unit/cartodb/redis_pool.test.js \
-  test/unit/cartodb/req2params.test.js \
-  test/acceptance/cache_validator.js \
-  test/acceptance/server.js
-
+mocha -t 10000 -u tdd ${MOCHA_OPTS} ${TESTS}
+ret=$?
 
 cleanup
+
+exit $ret

test/acceptance/cache_validator.js

@@ -1,31 +1,8 @@
 var assert      = require('../support/assert');
-var net = require('net');
 require(__dirname + '/../support/test_helper');
 var CacheValidator = require(__dirname + '/../../lib/cartodb/cache_validator');
-var tests = module.exports = {};
 
-function VarnishEmu(on_cmd_recieved, test_callback) {
-    var self = this;
-    var welcome_msg = 'hi, im a varnish emu, right?';
-
-    self.commands_recieved = [];
-
-    var server = net.createServer(function (socket) {
-      var command = '';
-      socket.write("200 " + welcome_msg.length + "\n");
-      socket.write(welcome_msg);
-      socket.on('data', function(data) {        
-        self.commands_recieved.push(data);
-        on_cmd_recieved && on_cmd_recieved(self.commands_recieved);
-        socket.write('200 0\n');
-      });
-    });
-    server.listen(1337, "127.0.0.1");
-    
-    server.on('listening', function(){
-      test_callback();
-    });    
-}
+var VarnishEmu = require('../support/VarnishEmu');
 
 suite('cache_validator', function() {
 

test/acceptance/multilayer.js

@@ -0,0 +1,727 @@
+var assert      = require('../support/assert');
+var tests       = module.exports = {};
+var _           = require('underscore');
+var redis       = require('redis');
+var querystring = require('querystring');
+var semver      = require('semver');
+var mapnik      = require('mapnik');
+var Step        = require('step');
+var strftime    = require('strftime');
+var SQLAPIEmu   = require(__dirname + '/../support/SQLAPIEmu.js');
+var redis_stats_db = 5;
+
+require(__dirname + '/../support/test_helper');
+
+var windshaft_fixtures = __dirname + '/../../node_modules/windshaft/test/fixtures';
+
+var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft');
+var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
+var server = new CartodbWindshaft(serverOptions);
+server.setMaxListeners(0);
+
+suite('multilayer', function() {
+
+    var redis_client = redis.createClient(global.environment.redis.port);
+    var sqlapi_server;
+    var expected_last_updated_epoch = 1234567890123; // this is hard-coded into SQLAPIEmu
+    var expected_last_updated = new Date(expected_last_updated_epoch).toISOString();
+
+    suiteSetup(function(done){
+      sqlapi_server = new SQLAPIEmu(global.environment.sqlapi.port, done);
+    });
+
+    test("layergroup with 2 layers, each with its style", function(done) {
+
+      var layergroup =  {
+        version: '1.0.0',
+        layers: [
+           { options: {
+               sql: 'select cartodb_id, ST_Translate(the_geom_webmercator, 5e6, 0) as the_geom_webmercator from test_table limit 2',
+               cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', 
+               cartocss_version: '2.0.1',
+               interactivity: 'cartodb_id'
+             } },
+           { options: {
+               sql: 'select cartodb_id, ST_Translate(the_geom_webmercator, -5e6, 0) as the_geom_webmercator from test_table limit 2 offset 2',
+               cartocss: '#layer { marker-fill:blue; marker-allow-overlap:true; }', 
+               cartocss_version: '2.0.2',
+               interactivity: 'cartodb_id'
+             } }
+        ]
+      };
+
+      var expected_token = "e34dd7e235138a062f8ba7ad051aa3a7";
+      Step(
+        function do_post()
+        {
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup',
+              method: 'POST',
+              headers: {host: 'localhost', 'Content-Type': 'application/json' },
+              data: JSON.stringify(layergroup)
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 200, res.body);
+              var parsedBody = JSON.parse(res.body);
+              var expectedBody = { layergroupid: expected_token };
+              // check last modified
+              var qTables = JSON.stringify({
+                'q': 'SELECT CDB_QueryTables($windshaft$'
+                    + layergroup.layers[0].options.sql + ';'
+                    + layergroup.layers[1].options.sql 
+                    + '$windshaft$)'
+              });
+              assert.equal(parsedBody.last_updated, expected_last_updated);
+              if ( expected_token ) {
+                assert.equal(parsedBody.layergroupid, expected_token + ':' + expected_last_updated_epoch);
+              }
+              else expected_token = parsedBody.layergroupid;
+              next(null, res);
+          });
+        },
+        function do_get_tile(err)
+        {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup/' + expected_token + ':cb0/0/0/0.png',
+              method: 'GET',
+              headers: {host: 'localhost' },
+              encoding: 'binary'
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 200, res.body);
+              assert.equal(res.headers['content-type'], "image/png");
+
+              // Check Cache-Control
+              var cc = res.headers['cache-control'];
+              assert.equal(cc, 'public,max-age=31536000');  // 1 year
+
+              // Check X-Cache-Channel
+              cc = res.headers['x-cache-channel'];
+              assert.ok(cc); 
+              var dbname = 'cartodb_test_user_1_db'
+              assert.equal(cc.substring(0, dbname.length), dbname);
+              var jsonquery = cc.substring(dbname.length+1);
+              var sentquery = JSON.parse(jsonquery);
+              assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$'
+                + layergroup.layers[0].options.sql + ';'
+                + layergroup.layers[1].options.sql 
+                + '$windshaft$)');
+
+              assert.imageEqualsFile(res.body, 'test/fixtures/test_table_0_0_0_multilayer1.png', 2,
+                function(err, similarity) {
+                  next(err);
+              });
+          });
+        },
+        function do_get_grid_layer0(err)
+        {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup/' + expected_token
+                 + '/0/0/0/0.grid.json',
+              headers: {host: 'localhost' },
+              method: 'GET'
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 200, res.body);
+              assert.equal(res.headers['content-type'], "text/javascript; charset=utf-8; charset=utf-8");
+              assert.utfgridEqualsFile(res.body, 'test/fixtures/test_table_0_0_0_multilayer1.layer0.grid.json', 2,
+                function(err, similarity) {
+                  next(err);
+              });
+          });
+        },
+        function do_get_grid_layer1(err)
+        {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup/' + expected_token
+                 + '/1/0/0/0.grid.json',
+              headers: {host: 'localhost' },
+              method: 'GET'
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 200, res.body);
+              assert.equal(res.headers['content-type'], "text/javascript; charset=utf-8; charset=utf-8");
+              assert.utfgridEqualsFile(res.body, 'test/fixtures/test_table_0_0_0_multilayer1.layer1.grid.json', 2,
+                function(err, similarity) {
+                  next(err);
+              });
+          });
+        },
+        function finish(err) {
+          var errors = [];
+          if ( err ) {
+            errors.push(err.message);
+            console.log("Error: " + err);
+          }
+          redis_client.keys("map_style|cartodb_test_user_1_db|~" + expected_token, function(err, matches) {
+              if ( err ) errors.push(err.message);
+              assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches);
+              redis_client.del(matches, function(err) {
+                if ( err ) errors.push(err.message);
+                if ( errors.length ) done(new Error(errors));
+                else done(null);
+              });
+          });
+        }
+      );
+    });
+
+
+    test("layergroup can hold substitution tokens", function(done) {
+
+      var layergroup =  {
+        version: '1.0.0',
+        layers: [
+           { options: {
+               sql: 'select 1 as cartodb_id, '
+                  + 'ST_Buffer(!bbox!, -32*greatest(!pixel_width!,!pixel_height!)) as the_geom_webmercator',
+               cartocss: '#layer { polygon-fill:red; }', 
+               cartocss_version: '2.0.1',
+               interactivity: 'cartodb_id'
+             } }
+        ]
+      };
+
+      var expected_token  = "6d8e4ad5458e2d25cf0eef38e38717a6";
+      Step(
+        function do_post()
+        {
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup',
+              method: 'POST',
+              headers: {host: 'localhost', 'Content-Type': 'application/json' },
+              data: JSON.stringify(layergroup)
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 200, res.body);
+              var parsedBody = JSON.parse(res.body);
+              var expectedBody = { layergroupid: expected_token };
+              // check last modified
+              var qTables = JSON.stringify({
+                'q': 'SELECT CDB_QueryTables($windshaft$'
+                    + layergroup.layers[0].options.sql
+                    + '$windshaft$)'
+              });
+              assert.equal(parsedBody.last_updated, expected_last_updated);
+              if ( expected_token ) {
+                assert.equal(parsedBody.layergroupid, expected_token + ':' + expected_last_updated_epoch);
+              }
+              else expected_token = parsedBody.layergroupid;
+              next(null, res);
+          });
+        },
+        function do_get_tile1(err)
+        {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup/' + expected_token + ':cb10/1/0/0.png',
+              method: 'GET',
+              headers: {host: 'localhost' },
+              encoding: 'binary'
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 200, res.body);
+              assert.equal(res.headers['content-type'], "image/png");
+
+              // Check X-Cache-Channel
+              var cc = res.headers['x-cache-channel'];
+              assert.ok(cc); 
+              var dbname = 'cartodb_test_user_1_db'
+              assert.equal(cc.substring(0, dbname.length), dbname);
+              var jsonquery = cc.substring(dbname.length+1);
+              var sentquery = JSON.parse(jsonquery);
+              assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$'
+                + layergroup.layers[0].options.sql
+                    .replace(RegExp('!bbox!', 'g'), 'ST_MakeEnvelope(0,0,0,0)')
+                    .replace(RegExp('!pixel_width!', 'g'), '1')
+                    .replace(RegExp('!pixel_height!', 'g'), '1')
+                + '$windshaft$)');
+
+              assert.imageEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.png', 2,
+                function(err, similarity) {
+                  next(err);
+              });
+          });
+        },
+        function do_get_tile4(err)
+        {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup/' + expected_token + ':cb11/4/0/0.png',
+              method: 'GET',
+              headers: {host: 'localhost' },
+              encoding: 'binary'
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 200, res.body);
+              assert.equal(res.headers['content-type'], "image/png");
+
+              // Check X-Cache-Channel
+              var cc = res.headers['x-cache-channel'];
+              assert.ok(cc); 
+              var dbname = 'cartodb_test_user_1_db'
+              assert.equal(cc.substring(0, dbname.length), dbname);
+              var jsonquery = cc.substring(dbname.length+1);
+              var sentquery = JSON.parse(jsonquery);
+              assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$'
+                + layergroup.layers[0].options.sql
+                    .replace('!bbox!', 'ST_MakeEnvelope(0,0,0,0)')
+                    .replace('!pixel_width!', '1')
+                    .replace('!pixel_height!', '1')
+                + '$windshaft$)');
+
+              assert.imageEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.png', 2,
+                function(err, similarity) {
+                  next(err);
+              });
+          });
+        },
+        function do_get_grid1(err)
+        {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup/' + expected_token
+                 + '/0/1/0/0.grid.json',
+              headers: {host: 'localhost' },
+              method: 'GET'
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 200, res.body);
+              assert.equal(res.headers['content-type'], "text/javascript; charset=utf-8; charset=utf-8");
+              assert.utfgridEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.grid.json', 2,
+                function(err, similarity) {
+                  next(err);
+              });
+          });
+        },
+        function do_get_grid4(err)
+        {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup/' + expected_token
+                 + '/0/4/0/0.grid.json',
+              headers: {host: 'localhost' },
+              method: 'GET'
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 200, res.body);
+              assert.equal(res.headers['content-type'], "text/javascript; charset=utf-8; charset=utf-8");
+              assert.utfgridEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.grid.json', 2,
+                function(err, similarity) {
+                  next(err);
+              });
+          });
+        },
+        function finish(err) {
+          var errors = [];
+          if ( err ) {
+            errors.push(err.message);
+            console.log("Error: " + err);
+          }
+          redis_client.keys("map_style|cartodb_test_user_1_db|~" + expected_token, function(err, matches) {
+              if ( err ) errors.push(err.message);
+              assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches);
+              redis_client.del(matches, function(err) {
+                if ( err ) errors.push(err.message);
+                if ( errors.length ) done(new Error(errors));
+                else done(null);
+              });
+          });
+        }
+      );
+    });
+
+    test("layergroup creation raises mapviews counter", function(done) {
+      var layergroup =  {
+        stat_tag: 'random_tag',
+        version: '1.0.0',
+        layers: [
+           { options: {
+               sql: 'select 1 as cartodb_id, !pixel_height! as h, '
+                  + 'ST_Buffer(!bbox!, -32*greatest(!pixel_width!,!pixel_height!)) as the_geom_webmercator',
+               cartocss: '#layer { polygon-fill:red; }', 
+               cartocss_version: '2.0.1' 
+             } }
+        ]
+      };
+      var statskey = "user:localhost:mapviews";
+      var redis_stats_client = redis.createClient(global.environment.redis.port);
+      var expected_token; // will be set on first post and checked on second
+      var now = strftime("%Y%m%d", new Date());
+      var errors = [];
+      Step(
+        function clean_stats()
+        {
+          var next = this;
+          redis_stats_client.select(redis_stats_db, function(err) {
+            if ( err ) next(err);
+            else redis_stats_client.del(statskey+':global', next);
+          });
+        },
+        function do_post_1(err)
+        {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup',
+              method: 'POST',
+              headers: {host: 'localhost', 'Content-Type': 'application/json' },
+              data: JSON.stringify(layergroup)
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 200, res.body);
+              expected_token = JSON.parse(res.body).layergroupid;
+              redis_stats_client.zscore(statskey + ":global", now, next);
+          });
+        },
+        function check_global_stats_1(err, val) {
+          if ( err ) throw err;
+          assert.equal(val, 1, "Expected score of " + now + " in "
+              +  statskey + ":global to be 1, got " + val);
+          redis_stats_client.zscore(statskey+':stat_tag:random_tag', now, this);
+        },
+        function check_tag_stats_1_do_post_2(err, val) {
+          if ( err ) throw err;
+          assert.equal(val, 1, "Expected score of " + now + " in "
+              +  statskey + ":stat_tag:" + layergroup.stat_tag + " to be 1, got " + val);
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup',
+              method: 'POST',
+              headers: {host: 'localhost', 'Content-Type': 'application/json' },
+              data: JSON.stringify(layergroup)
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 200, res.body);
+              assert.equal(JSON.parse(res.body).layergroupid, expected_token);
+              redis_stats_client.zscore(statskey+':global', now, next);
+          });
+        },
+        function check_global_stats_2(err, val)
+        {
+          if ( err ) throw err;
+          assert.equal(val, 2, "Expected score of " + now + " in "
+              +  statskey + ":global to be 2, got " + val);
+          redis_stats_client.zscore(statskey+':stat_tag:' + layergroup.stat_tag, now, this);
+        },
+        function check_tag_stats_2(err, val)
+        {
+          if ( err ) throw err;
+          assert.equal(val, 2, "Expected score of " + now + " in "
+              +  statskey + ":stat_tag:" + layergroup.stat_tag + " to be 2, got " + val);
+          return 1;
+        },
+        function cleanup_map_style(err) {
+          if ( err ) errors.push('' + err);
+          var next = this;
+          // trip epoch
+          expected_token = expected_token.split(':')[0];
+          redis_client.keys("map_style|cartodb_test_user_1_db|~" + expected_token, function(err, matches) {
+              redis_client.del(matches, next);
+          });
+        },
+        function cleanup_stats(err) {
+          if ( err ) errors.push('' + err);
+          redis_client.del([statskey+':global', statskey+':stat_tag:'+layergroup.stat_tag], this);
+        },
+        function finish(err) {
+          if ( err ) errors.push('' + err);
+          if ( errors.length ) done(new Error(errors.join(',')));
+          else done(null);
+        }
+      );
+    });
+
+    test("layergroup creation fails if CartoCSS is bogus", function(done) {
+      var layergroup =  {
+        stat_tag: 'random_tag',
+        version: '1.0.0',
+        layers: [
+           { options: {
+               sql: 'select 1 as cartodb_id, !pixel_height! as h'
+                  + 'ST_Buffer(!bbox!, -32*greatest(!pixel_width!,!pixel_height!)) as the_geom_webmercator',
+               cartocss: '#layer { polygon-fit:red; }', 
+               cartocss_version: '2.0.1' 
+             } }
+        ]
+      };
+      assert.response(server, {
+          url: '/tiles/layergroup',
+          method: 'POST',
+          headers: {host: 'localhost', 'Content-Type': 'application/json' },
+          data: JSON.stringify(layergroup)
+      }, {}, function(res) {
+          assert.equal(res.statusCode, 400, res.body);
+          var parsed = JSON.parse(res.body);
+          assert.ok(parsed.errors[0].match(/^style0/));
+          assert.ok(parsed.errors[0].match(/Unrecognized rule: polygon-fit/));
+          done();
+      });
+    });
+
+    test("layergroup with 2 private-table layers", function(done) {
+
+      var layergroup =  {
+        version: '1.0.0',
+        layers: [
+           { options: {
+               sql: 'select * from test_table_private_1 where cartodb_id=1',
+               cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', 
+               cartocss_version: '2.1.0',
+               interactivity: 'cartodb_id'
+             } },
+           { options: {
+               sql: 'select * from test_table_private_1 where cartodb_id=2',
+               cartocss: '#layer { marker-fill:blue; marker-allow-overlap:true; }', 
+               cartocss_version: '2.1.0',
+               interactivity: 'cartodb_id'
+             } }
+        ]
+      };
+
+      var expected_token = "b4ed64d93a411a59f330ab3d798e4009";
+      Step(
+        function do_post()
+        {
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup?map_key=1234',
+              method: 'POST',
+              headers: {host: 'localhost', 'Content-Type': 'application/json' },
+              data: JSON.stringify(layergroup)
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 200, res.body);
+              var parsedBody = JSON.parse(res.body);
+              var expectedBody = { layergroupid: expected_token };
+              // check last modified
+              var qTables = JSON.stringify({
+                'q': 'SELECT CDB_QueryTables($windshaft$'
+                    + layergroup.layers[0].options.sql + ';'
+                    + layergroup.layers[1].options.sql 
+                    + '$windshaft$)'
+              });
+              assert.equal(parsedBody.last_updated, expected_last_updated);
+              if ( expected_token ) {
+                assert.equal(parsedBody.layergroupid, expected_token + ':' + expected_last_updated_epoch);
+              }
+              else expected_token = parsedBody.layergroupid;
+              next(null, res);
+          });
+        },
+        function do_get_tile(err)
+        {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup/' + expected_token + ':cb0/0/0/0.png?map_key=1234',
+              method: 'GET',
+              headers: {host: 'localhost' },
+              encoding: 'binary'
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 200, res.body);
+              assert.equal(res.headers['content-type'], "image/png");
+
+              // Check X-Cache-Channel
+              var cc = res.headers['x-cache-channel'];
+              assert.ok(cc); 
+              var dbname = 'cartodb_test_user_1_db'
+              assert.equal(cc.substring(0, dbname.length), dbname);
+              next(err);
+          });
+        },
+        function do_get_grid_layer0(err)
+        {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup/' + expected_token
+                 + '/0/0/0/0.grid.json?map_key=1234',
+              headers: {host: 'localhost' },
+              method: 'GET'
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 200, res.body);
+              next(err);
+          });
+        },
+        function do_get_grid_layer1(err)
+        {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup/' + expected_token
+                 + '/1/0/0/0.grid.json?map_key=1234',
+              headers: {host: 'localhost' },
+              method: 'GET'
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 200, res.body);
+              assert.equal(res.headers['content-type'], "text/javascript; charset=utf-8; charset=utf-8");
+              next(err);
+          });
+        },
+        function do_get_tile_unauth(err)
+        {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup/' + expected_token + ':cb0/0/0/0.png',
+              method: 'GET',
+              headers: {host: 'localhost' },
+              encoding: 'binary'
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 401);
+              var re = RegExp('permission denied');
+              assert.ok(res.body.match(re), 'No "permission denied" error: ' + res.body);
+              next(err);
+          });
+        },
+        function do_get_grid_layer0_unauth(err)
+        {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup/' + expected_token
+                 + '/0/0/0/0.grid.json',
+              headers: {host: 'localhost' },
+              method: 'GET'
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 401);
+              var re = RegExp('permission denied');
+              assert.ok(res.body.match(re), 'No "permission denied" error: ' + res.body);
+              next(err);
+          });
+        },
+        function do_get_grid_layer1_unauth(err)
+        {
+          if ( err ) throw err;
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup/' + expected_token
+                 + '/1/0/0/0.grid.json',
+              headers: {host: 'localhost' },
+              method: 'GET'
+          }, {}, function(res) {
+              assert.equal(res.statusCode, 401);
+              var re = RegExp('permission denied');
+              assert.ok(res.body.match(re), 'No "permission denied" error: ' + res.body);
+              next(err);
+          });
+        },
+        function finish(err) {
+          var errors = [];
+          if ( err ) {
+            errors.push(err.message);
+            console.log("Error: " + err);
+          }
+          redis_client.keys("map_style|cartodb_test_user_1_db|~" + expected_token, function(err, matches) {
+              if ( err ) errors.push(err.message);
+              assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches);
+              redis_client.del(matches, function(err) {
+                if ( err ) errors.push(err.message);
+                if ( errors.length ) done(new Error(errors));
+                else done(null);
+              });
+          });
+        }
+      );
+    });
+
+    // https://github.com/cartodb/Windshaft-cartodb/issues/81
+    test("invalid text-name in CartoCSS", function(done) {
+
+      var layergroup =  {
+        version: '1.0.1',
+        layers: [
+           { options: {
+               sql: "select 1 as cartodb_id, 'SRID=3857;POINT(0 0)'::geometry as the_geom_webmercator",
+               cartocss: '#sample { text-name: cartodb_id; text-face-name: "Dejagnu"; }',
+               cartocss_version: '2.1.0',
+             } }
+        ]
+      };
+
+      assert.response(server, {
+          url: '/tiles/layergroup?',
+          method: 'POST',
+          headers: {host: 'localhost', 'Content-Type': 'application/json' },
+          data: JSON.stringify(layergroup)
+      }, {}, function(res) {
+          assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body);
+          var parsed = JSON.parse(res.body);
+          assert.deepEqual(parsed, {"errors":["style0:1:10 Invalid value for text-name, the type expression is expected. cartodb_id (of type keyword)  was given."]});
+          done();
+      });
+    });
+
+    test("quotes CartoCSS", function(done) {
+
+      var layergroup =  {
+        version: '1.0.1',
+        layers: [
+           { options: {
+               sql: "select 'single''quote' as n, 'SRID=3857;POINT(0 0)'::geometry as the_geom_webmercator",
+               cartocss: '#s [n="single\'quote" ] { marker-fill:red; }',
+               cartocss_version: '2.1.0',
+             } },
+           { options: {
+               sql: "select 'double\"quote' as n, 'SRID=3857;POINT(2 0)'::geometry as the_geom_webmercator",
+               cartocss: '#s [n="double\\"quote" ] { marker-fill:red; }',
+               cartocss_version: '2.1.0',
+             } }
+        ]
+      };
+
+      assert.response(server, {
+          url: '/tiles/layergroup?',
+          method: 'POST',
+          headers: {host: 'localhost', 'Content-Type': 'application/json' },
+          data: JSON.stringify(layergroup)
+      }, {}, function(res) {
+          assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+          done();
+      });
+    });
+
+    // See https://github.com/CartoDB/Windshaft-cartodb/issues/87
+    test("exponential notation in CartoCSS filter values", function(done) {
+      var layergroup =  {
+        version: '1.0.1',
+        layers: [
+           { options: {
+               sql: "select .4 as n, 'SRID=3857;POINT(0 0)'::geometry as the_geom_webmercator",
+               cartocss: '#s [n<=.2e-2] { marker-fill:red; }',
+               cartocss_version: '2.1.0',
+             } }
+        ]
+      };
+      assert.response(server, {
+          url: '/tiles/layergroup?',
+          method: 'POST',
+          headers: {host: 'localhost', 'Content-Type': 'application/json' },
+          data: JSON.stringify(layergroup)
+      }, {}, function(res) {
+          assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+          done();
+      });
+    });
+
+    suiteTeardown(function(done) {
+
+        // This test will add map_style records, like
+        // 'map_style|null|publicuser|my_table',
+        redis_client.keys("map_style|*", function(err, matches) {
+            redis_client.del(matches, function(err) {
+              redis_client.select(5, function(err, matches) {
+                redis_client.keys("user:localhost:mapviews*", function(err, matches) {
+                  redis_client.del(matches, function(err) {
+                    sqlapi_server.close(done);
+                  });
+                });
+              });
+            });
+        });
+
+    });
+    
+});
+

test/fixtures/test_multilayer_bbox.grid.json

@@ -0,0 +1 @@
+{"grid":["                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"],"keys":["","1"],"data":{"1":{"cartodb_id":1}}}

test/fixtures/test_table_0_0_0_multilayer1.layer0.grid.json

@@ -0,0 +1 @@
+{"grid":["                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                      !!!                       ","                                     !!!!!!                     ","                                    !!!!!!!                     ","                                    !!!!!!!!                    ","                                    !!!!!!!!                    ","                                    !!!!!!!!                    ","                                    !!!!!!!                     ","                                     !!!!!!                     ","                                      !!!                       ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                "],"keys":["","2"],"data":{"2":{"cartodb_id":2}}}

test/fixtures/test_table_0_0_0_multilayer1.layer1.grid.json

@@ -0,0 +1 @@
+{"grid":["                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                       !!                                       ","                      !!!                                       ","                       !!                                       ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                "],"keys":["","2"],"data":{"2":{"cartodb_id":4}}}

test/support/SQLAPIEmu.js

@@ -0,0 +1,38 @@
+var http        = require('http');
+var url         = require('url');
+
+var o = function(port, cb) {
+
+  this.queries = [];
+  var that = this;
+  this.sqlapi_server = http.createServer(function(req,res) {
+        var query = url.parse(req.url, true).query;
+        that.queries.push(query);
+        if ( query.q.match('SQLAPIERROR') ) {
+          res.statusCode = 400;
+          res.write(JSON.stringify({'error':'Some error occurred'}));
+        } else if ( query.q.match('EPOCH.* as max') ) {
+          // This is the structure of the known query sent by tiler
+          var row = {
+            'max': 1234567890.123
+          };
+          res.write(JSON.stringify({rows: [ row ]}));
+        } else {
+          var qs = JSON.stringify(query);
+          var row = {
+            // This is the structure of the known query sent by tiler
+            'cdb_querytables': '{' + qs + '}',
+            'max': qs
+          };
+          res.write(JSON.stringify({rows: [ row ]}));
+        }
+        res.end();
+   }).listen(port, cb);
+};
+
+o.prototype.close = function(cb) {
+  this.sqlapi_server.close(cb);
+};
+
+module.exports = o;
+

test/support/VarnishEmu.js

@@ -0,0 +1,25 @@
+var net = require('net');
+
+module.exports = function(on_cmd_recieved, test_callback) {
+    var self = this;
+    var welcome_msg = 'hi, im a varnish emu, right?';
+
+    self.commands_recieved = [];
+
+    var server = net.createServer(function (socket) {
+      var command = '';
+      socket.write("200 " + welcome_msg.length + "\n");
+      socket.write(welcome_msg);
+      socket.on('data', function(data) {
+        self.commands_recieved.push(data);
+        on_cmd_recieved && on_cmd_recieved(self.commands_recieved);
+        socket.write('200 0\n');
+      });
+    });
+    server.listen(1337, "127.0.0.1");
+
+    server.on('listening', function(){
+      test_callback();
+    });
+};
+

test/support/assert.js

@@ -8,29 +8,93 @@ var exec = require('child_process').exec;
 
 var assert = module.exports = exports = require('assert');
 
-assert.imageEqualsFile = function(buffer, file_b, callback) {
+// @param tolerance number of tolerated grid cell differences
+assert.utfgridEqualsFile = function(buffer, file_b, tolerance, callback) {
+    fs.writeFileSync('/tmp/grid.json', buffer, 'binary'); // <-- to debug/update
+    var expected_json = JSON.parse(fs.readFileSync(file_b, 'utf8'));
+
+    var err = null;
+
+    var Celldiff = function(x, y, ev, ov) {
+      this.x = x;
+      this.y = y;
+      this.ev = ev;
+      this.ov = ov;
+    };
+
+    Celldiff.prototype.toString = function() {
+      return '(' + this.x + ',' + this.y + ')["' + this.ev + '" != "' + this.ov + '"]';
+    };
+
+    try {
+      var obtained_json = JSON.parse(buffer);
+
+      // compare grid
+      var obtained_grid = obtained_json.grid;
+      var expected_grid = expected_json.grid;
+      var nrows = obtained_grid.length
+      if (nrows != expected_grid.length) {
+        throw new Error( "Obtained grid rows (" + nrows +
+                    ") != expected grid rows (" + expected_grid.length + ")" );
+      }
+      var celldiff = [];
+      for (var i=0; i<nrows; ++i) {
+        var ocols = obtained_grid[i];
+        var ecols = expected_grid[i];
+        var ncols = ocols.length;
+        if ( ncols != ecols.length ) {
+          throw new Error( "Obtained grid cols (" + ncols +
+                   ") != expected grid cols (" + ecols.length +
+                   ") on row " + i ); 
+        }
+        for (var j=0; j<ncols; ++j) {
+          var ocell = ocols[j];
+          var ecell = ecols[j];
+          if ( ocell !== ecell ) {
+            celldiff.push(new Celldiff(i, j, ecell, ocell));
+          }
+        }
+      }
+
+      if ( celldiff.length > tolerance ) {
+        throw new Error( celldiff.length + " cell differences: " + celldiff );
+      }
+
+      assert.deepEqual(obtained_json.keys, expected_json.keys);
+    } catch (e) { err = e; }
+
+    callback(err);
+};
+
+//
+// @param tol tolerated color distance as a percent over max channel value
+//            by default this is zero. For meaningful values, see
+//            http://www.imagemagick.org/script/command-line-options.php#metric
+//
+assert.imageEqualsFile = function(buffer, file_b, tol, callback) {
     if (!callback) callback = function(err) { if (err) throw err; };
     file_b = path.resolve(file_b);
-    var file_a = '/tmp/' + (Math.random() * 1e16);
+    var file_a = '/tmp/windshaft-test-image-test.png'; // + (Math.random() * 1e16); // TODO: make predictable 
     var err = fs.writeFileSync(file_a, buffer, 'binary');
     if (err) throw err;
 
-    exec('compare -metric PSNR "' + file_a + '" "' +
+    var fuzz = tol + '%';
+    exec('compare -fuzz ' + fuzz + ' -metric AE "' + file_a + '" "' +
             file_b + '" /dev/null', function(err, stdout, stderr) {
         if (err) {
             fs.unlinkSync(file_a);
             callback(err);
         } else {
             stderr = stderr.trim();
-            if (stderr === 'inf') {
-                fs.unlinkSync(file_a);
-                callback(null);
+            var similarity = parseFloat(stderr);
+            if ( similarity > 0 ) {
+              var err = new Error('Images not equal(' + similarity + '): ' +
+                      file_a  + '    ' + file_b);
+              err.similarity = similarity;
+              callback(err);
             } else {
-                var similarity = parseFloat(stderr);
-                var err = new Error('Images not equal(' + similarity + '): ' +
-                        file_a  + '    ' + file_b);
-                err.similarity = similarity;
-                callback(err);
+              fs.unlinkSync(file_a);
+              callback(null);
             }
         }
     });

test/support/config.js

@@ -1,5 +1,8 @@
 var _ = require('underscore');
 
+
+require(__dirname + '/test_helper');
+
 module.exports = function(opts) {
 
     var config = {
@@ -7,7 +10,7 @@ module.exports = function(opts) {
             max: 10, 
             idleTimeoutMillis: 1, 
             reapIntervalMillis: 1,
-            port: 6333 // TODO: read from test env ?
+            port: global.environment.redis.port
         }
     }
 

test/support/prepare_db.sh

@@ -17,7 +17,7 @@ die() {
 }
 
 TEST_DB="cartodb_test_user_1_db"
-REDIS_PORT=6333
+if test -z "$REDIS_PORT"; then REDIS_PORT=6333; fi
 
 echo "preparing postgres..."
 dropdb "${TEST_DB}"
@@ -26,11 +26,12 @@ psql "${TEST_DB}" < ./sql/windshaft.test.sql
 psql "${TEST_DB}" < ./sql/gadm4.sql
 
 echo "preparing redis..."
-echo "HSET rails:users:vizzuality id 1" | redis-cli -p ${REDIS_PORT} -n 5
-echo 'HSET rails:users:vizzuality database_name "'"${TEST_DB}"'"' | redis-cli -p ${REDIS_PORT} -n 5
-echo "HSET rails:users:vizzuality map_key 1234" | redis-cli -p ${REDIS_PORT} -n 5
-echo "SADD rails:users:vizzuality:map_key 1235" | redis-cli -p ${REDIS_PORT} -n 5
+echo "HSET rails:users:localhost id 1" | redis-cli -p ${REDIS_PORT} -n 5
+echo 'HSET rails:users:localhost database_name "'"${TEST_DB}"'"' | redis-cli -p ${REDIS_PORT} -n 5
+echo "HSET rails:users:localhost map_key 1234" | redis-cli -p ${REDIS_PORT} -n 5
+echo "SADD rails:users:localhost:map_key 1235" | redis-cli -p ${REDIS_PORT} -n 5
 echo 'HSET rails:'"${TEST_DB}"':my_table infowindow "this, that, the other"' | redis-cli -p ${REDIS_PORT} -n 0
 echo 'HSET rails:'"${TEST_DB}"':test_table_private_1 privacy "0"' | redis-cli -p ${REDIS_PORT} -n 0
 
-echo "Finished preparing data. Run tests with expresso."
+echo "Finished preparing data. Ready to run tests"
+

test/support/sql/windshaft.test.sql

@@ -51,11 +51,12 @@ SELECT pg_catalog.setval('test_table_cartodb_id_seq', 60, true);
 
 ALTER TABLE test_table ALTER COLUMN cartodb_id SET DEFAULT nextval('test_table_cartodb_id_seq'::regclass);
 
-INSERT INTO test_table VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241');
-INSERT INTO test_table VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241');
-INSERT INTO test_table VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.324', 3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241');
-INSERT INTO test_table VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.329509', 4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241');
-INSERT INTO test_table VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.334931', 5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241');
+INSERT INTO test_table VALUES
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.324', 3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.329509', 4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.334931', 5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241');
 
 ALTER TABLE ONLY test_table ADD CONSTRAINT test_table_pkey PRIMARY KEY (cartodb_id);
 
@@ -95,11 +96,12 @@ SELECT pg_catalog.setval('test_table_2_cartodb_id_seq', 60, true);
 
 ALTER TABLE test_table_2 ALTER COLUMN cartodb_id SET DEFAULT nextval('test_table_2_cartodb_id_seq'::regclass);
 
-INSERT INTO test_table_2 VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241');
-INSERT INTO test_table_2 VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241');
-INSERT INTO test_table_2 VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.324', 3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241');
-INSERT INTO test_table_2 VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.329509', 4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241');
-INSERT INTO test_table_2 VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.334931', 5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241');
+INSERT INTO test_table_2 VALUES
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.324', 3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.329509', 4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.334931', 5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241');
 
 ALTER TABLE ONLY test_table_2 ADD CONSTRAINT test_table_2_pkey PRIMARY KEY (cartodb_id);
 
@@ -139,11 +141,12 @@ SELECT pg_catalog.setval('test_table_3_cartodb_id_seq', 60, true);
 
 ALTER TABLE test_table_3 ALTER COLUMN cartodb_id SET DEFAULT nextval('test_table_3_cartodb_id_seq'::regclass);
 
-INSERT INTO test_table_3 VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241');
-INSERT INTO test_table_3 VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241');
-INSERT INTO test_table_3 VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.324', 3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241');
-INSERT INTO test_table_3 VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.329509', 4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241');
-INSERT INTO test_table_3 VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.334931', 5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241');
+INSERT INTO test_table_3 VALUES
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.324', 3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.329509', 4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.334931', 5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241');
 
 ALTER TABLE ONLY test_table_3 ADD CONSTRAINT test_table_3_pkey PRIMARY KEY (cartodb_id);
 

test/support/test_helper.js

@@ -6,6 +6,7 @@
  */
 
 var _ = require('underscore');
+var LZMA  = require('lzma/lzma_worker.js').LZMA;
 
 // set environment specific variables
 global.settings     = require(__dirname + '/../../config/settings');
@@ -13,4 +14,21 @@ global.environment  = require(__dirname + '/../../config/environments/test');
 _.extend(global.settings, global.environment);
 
 
+// Utility function to compress & encode LZMA
+function lzma_compress_to_base64(payload, mode, callback) {
+  LZMA.compress(payload, mode, 
+    function(ints) {
+      ints = ints.map(function(c) { return String.fromCharCode(c + 128) }).join('')
+      var base64 = new Buffer(ints, 'binary').toString('base64');
+      callback(null, base64);
+    },
+    function(percent) {
+      //console.log("Compressing: " + percent + "%");
+    }
+  );
+}
+
+module.exports = {
+  lzma_compress_to_base64: lzma_compress_to_base64
+}
 

test/unit/cartodb/redis_pool.test.js

@@ -38,6 +38,7 @@ suite('redis_pool', function() {
     
     test('calling aquire returns a redis client object that can get/set', function(done){
       redis_pool.acquire(0, function(err, client){
+        if ( err ) { done(err); return; }
         client.set("key","value");
         client.get("key", function(err,data){      
           assert.equal(data, "value");      
@@ -49,6 +50,7 @@ suite('redis_pool', function() {
     
     test('calling aquire on another DB returns a redis client object that can get/set', function(done){
       redis_pool.acquire(2, function(err, client){
+        if ( err ) { done(err); return; }
         client.set("key","value");
         client.get("key", function(err,data){      
           assert.equal(data, "value");      

test/unit/cartodb/req2params.test.js

@@ -14,27 +14,27 @@ suite('req2params', function() {
     });
 
     test('cleans up request', function(done){
-      opts.req2params({headers: { host:'h1' }, query: {dbuser:'hacker',dbname:'secret'}}, function(err, req) {
+      opts.req2params({headers: { host:'localhost' }, query: {dbuser:'hacker',dbname:'secret'}}, function(err, req) {
           if ( err ) { console.log(err); throw new Error(err); }
           assert.ok(_.isObject(req.query), 'request has query');
           assert.ok(!req.query.hasOwnProperty('dbuser'), 'dbuser was removed from query');
           assert.ok(req.hasOwnProperty('params'), 'request has params');
           assert.ok(req.params.hasOwnProperty('interactivity'), 'request params have interactivity');
-          assert.ok(_.isNull(req.params.dbname), 'could forge dbname');
-          assert.ok(!req.hasOwnProperty('dbuser'), 'could inject dbuser ('+req.params.dbuser+')');
+          assert.equal(req.params.dbname, 'cartodb_test_user_1_db', 'could forge dbname: '+ req.params.dbname);
+          assert.ok(!req.params.hasOwnProperty('dbuser'), 'could inject dbuser ('+req.params.dbuser+')');
           done();
       });
     });
 
     test('sets dbname from redis metadata', function(done){
-      opts.req2params({headers: { host:'vizzuality' }, query: {} }, function(err, req) {
+      opts.req2params({headers: { host:'localhost' }, query: {} }, function(err, req) {
           if ( err ) { console.log(err); throw new Error(err); }
           //console.dir(req);
           assert.ok(_.isObject(req.query), 'request has query');
           assert.ok(!req.query.hasOwnProperty('dbuser'), 'dbuser was removed from query');
           assert.ok(req.hasOwnProperty('params'), 'request has params');
           assert.ok(req.params.hasOwnProperty('interactivity'), 'request params have interactivity');
-          // database_name for user "vizzuality" (see test/support/prepare_db.sh)
+          // database_name for user "localhost" (see test/support/prepare_db.sh)
           assert.equal(req.params.dbname, 'cartodb_test_user_1_db');
           // unauthenticated request gets no dbuser
           assert.ok(!req.params.hasOwnProperty('dbuser'), 'could inject dbuser ('+req.params.dbuser+')');
@@ -43,24 +43,43 @@ suite('req2params', function() {
     });
 
     test('sets also dbuser for authenticated requests', function(done){
-      opts.req2params({headers: { host:'vizzuality' }, query: {map_key: '1234'} }, function(err, req) {
+      opts.req2params({headers: { host:'localhost' }, query: {map_key: '1234'} }, function(err, req) {
           if ( err ) { console.log(err); throw new Error(err); }
           //console.dir(req);
           assert.ok(_.isObject(req.query), 'request has query');
           assert.ok(!req.query.hasOwnProperty('dbuser'), 'dbuser was removed from query');
           assert.ok(req.hasOwnProperty('params'), 'request has params');
           assert.ok(req.params.hasOwnProperty('interactivity'), 'request params have interactivity');
-          // database_name for user "vizzuality" (see test/support/prepare_db.sh)
+          // database_name for user "localhost" (see test/support/prepare_db.sh)
           assert.equal(req.params.dbname, 'cartodb_test_user_1_db');
-          // id for user "vizzuality" (see test/support/prepare_db.sh)
-          assert.equal(req.dbuser, 'test_cartodb_user_1');
+          // id for user "localhost" (see test/support/prepare_db.sh)
+          assert.equal(req.params.dbuser, 'test_cartodb_user_1');
  
-          opts.req2params({headers: { host:'vizzuality' }, query: {map_key: '1235'} }, function(err, req) {
+          opts.req2params({headers: { host:'localhost' }, query: {map_key: '1235'} }, function(err, req) {
               // wrong key resets params to no user
-              assert.ok(!req.hasOwnProperty('dbuser'), 'could inject dbuser ('+req.params.dbuser+')');
+              assert.ok(!req.params.hasOwnProperty('dbuser'), 'could inject dbuser ('+req.params.dbuser+')');
               done();
           });
       });
     });
+
+    test('it should extend params with decoded lzma', function(done) {
+      var qo = {
+        style: 'test',
+        style_version: '2.1.0',
+        cache_buster: 5
+      };
+      test_helper.lzma_compress_to_base64(JSON.stringify(qo), 1, function(err, data) {
+        opts.req2params({ query: { non_included: 'toberemoved', api_key: 'test', style: 'override', lzma: data }}, function(err, req) {
+          var query = req.params
+          assert.equal(qo.style, query.style)
+          assert.equal(qo.style_version, query.style_version)
+          assert.equal(qo.cache_buster, query.cache_buster)
+          assert.equal('test', query.api_key)
+          assert.equal(undefined, query.non_included)
+          done();
+        });
+      });
+    });
     
 });

tools/convert_database_styles

@@ -0,0 +1,81 @@
+#!/usr/bin/env node
+
+var path = require('path');
+
+var grainstore = require('../node_modules/windshaft/node_modules/grainstore');
+var mapnik = require('mapnik');
+var redis = require('redis');
+
+function usage(me, exitcode) {
+  console.log("Usage: " + me + " <database_name> <table_name> [<target_mapnik_version>]");
+  process.exit(exitcode);
+}
+
+var node_path = process.argv.shift(); 
+var script_path = process.argv.shift(); 
+var me = path.basename(script_path);
+
+var database_name = process.argv.shift()
+var table_name = process.argv.shift()
+var MAPNIK_VERSION = process.argv.shift()
+
+
+if ( ! MAPNIK_VERSION ) {
+  MAPNIK_VERSION = mapnik.versions.mapnik;
+}
+
+if ( ! database_name || ! table_name) {
+  usage(me, 1);
+}
+
+var REDIS_PORT = 6379; // TODO: make a command line parameter
+
+var dbnum = 0;
+
+var mml_store = new grainstore.MMLStore({port:REDIS_PORT}, {mapnik_version:MAPNIK_VERSION});
+
+var failures = [];
+
+var client = redis.createClient(REDIS_PORT, 'localhost');
+client.on('connect', function() {
+  client.select(dbnum);
+  client.keys('map_style|' + database_name + '|' + table_name, function(err, matches) {
+
+    processNext = function() {
+      if ( ! matches.length ) process.exit(failures.length);
+      var k = matches.shift();
+
+      if ( /map_style\|.*\|.*\|/.test(k) ) {
+        //console.warn("Key " + k + " is EXTENDED, skipping");
+        processNext();
+      } 
+
+      var out = 'map_style|' + database_name + '|' + table_name + ': ';
+
+      var mml_builder = mml_store.mml_builder({
+        dbname:database_name, 
+        table:table_name},
+        function(err, payload) {
+          if ( err ) { 
+            console.warn(out + err.message); 
+            failures.push(k); processNext(); 
+          }
+          else {
+            mml_builder.resetStyle(function(err, data) {
+              if ( err ) { 
+                console.warn(out + err.message); 
+                failures.push(k); 
+              }
+              else console.log(out + 'OK');
+              processNext();
+            }, true);
+          }
+      });
+
+    };
+
+    processNext();
+
+  });
+
+});

tools/multilayer_token

@@ -0,0 +1,40 @@
+#!/bin/sh
+
+verbose=no
+tiler_url=http://dev.localhost.lan:8181/tiles/layergroup
+
+# This is for direct windshaft connection
+#tiler_url=http://dev.localhost.lan:8083/database/cartodb_dev_user_1_db/layergroup
+
+while test -n "$1"; do
+  if test "$1" = "-v"; then
+    verbose=yes
+  elif test -z "$cfg"; then
+    cfg="$1"
+  else
+    tiler_url="$1"
+  fi
+  shift
+done
+
+if test -z "$cfg"; then
+  echo "Usage: $0 [-v] <config_file> [<tiler_url>]" >&2
+  echo "Default <tiler_url> is ${tiler_url}" >&2
+  exit 1
+fi
+
+cmd="curl -skH Content-Type:application/json --data-binary @- ${tiler_url}"
+if test x${verbose} = xyes; then
+  cmd="${cmd} -v"
+fi
+res=`cat ${cfg} | tr '\n' ' ' | ${cmd}`
+if test $? -gt 0; then
+  echo "curl command failed: ${cmd}"
+fi
+
+if test x${verbose} = xyes; then
+  echo "${res}"
+fi
+
+tok=`echo "$res" | sed 's/.*"layergroupid":"\([^"]*\)".*/\1/'`
+echo $tok

tools/munin/Makefile

@@ -0,0 +1,19 @@
+MUNIN_PLUGINS_DIR=/etc/munin/plugins
+MUNIN_PLUGINS_CONFIG_DIR=/etc/munin/plugin-conf.d
+PWD=$(shell pwd)
+
+all: windshaft.conf
+
+windshaft.conf: windshaft.conf.in
+	sed 's#@PWD@#$(PWD)#' < $< > $@
+
+install-munin-plugin-conf: windshaft.conf 
+	install -m 644 $< $(MUNIN_PLUGINS_CONFIG_DIR)/windshaft.conf
+
+install-munin-plugin: windshaft
+	install -m 755 $< $(MUNIN_PLUGINS_DIR)/windshaft
+
+install: install-munin-plugin install-munin-plugin-conf
+
+clean:
+	rm -f windshaft.conf

tools/munin/windshaft

@@ -0,0 +1,87 @@
+#!/bin/sh
+
+envnik=$(basename "${TILER_ENVIRONMENT}" .js)
+
+if test "$1" = "config"; then
+echo "graph_title fd usage (${envnik})"
+cat <<"EOM"
+graph_vlabel number of file descriptors
+graph_category windshaft
+graph_scale no
+procs.label Number of tiler processes
+pgsql.label PostgreSQL connections (max)
+redis.label Redis connections (max)
+http.label Incoming http requests (max)
+caches.label Number of renderer caches (max)
+nfd.label Total file descriptors (max)
+EOM
+exit 0
+fi
+
+if test x"$1" != x; then
+  TILER_ENVIRONMENT=$(cd $(dirname $0); pwd)/../../config/environments/${1}.js
+fi
+
+if test -z "$TILER_ENVIRONMENT"; then
+  echo "Usage: $0 [<environment>]" >&2
+  echo "   or: [TILER_ENVIRONMENT=<environment>] $0" >&2
+  exit 1
+fi
+
+http_port=$(echo "console.log(require('${TILER_ENVIRONMENT}').port)" | node) || exit 1
+pgsql_port=$(echo "console.log(require('${TILER_ENVIRONMENT}').postgres.port)" | node) || exit 1
+redis_port=$(echo "console.log(require('${TILER_ENVIRONMENT}').redis.port)" | node) || exit 1
+
+pids=$(lsof -i :${http_port} | grep LISTEN | awk '{print $2}')
+nworkers=$(echo "${pids}" | wc -l)
+pids=$(echo "${pids}" | paste -sd ' ')
+
+if test -z "${pids}"; then
+  echo "No processes found listening on tcp port '${http_port}'" >&2
+  exit 1
+fi
+
+tmpreport="/tmp/checkfd.$$.txt"
+
+lsof -Pp $(echo "${pids}" | tr ' ' ',') > "${tmpreport}"
+
+maxdb=0
+maxredis=0
+maxhttp=0
+maxtot=0
+maxcache=0
+
+for pid in ${pids}; do
+
+  cnt=$(grep "${pid}" "${tmpreport}" | grep ":${pgsql_port} " | wc -l);
+  if test $cnt -gt $maxdb; then maxdb=$cnt; fi
+  
+  cnt=$(grep "${pid}" "${tmpreport}" | grep ":${redis_port} " | wc -l);
+  if test $cnt -gt $maxredis; then maxredis=$cnt; fi
+
+  cnt=$(grep "${pid}" "${tmpreport}" | grep ":${http_port}-" | grep -v "LISTEN" | wc -l);
+  if test $cnt -gt $maxhttp; then maxhttp=$cnt; fi
+
+  cnt=$(grep "${pid}" "${tmpreport}" | wc -l);
+  if test $cnt -gt $maxtot; then maxtot=$cnt; fi
+
+  log=$(grep "${pid}" "${tmpreport}" | grep -w 1w | awk '{print $9}')
+  if test -e "${log}"; then
+    kill -USR2 "${pid}"
+    cnt=$(tac ${log} | sed -n -e '/ItemKey/p;/^RenderCache/q' | wc -l)
+    if test $cnt -gt $maxcache; then maxcache=$cnt; fi
+  else
+    # report the error...
+    maxcache=-1
+  fi
+
+done
+
+echo "procs.value ${nworkers}"
+echo "pgsql.value ${maxdb}"
+echo "redis.value ${maxredis}"
+echo "http.value ${maxhttp}"
+echo "caches.value ${maxcache}"
+echo "nfd.value ${maxtot}"
+
+rm -f "${tmpreport}"

tools/munin/windshaft.conf.in

@@ -0,0 +1,5 @@
+# Configuration file for munin plugin
+
+[windshaft]
+user root
+env.TILER_ENVIRONMENT @PWD@/../../config/environments/production.js

tools/reset_styles

@@ -1,35 +1,105 @@
 #!/usr/bin/env node
 
-// Reset redis-stored XML styles so that they are regenerated
-// from CartoCSS on next tile request
+/*
 
-var redis = require('redis')
+This scripts drops all extended map_style keys in redis and regenerates
+the XML caches in all the base ones to target the configured mapnik_version.
 
-var REDIS_PORT = 6379; // TODO: make a parameter
+Optionally (with --convert) it also re-writes the CartoCSS if needed
+to target the configured mapnik_version.
+
+It is recommended to make a backup of the redis database before using
+this script.
+
+*/
+
+var path = require('path');
+
+// Reset all styles in the store
+var grainstore = require('../node_modules/windshaft/node_modules/grainstore/lib/grainstore');
+var mapnik = require('mapnik');
+var redis = require('redis');
+
+function usage(me, exitcode) {
+  console.log("Usage: " + me + " [--convert] <environment>");
+  process.exit(exitcode);
+}
+
+var doConvert = false;
+
+var node_path = process.argv.shift(); 
+var script_path = process.argv.shift(); 
+var me = path.basename(script_path);
+var ENV;
+var arg;
+while ( arg = process.argv.shift() ) {
+  if ( arg == '--convert' ) {
+    doConvert = true;
+  }
+  else if ( ! ENV ) {
+    ENV = arg;
+  }
+  else {
+    usage(me, 1);
+  }
+}
+
+if ( ! ENV ) usage(me, 1);
+
+global.environment = require('../config/environments/' + ENV);
+var serverOptions = require('../lib/cartodb/server_options'); // _after_ setting global.environment
+
+var MAPNIK_VERSION = global.environment.mapnik_version || mapnik.versions.mapnik;
+
+console.log( (doConvert ? "Converting" : "Resetting" ) + ' all styles to target ' + MAPNIK_VERSION);
 
 var dbnum = 0;
 
-var client = redis.createClient(REDIS_PORT, 'localhost');
+var mml_store = new grainstore.MMLStore(serverOptions.redis, serverOptions.grainstore);
+
+var failures = [];
+
+var client = redis.createClient(serverOptions.redis.port, serverOptions.redis.host);
 client.on('connect', function() {
   client.select(dbnum);
   client.keys('map_style|*', function(err, matches) {
 
     processNext = function() {
-      if ( ! matches.length ) process.exit(0);
+      if ( ! matches.length ) process.exit(failures.length);
       var k = matches.shift();
-      console.log("Resetting XML in key: " + k);
-      client.get(k, function(err, val) {
-        if ( err ) throw err;
-        val = JSON.parse(val);
-        delete val.xml;
-        client.set(k, JSON.stringify(val), function() {
-          console.log("done with style " + k);
+
+      if ( /map_style\|.*\|.*\|/.test(k) ) {
+        // See https://github.com/Vizzuality/Windshaft-cartodb/issues/58
+        //console.warn("Key " + k + " is EXTENDED, dropping");
+        client.del(k, function(err) {
+          if ( err ) console.warn("Error dropping key " + k);
           processNext();
         });
+        return;
+      } 
+
+      var params = RegExp(/map_style\|(.*)\|(.*)/).exec(k);
+      var db = params[1];
+      var tab = params[2];
+      var out = 'map_style|' + db + '|' + tab + ': ';
+
+      var mml_builder = mml_store.mml_builder({dbname:db, table:tab},
+                                              function(err, payload) {
+
+          if ( err ) { console.warn(out + err.message); failures.push(k); processNext(); }
+          else {
+            mml_builder.resetStyle(function(err, data) {
+              if ( err ) { console.warn(out + err.message); failures.push(k); }
+              else console.log(out + 'OK' + ( doConvert ? ' (converted)' : '' ));
+              processNext();
+            }, doConvert);
+          }
       });
-    }
+
+    };
 
     processNext();
 
   });
+
 });

tools/show_style

@@ -0,0 +1,29 @@
+#!/bin/sh
+
+# TODO: port to node, if you really need it
+
+REDIS_PORT=6379 # default port
+
+
+if test -z "$1"; then
+  echo "Usage: $0 <username> [<tablename>|~<token>]" >&2
+  exit 1
+fi
+
+username="$1"
+token="$2"
+
+dbname=`redis-cli -p ${REDIS_PORT} -n 5 hget "rails:users:${username}" "database_name"`
+if test $? -ne 0; then
+  exit 1
+fi
+if test -z "${dbname}"; then
+  echo "Username ${username} unknown by redis (try CARTODB/script/restore_redis?)" >&2
+  exit 1
+fi
+echo "Database name for user ${username}: ${dbname}" # only if verbose?
+if test -n "$token"; then
+  redis-cli get "map_style|${dbname}|${token}" | sed -e 's/\\n/\n/g' -e 's/\\//g'
+else
+  redis-cli keys "map_style|${dbname}|*"
+fi