Release 1.21.0

Allow a different cache-control max-age for layergroup responses

.gitignore

@@ -1,2 +1,9 @@
-node_modules
-.idea
+node_modules*
+config.status*
+config/environments/*.js
+.idea
+tools/munin/windshaft.conf
+logs/
+pids/
+redis.pid
+test.log

.travis.yml

@@ -0,0 +1,20 @@
+addons:
+  postgresql: "9.3"
+
+before_install:
+  - createdb template_postgis
+  - psql -c "CREATE EXTENSION postgis" template_postgis
+
+env:
+  - NPROCS=1 JOBS=1 PGUSER=postgres
+
+language: node_js
+node_js:
+  - "0.8"
+  - "0.10"
+
+notifications:
+  irc:
+    channels:
+      - "irc.freenode.org#cartodb"
+    use_notice: true

HOWTO_RELEASE

@@ -0,0 +1,20 @@
+1. Test (make clean all check), fix if broken before proceeding
+2. Ensure proper version in package.json 
+3. Ensure NEWS section exists for the new version, review it, add release date
+4. Drop npm-shrinkwrap.json
+5. Run npm shrinkwrap to recreate npm-shrinkwrap.json
+6. Commit package.json, npm-shrinwrap.json, NEWS
+7. git tag -a Major.Minor.Patch # use NEWS section as content
+8. Announce on cartodb@googlegroups.com
+9. Stub NEWS/package for next version
+
+Versions:
+
+Bugfix releases increment Patch component of version.
+Feature releases increment Minor and set Patch to zero.
+If backward compatibility is broken, increment Major and
+set to zero Minor and Patch.
+
+Branches named 'b<Major>.<Minor>' are kept for any critical
+fix that might need to be shipped before next feature release
+is ready.

LICENCE

@@ -1,27 +0,0 @@
-Copyright (c) 2011, Vizzuality 
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-1. Redistributions of source code must retain the above copyright
-   notice, this list of conditions and the following disclaimer.
-2. Redistributions in binary form must reproduce the above copyright
-   notice, this list of conditions and the following disclaimer in the
-   documentation and/or other materials provided with the distribution.
-3. All advertising materials mentioning features or use of this software
-   must display the following acknowledgement:
-   This product includes software developed by Vizzuality.
-4. Neither the name of Vizzuality nor the
-   names of its contributors may be used to endorse or promote products
-   derived from this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS "AS IS" AND ANY
-EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY
-DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2014, Vizzuality
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without 
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this 
+list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, 
+this list of conditions and the following disclaimer in the documentation 
+and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors 
+may be used to endorse or promote products derived from this software without 
+specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

Makefile

@@ -1,8 +1,35 @@
+srcdir=$(shell pwd)
+
 all:
 	npm install
 
 clean:
 	rm -rf node_modules/*
 
-check:
-	./run_tests.sh
+distclean: clean
+	rm config.status*
+
+config.status--test:
+	./configure --environment=test
+
+config/environments/test.js: config.status--test
+	./config.status--test 
+
+check-local: config/environments/test.js
+	./run_tests.sh ${RUNTESTFLAGS} \
+    test/unit/cartodb/*.js \
+    test/acceptance/*.js 
+
+check-submodules:
+	PATH="$$PATH:$(srcdir)/node_modules/.bin/"; \
+	for sub in windshaft grainstore node-varnish mapnik; do \
+	  if test -e node_modules/$${sub}; then \
+	      echo "Testing submodule $${sub}"; \
+	      make -C node_modules/$${sub} check || exit 1; \
+	  fi; \
+	done
+
+check-full: check-local check-submodules
+
+check: check-local
+

NEWS.md

@@ -0,0 +1,626 @@
+1.21.0 -- 2014-10-24
+--------------------
+
+New features:
+ - Allow a different cache-control max-age for layergroup responses
+
+
+1.20.2 -- 2014-10-20
+--------------------
+
+Announcements:
+ - Upgrades windshaft to 0.31.0
+
+
+1.20.1 -- 2014-10-17
+--------------------
+
+Announcements:
+ - Upgrades redis-mpool to 0.3.0
+
+
+1.20.0 -- 2014-10-15
+--------------------
+
+New features:
+ - Report to statsd the status of redis pools
+ - Upgrades Windshaft to start reporting redis/renderers/mapnik pool metrics
+
+Enhancements:
+ - Share one redis-mpool across the application
+
+
+1.19.0 -- 2014-10-14
+--------------------
+
+Announcements:
+ - Dropping support for npm <1.2.1
+   npm-shrinkwrap.json is incompatible when generated with npm >=1.2.1 and consumed by npm <1.2.1
+ - Upgrades windshaft to 0.28.2
+ - Generates npm-shrinkwrap.json with npm >1.2.0
+
+
+1.18.2 -- 2014-10-13
+--------------------
+
+Bug fixes:
+ - Defaults resultSet to object if undefined in QueryTablesApi
+
+Announcements:
+ - Upgrades windshaft to 0.28.1
+
+
+1.18.1 -- 2014-10-13
+--------------------
+
+New features:
+ - Allow to add more node.js' threadpool workers via process.env.UV_THREADPOOL_SIZE
+
+
+1.18.0 -- 2014-10-03
+--------------------
+
+Announcements:
+ - Comes back to use mapnik 2.3.x based on cartodb/node-mapnik@1.4.15-cdb from windshaft@0.28.0
+
+
+1.17.2 -- 2014-10-01
+--------------------
+
+Announcements:
+ - Upgrades windshaft to 0.27.2 which downgrades node-mapnik to 0.7.26-cdb1
+
+
+1.17.1 -- 2014-09-30
+--------------------
+
+Announcements:
+ - Upgrades windshaft to 0.27.1 which downgrades node-mapnik to 1.4.10
+
+Enhancements:
+ - TTL for template locks so they are not kept forever
+ - Upgrades mocha
+
+
+1.17.0 -- 2014-09-25
+--------------------
+
+New features:
+ - Starts using mapnik 2.3.x
+
+Enhancements:
+ - Upgrades windshaft and cartodb-redis
+ - Supports `!scale_denominator!` dynamic param in SQL queries
+ - Metrics revamp: removes and adds some metrics
+ - Adds poolSize configuration for mapnik
+
+1.16.1 -- 2014-08-19
+--------------------
+
+Enhancements:
+ - Upgrades cartodb-redis
+
+1.16.0 -- 2014-08-18
+--------------------
+
+New features:
+ - Configurable QueryTablesAPI to call directly postgresql using cartodb-psql
+   or to keep using a request to the SQL API
+
+Enhancements:
+ - Removes mapnik dependency as it now relies on Windshaft to check mapnik version
+ - Upgrades dependencies:
+    - underscore
+    - lzma
+    - log4js
+    - rollbar
+    - windshaft
+    - request
+
+1.15.0 -- 2014-08-13
+--------------------
+Enhancements:
+ - Upgrades dependencies:
+    - redis-mpool
+    - cartodb-redis
+    - windshaft
+ - Specifies name in the redis pool
+ - Slow pool configuration in example configurations
+
+
+1.14.0 -- 2014-08-07
+--------------------
+
+Enhancements:
+ - SQL API requests moved to its own entity
+
+New features:
+ - Affected tables and last updated time for a query are performed in a single
+   request to the SQL API
+ - Allow specifying the tile format, upgrades windshaft and grainstore
+   dependencies for this matter
+
+
+1.13.1 -- 2014-08-04
+--------------------
+
+Enhancements:
+ - Profiler header sent as JSON string
+
+
+1.13.0 -- 2014-07-30
+--------------------
+
+New features:
+ - Support for postgresql schemas
+ - Use public user from redis
+ - Support for several auth tokens
+
+1.12.1 -- 2014-06-24
+--------------------
+
+Enhancements:
+ - Caches layergroup and sets X-Cache-Channel in GET requests also in named maps
+
+1.12.0 -- 2014-06-24
+--------------------
+
+New features:
+ - Caches layergroup and sets X-Cache-Channel in GET requests
+
+1.11.1 -- 2014-05-07
+--------------------
+
+Enhancements:
+
+ - Upgrade Windshaft to 0.21.0, see
+ http://github.com/CartoDB/Windshaft/blob/0.21.0/NEWS
+
+1.11.0 -- 2014-04-28
+--------------------
+
+New features:
+
+ - Add support for log_filename directive
+ - Reopen log file on SIGHUP, for better logrotate integration
+
+Enhancements:
+
+ - Set default PostgreSQL application name to "cartodb_tiler"
+
+1.10.2 -- 2014-04-08
+--------------------
+
+Bug fixes:
+
+ - Fix show_style tool broken since 1.8.1
+ - Fix X-Cache-Channel of tiles accessed via signed token (#188)
+
+1.10.1 -- 2014-03-21
+--------------------
+
+Bug fixes:
+
+ - Do not cache non-success jsonp responses (#186)
+
+1.10.0 -- 2014-03-20
+--------------------
+
+New features:
+
+ - Add optional support for rollbar (#150)
+
+Enhancements:
+
+ - Do not send connection details to client (#183)
+ - Upgrade node-varnish to 0.3.0
+ - Upgrade Windshaft to 0.20.0, see
+   http://github.com/CartoDB/Windshaft/blob/0.20.0/NEWS
+ - Include tiler version in startup log
+ - Install an uncaught exception handler
+ - Require own fork of node-mapnik, with temptative fix
+   for libxml usage (glibc detected corruptions)
+
+Other changes:
+
+ - Switch to 3-clause BSD license (#184)
+
+1.9.0 -- 2014-03-10
+-------------------
+
+New features:
+
+ - Allow to set server related configuration in serverMetadata (#182)
+
+1.8.5 -- 2014-03-10
+-------------------
+
+Enhancements:
+
+ - Set statsd prefix for all endpoints 
+ - Respond with a permission denied on attempt to access map tiles waiving
+   signature of someone who had not left any (#170)
+ - Do not log an error on GET / (#177)
+ - Do not UNWATCH on every redis client release (#161)
+ - Include API docs (#164)
+ - Add "cacheDns" statsd setting in the example configs
+ - Do not send duplicated stats on template instanciation
+ - Do not die on dns resolution errors (#178, #180)
+
+Bug fixes:
+
+ - Do not cache map creation responses (#176)
+
+1.8.4 -- 2014-03-03
+-------------------
+
+Enhancements:
+
+ - Really skip CDB_TableMetadata lookup for sql affected by no tables (#169)
+ - Upgrade windshaft to 0.19.2, see node_modules/windshaft/NEWS
+ - Clarify obscure "ECONNREFUSED" error message (#171)
+ - Change some http status responses to be more appropriate to the case
+ - Forbid using map signatures of foreign users (#172)
+ - Forbid instanciating templates of foreign users (#173)
+ - Allow passing environment configuration name via NODE_ENV to app.js
+ - Print environment configuration name on app start
+
+Bug fixes:
+
+ - Fix database connection settings on template instanciation (#174)
+
+1.8.3 -- 2014-02-27
+-------------------
+
+Enhancements:
+
+ - Upgrades windshaft to 0.19.1 with many performance improvements,
+   See node_modules/windshaft/NEWS 
+ - Improve speed of instanciating a map (#147, #159, #165)
+ - Give meaningful error on attempts to use map tokens
+   with attribute service (#156)
+ - Reduce sql-api communication timeout, and allow overriding (#167)
+   [ new sqlapi.timeout directive, defaults to 100 ms ]
+ - Do not query CDB_TableMetadata for queries affected by no tables (#168)
+
+1.8.2 -- 2014-02-25
+-------------------
+
+Enhancements:
+
+ * Allow using ":host" as part of statsd.prefix (#153)
+ * Expand "addCacheChannel" stats
+ * Allow using GET with sql-api for queries shorter than configured len (#155)
+   [ new sqlapi.max_get_sql_length directive, defaults to 2048 ]
+ * Do not log an error for a legit request requiring no X-Cache-Channel
+
+Bug fixes:
+
+ * Fix munin plugin after log format changes (#154)
+
+1.8.1 -- 2014-02-19
+-------------------
+
+Enhancements:
+
+ * Use log4js logger (#138)
+
+Bug fixes:
+
+ * Always generate X-Cache-Channel for token-based tile responses (#152)
+
+1.8.0 -- 2014-02-18
+-------------------
+
+Enhancements:
+
+ * Add script to flush caches (#140)
+ * Add script to list templates
+ * Add statsd support (#139)
+ * Add support for specifying a varnish password
+ * Avoid sending multiple varnish invalidation at once (#135)
+ * Tested with node-0.10 (#141)
+ * Use single redis pooler for torque and grainstore
+ * Reduce cost of garbage collection for localized resources
+ * Allow limiting number of templates for each user (#136)
+ * Allow configuring TTL of mapConfigs via "mapConfigTTL"
+
+1.7.1 -- 2014-02-11
+-------------------
+
+Enhancements:
+
+ * Disable debug logging unless "debug" config param evaluates to true (#137)
+ * Require windshaft 0.17.2 for further reducing log noise (#137)
+
+1.7.0 -- 2014-02-11
+-------------------
+
+New features:
+
+ * Add support for torque tiles (#112)
+ * Add attributes service (#118)
+ * Implement Unified Map API (#126)
+ * Make endpoints configurable (#127)
+
+Enhancements:
+
+ * Allow specifying fixed sqlapi host address (#117)
+ * Include template hash in template instance response, to keep caches
+   of different instances separated (#105)
+
+Bug fixes:
+
+ * Allow space padding in template variables usage (#129)
+ * Allow passing numbers as values for numeric template variables (#130)
+
+
+1.6.3 -- 2014-01-30
+-------------------
+
+Bug fixes: 
+
+* layergroup accept both map_key and api_key (#91)
+* Fix public instanciation of signed template accessing private data (#114)
+* Fix show_style in presence of complex styles
+* Fix use of maxzoom in layergroup config (via windshaft-0.15.1)
+
+Enhancements:
+
+* Add support for instanciating a template map with JSONP (#116)
+* Stop processing XML on renderer creation, not needed anymore since 1.6.1
+  introduced on-demand XML generation.
+
+1.6.2 -- 2014-01-23
+-------------------
+
+Bug fixes:
+
+* Fix support for long (>64k chars) queries in layergroup creation (#111)
+
+Enhancements:
+
+* Enhance tools/show_style to accept an environment parameter and
+  print XML style now it is not in redis anymore (#110)
+* Support CORS in template instanciation endpoint (#113)
+
+1.6.1 -- 2014-01-15
+-------------------
+
+Bug fixes:
+
+* Drop cache headers from error responses (#107)
+* Localize external CartoCSS resources at renderer creation time (#108)
+
+1.6.0 -- 2014-01-10
+-------------------
+
+New features:
+
+ * Add 'user_from_host' directive to generalize username extraction (#100)
+ * Implement signed template maps (#98)
+
+
+Other changes:
+
+ * Update cartodb-redis dependency to "~0.3.0"
+ * Update redis-server dependency to "2.4.0+"
+
+1.5.2 -- 2013-12-05
+-------------------
+
+Bug fixes:
+
+* Fix configuration-level compatibility with versions prior to 1.5 (#96)
+* Fix use of old layergroups on mapnik upgrade (#97)
+
+1.5.1 -- 2013-11-28
+-------------------
+
+Bug fixes:
+
+* Survive presence of malformed CartoCSS in redis (#94)
+* Accept useless point-transform:scale directives (#93)
+
+1.5.0 -- 2013-11-19
+-------------------
+
+NOTE: new configuration directives `postgres_auth_pass` and
+      `postgres.password` added; see config/environments/*.example
+      for documentation.
+
+Improvements:
+
+* Add support for configuring database connection passwords
+* Optionally read user-specific database_host and database_password
+  from redis as per CartoDB-2.5.0 model (#88, #89)
+* Do not force ending dot in SQL-API hostname, for easier testing
+
+Bug fixes:
+
+* Return CORS headers when creating layergroups via GET (windshaft/#92)
+* Fix http status on database authentication error (windshaft/#94)
+* Fix text-face-name error at layergroup creation (windshaft/#93)
+
+Other changes:
+
+* CartoDB redis interaction delegated to "cartodb-redis" module
+
+
+1.4.1 -- 2013-11-08
+-------------------
+
+* Fix support for exponential notation in CartoCSS filter values (#87)
+
+1.4.0 -- 2013-10-31
+-------------------
+
+* Add Support for Mapnik-2.2.0 (#78)
+
+1.3.6 -- 2013-10-11
+-------------------
+
+* Restore support for node-0.8.9 accidentally dropped by 1.3.5
+  NOTE: needs removing node_modules/windshaft and re-running npm install
+
+1.3.5 -- 2013-10-03
+-------------------
+
+* Fixing apostrophes in CartoCSS
+* Fix "sql/table must contain zoom variable" error when using
+  "[ zoom > 3]" CartoCSS snippets (note the space)
+* Fix backward compatibility handling of sqlapi.host configuration (#82)
+* Fix error for invalid text-name in CartoCSS (#81)
+* Do not let anonymous requests use authorized renderer caches 
+
+1.3.4
+------
+
+NOTE: configuration sqlapi.host renamed to sqlapi.domain
+      (support for "sqlapi.host" is retained for backward compatibility)
+
+* Improve empty CartoCSS error message
+* Improve invalid mapnik-geometry-type CSS error message
+* Fix race condition in localization of network resources
+
+1.3.3
+------
+* Set Last-Modified header to allow for 304 responses
+* Add profiling support (needs useProfiler in env config file)
+* Fix double-checking for layergroups with no interactivity
+* Log full layergroup config at creation time (#76)
+
+1.3.2
+------
+* Set default layergroup TTL to 2 hours
+* Serve multilayer tiles and grid with persistent cache control
+
+1.3.1
+------
+* Fix deadlock on new style creation
+* Fix database authentication with multi-table layergroups
+* Add tile and grid fetching checks at layergroup creation time
+* Fix SQL error reporting to NOT split on newline
+* Fix support for CartoCSS attachments
+
+1.3.0
+------
+* Change stats format for multilayer map token request, see
+  http://github.com/Vizzuality/Windshaft-cartodb/wiki/Redis-stats-format
+
+1.2.1
+------
+* Fix multilayer post from firefox
+* Fix multilayer cartocss layer name handling
+
+1.2.0
+------
+* Multilayer API changes
+  * Layers passed by index in grid fetching url
+  * Interactivity only specified in layergroup config
+  * Embed cache_buster within token 
+  * Use ISO format for last_modified timestamp
+* Expected LZMA encoding changed to base64
+
+1.1.10
+------
+* Fix regression with default interactivity parameter (#74)
+* More verbose logging for SQL api connection errors
+* Write stats for multilayer map token request
+
+1.1.9
+-----
+* Handle SQL API errors by requesting no Varnish cache
+* Fix X-Cache-Channel for multilayer (by token) responses
+* Add last_modified field to layergroup creation response (#72)
+* Deprecate signal handler for USR1, add handler for USR2 (#71)
+* Fix support for ampersend characters in CartoCSS
+* Add support for LZMA compressed GET parameters
+* Add support for creating layergroups via GET
+
+1.1.8
+-----
+* Require Windshaft-0.9.1, to reduce harmfulness of cache_buster param
+
+1.1.7 (DD//MM//YY)
+-----
+* Do not let /etc/services confuse FD checker (munin plugin)
+* Multilayer support (#72)
+* Expose renderer settings in the environment config files
+
+1.1.6 (19//02//13)
+-----
+* Require windshaft 0.8.5, fixing some stability issues
+  and providing cache info on request
+* Require grainstore 0.10.9, fixing an issue with multi-geom markers
+* Enhance run_tests.sh to allow running single tests and skipping preparation
+* Fix async throws in getGeometryType, getInfoWindow and getMapMetadata
+* Survive connection refusals from redis
+* Add maxConnection environment configuration, default to 128
+
+1.1.5 (DD//MM//YY)
+-----
+* Fix bogus cached return of utf grid for fully contained tiles (#67)
+
+1.1.4 (DD//MM//YY)
+-----
+* Reduce default extent to allow for consistent proj4 round-tripping 
+* Enhance reset_styles script to use full configuration (#62)
+* Have reset_styles script also drop extended keys (#58)
+* Fix example postgis parameter for simplifying input geoms (#63)
+* Add row_limit to example config (#64)
+
+1.1.3 (30//11//12)
+-----
+* Fix reset_styles script to really skip extended keys
+* CartoCSS versioning
+ * Mapnik-version dependent default styles
+ * Enhance 2.0 -> 2.1 transforms:
+  * styles with conditional markers
+  * scale arrow markers by 50%
+
+1.1.2 (DD//MM//YY)
+-----
+* CartoCSS versioning
+ * Fix use of "style_version" with GET (inline styles)
+ * Enhance 2.0 -> 2.1 transforms:
+  * styles with no semicolon
+  * markers shift due to geometry clipping
+
+1.1.1 (DD//MM//YY)
+-----
+* Add support for persistent client cache headers
+* Fix crash on unknown user (#55)
+* Add /version entry point
+* CartoCSS versioning
+ * Include style_version in GET /style response
+ * Support style_version and style_convert parameters in POST /style request
+ * Support style_version in GET /:z/:x/:y request
+
+1.1.0 (30/10/12)
+=======
+* Add /version entry point
+* CartoCSS versioning
+ * Include version in GET /style response
+ * Support version and convert parameters in POST /style request
+ * Autodetect target mapnik version and let config override it
+ * Add tools/reset_styles script to batch-reset (and optionally convert) styles
+* Configurable logging format (#4)
+* Detailed error on missing user metadata 
+* Properly handle unauthenticated requests for metadata
+* Accept "api_key" in addition to "map_key",
+  both in query_string and POST body (#38)
+* Add ./configure script
+* Allow listening on host IP
+* Replaced environment configs by .example ones
+* Fixed some issues with cluster2
+
+1.0.0 (03/10/12)
+-----
+* Migrated to node 0.8.x.
+
+0.9.0 (25/09/12)
+-----
+* External resources in CartoCSS
+* Added X-Cache-Channel header in all the tiler GET requests
+* Small fixes

README.md

@@ -1,21 +1,53 @@
 Windshaft-CartoDB
 ==================
 
-NOTE: requires node-0.4.x
+[![Build Status](http://travis-ci.org/CartoDB/Windshaft-cartodb.png)]
+(http://travis-ci.org/CartoDB/Windshaft-cartodb)
 
 This is the CartoDB map tiler. It extends Windshaft with some extra
 functionality and custom filters for authentication
 
 * reads dbname from subdomain and cartodb redis for pretty tile urls
-* configures windshaft to publish cartodb_id as the interactivity layer
+* configures windshaft to publish ``cartodb_id`` as the interactivity layer
 * gets the default geometry type from the cartodb redis store
 * allows tiles to be styled individually
 * provides a link to varnish high speed cache
-* provides a infowindow endpoint for windshaft
-* provides a map_metadata endpoint for windshaft
+* provides a ``infowindow`` endpoint for windshaft (DEPRECATED)
+* provides a ``map_metadata`` endpoint for windshaft (DEPRECATED)
+* provides signed template maps API
+  (http://github.com/CartoDB/Windshaft-cartodb/wiki/Template-maps)
 
-Install
--------
+Requirements
+------------
+
+ [core]
+ - node-0.8.x+
+ - PostgreSQL-8.3+
+ - PostGIS-1.5.0+
+ - Redis 2.4.0+ (http://www.redis.io)
+ - Mapnik 2.0 or 2.1 
+
+ [for cache control]
+ - CartoDB-SQL-API 1.0.0+
+ - CartoDB 0.9.5+ (for ``CDB_QueryTables``)
+ - Varnish (http://www.varnish-cache.org)
+
+ [for running the testsuite]
+ - Imagemagick (http://www.imagemagick.org)
+
+Configure
+---------
+
+Create the config/environments/<env>.js files (there are .example files
+to start from). You can optionally use the ./configure script for this,
+see ```./configure --help``` to see available options.
+
+Look at lib/cartodb/server_options.js for more on config
+
+Build/install
+-------------
+
+To fetch and build all node-based dependencies, run:
 
 ```
 git clone
@@ -28,22 +60,18 @@ happen to have startup errors you may need to force rebuilding those
 modules. At any time just wipe out the node_modules/ directory and run
 ```npm install``` again.
 
-Configure
----------
-
-Edit config/environments/<env>.js files
-
-Look at lib/cartodb/server_options for more on config
 
 Run
 ---
 
 ```
-node app.js [development | staging | production]
+node app.js <env> 
 ```
 
-Note that caches are kept in redis. If you're not seeing what
-you expect there may be out-of-sync records in there.
+Where <env> is the name of a configuration file under config/environments/.
+
+Note that caches are kept in redis. If you're not seeing what you expect
+there may be out-of-sync records in there.
 Take a look: http://redis.io/commands
 
 
@@ -58,8 +86,16 @@ Args:
 
 * sql - plain SQL arguments
 * interactivity - specify the column to use in UTFGrid
-* cache_buster - if needed you can add a cachebuster to make sure you're
-  rendering new
+* cache_buster - Specify an identifier for the internal tile cache.
+                 Requesting tiles with the same cache_buster value may
+                 result in being served a cached version of the tile
+                 (even when requesting a tile for the first time, as tiles
+                 can be prepared in advance)
+* cache_policy - Set to "persist" to have the server send an Cache-Control
+                 header requesting caching devices to keep the response
+                 cached as much as possible. This is best used with a
+                 timestamp value in cache_buster for manual control of
+                 updates.
 * geom_type - override the cartodb default
 * style - override the default map style with Carto
 
@@ -71,6 +107,8 @@ Args:
 Args:
 
 * style - the style in CartoCSS you want to set
+* style_version - the version of the style for POST
+* style_convert - request conversion to target version (both POST and GET)
 
 
 **INFOWINDOW**

app.js

@@ -7,31 +7,119 @@
  * environments: [development, production]
  */
 
+var path = require('path'),
+    fs = require('fs'),
+    RedisPool = require('redis-mpool')
+;
+
+
+if ( process.argv[2] ) ENV = process.argv[2];
+else if ( process.env['NODE_ENV'] ) ENV = process.env['NODE_ENV'];
+else ENV = 'development';
+
+process.env['NODE_ENV'] = ENV;
 
 // sanity check
-var ENV = process.argv[2]
-if (ENV != 'development' && ENV != 'production'){
+if (ENV != 'development' && ENV != 'production' && ENV != 'staging' ){
     console.error("\nnode app.js [environment]");
-    console.error("environments: [development, production]\n");
+    console.error("environments: development, production, staging\n");
     process.exit(1);
 }
 
-var _ = require('underscore')
-    , Step       = require('step')
-	, CartodbWindshaft = require('./lib/cartodb/cartodb_windshaft');
+var _ = require('underscore');
 
 // set environment specific variables
-global.settings     = require(__dirname + '/config/settings');
 global.environment  = require(__dirname + '/config/environments/' + ENV);
-_.extend(global.settings, global.environment);
+global.environment.api_hostname = require('os').hostname().split('.')[0];
 
-// Include cart_data.js only _after_ the "global" variable is set
+global.log4js = require('log4js');
+log4js_config = {
+  appenders: [],
+  replaceConsole:true
+};
+
+if (global.environment.uv_threadpool_size) {
+    process.env.UV_THREADPOOL_SIZE = global.environment.uv_threadpool_size;
+}
+
+if ( global.environment.log_filename ) {
+  var logdir = path.dirname(global.environment.log_filename);
+  // See cwd inlog4js.configure call below
+  logdir = path.resolve(__dirname, logdir);
+  if ( ! fs.existsSync(logdir) ) {
+    console.error("Log filename directory does not exist: " + logdir);
+    process.exit(1);
+  }
+  console.log("Logs will be written to " + global.environment.log_filename);
+  log4js_config.appenders.push(
+    { type: "file", filename: global.environment.log_filename }
+  );
+} else {
+  log4js_config.appenders.push(
+    { type: "console", layout: { type:'basic' } }
+  );
+}
+
+if ( global.environment.rollbar ) {
+  log4js_config.appenders.push({
+    type: __dirname + "/lib/cartodb/log4js_rollbar.js",
+    options: global.environment.rollbar
+  });
+}
+
+log4js.configure(log4js_config, { cwd: __dirname });
+global.logger = log4js.getLogger();
+
+var redisOpts = _.extend(global.environment.redis, { name: 'windshaft' }),
+    redisPool = new RedisPool(redisOpts);
+
+// Include cartodb_windshaft only _after_ the "global" variable is set
 // See https://github.com/Vizzuality/Windshaft-cartodb/issues/28
-var cartoData = require('./lib/cartodb/carto_data');
-
-var Windshaft = require('windshaft');
-var serverOptions = require('./lib/cartodb/server_options');
+var CartodbWindshaft = require('./lib/cartodb/cartodb_windshaft'),
+    serverOptions = require('./lib/cartodb/server_options')(redisPool);
 
 ws = CartodbWindshaft(serverOptions);
-ws.listen(global.environment.port);
-console.log("Windshaft tileserver started on port " + global.environment.port);
+
+if (global.statsClient) {
+    redisPool.on('status', function(status) {
+        var keyPrefix = 'windshaft.redis-pool.' + status.name + '.db' + status.db + '.';
+        global.statsClient.gauge(keyPrefix + 'count', status.count);
+        global.statsClient.gauge(keyPrefix + 'unused', status.unused);
+        global.statsClient.gauge(keyPrefix + 'waiting', status.waiting);
+    });
+}
+
+// Maximum number of connections for one process
+// 128 is a good number if you have up to 1024 filedescriptors
+// 4 is good if you have max 32 filedescriptors
+// 1 is good if you have max 16 filedescriptors
+ws.maxConnections = global.environment.maxConnections || 128;
+
+ws.listen(global.environment.port, global.environment.host);
+
+var version = require("./package").version;
+
+ws.on('listening', function() {
+  console.log("Windshaft tileserver " + version + " started on "
+              + global.environment.host + ':' + global.environment.port
+              + " (" + ENV + ")");
+});
+
+// DEPRECATED, use SIGUSR2
+process.on('SIGUSR1', function() {
+  console.log('WARNING: handling of SIGUSR1 by Windshaft-CartoDB is deprecated, please send SIGUSR2 instead');
+  ws.dumpCacheStats();
+});
+
+process.on('SIGUSR2', function() {
+  ws.dumpCacheStats();
+});
+
+process.on('SIGHUP', function() {
+  log4js.configure(log4js_config);
+  console.log('Log files reloaded');
+});
+
+process.on('uncaughtException', function(err) {
+  logger.error('Uncaught exception: ' + err.stack); 
+});

cluster.js

@@ -1,45 +0,0 @@
-/*
- * Windshaft-CartoDB
- * ===============
- *
- * ./app.js [environment]
- *
- * environments: [development, production]
- */
-
-var cluster = require('cluster');
-
-// sanity check
-var ENV = process.argv[2]
-if (ENV != 'development' && ENV != 'production' && ENV != 'staging'){
-    console.error("\nnode app.js [environment]");
-    console.error("environments: [development, production, staging]\n");
-    process.exit(1);
-}
-
-var _ = require('underscore')
-    , Step       = require('step')
-	, CartodbWindshaft = require('./lib/cartodb/cartodb_windshaft');
-    
-
-// set environment specific variables
-global.settings     = require(__dirname + '/config/settings');
-global.environment  = require(__dirname + '/config/environments/' + ENV);
-_.extend(global.settings, global.environment);
-
-// Include cart_data.js only _after_ the "global" variable is set
-// See https://github.com/Vizzuality/Windshaft-cartodb/issues/28
-var cartoData = require('./lib/cartodb/carto_data');
-
-var Windshaft = require('windshaft');
-var serverOptions = require('./lib/cartodb/server_options');
-
-ws = CartodbWindshaft(serverOptions);
-cluster(ws)
-  .use(cluster.logger('logs'))
-  .use(cluster.stats())
-  .use(cluster.pidfiles('pids'))
-  .set('workers', 1)
-  .listen(global.environment.port, global.environment.host);
-
-console.log("Windshaft tileserver started on port " + global.environment.port);

config/environments/development.js

@@ -1,43 +0,0 @@
-var config = {
-     environment: 'development'
-    ,port: 8181
-    ,host: '127.0.0.1'
-    ,enable_cors: true
-    ,cache_enabled: false
-    ,postgres_auth_user: 'development_cartodb_user_<%= user_id %>'
-    ,postgres: {
-        type: "postgis",
-        user: "publicuser",
-        host: '127.0.0.1',
-        port: 5432,
-        extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188",
-        /* experimental
-        geometry_field: "the_geom",
-        extent: "-180,-90,180,90",
-        srid: 4326,
-        */
-        simplify: true
-    }
-    ,millstone: {
-        cache_basedir: '/tmp/cdb-tiler-dev/millstone-dev'
-    }
-    ,redis: {
-        host: '127.0.0.1',
-        port: 6379,
-        idleTimeoutMillis: 1,
-        reapIntervalMillis: 1
-    }
-    ,sqlapi: {
-        protocol: 'http',
-        host: 'localhost.lan',
-        port: 8080,
-        version: 'v1'
-    }
-    ,varnish: {
-        host: 'localhost',
-        port: 6082,
-        ttl: 86400
-    }
-};
-
-module.exports = config;

config/environments/development.js.example

@@ -0,0 +1,155 @@
+var config = {
+     environment: 'development'
+    ,port: 8181
+    ,host: '127.0.0.1'
+    ,uv_threadpool_size: undefined
+    // Regular expression pattern to extract username
+    // from hostname. Must have a single grabbing block.
+    ,user_from_host: '^(.*)\\.localhost'
+
+    // Base URLs for the APIs
+    //
+    // See http://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API
+    // 
+    // Base url for the Templated Maps API
+    // "/api/v1/map/named" is the new API,
+    // "/tiles/template" is for compatibility with versions up to 1.6.x
+    ,base_url_templated: '(?:/api/v1/map/named|/tiles/template)'
+    // Base url for the Detached Maps API
+    // "maps" is the the new API,
+    // "tiles/layergroup" is for compatibility with versions up to 1.6.x
+    ,base_url_detached: '(?:/api/v1/map|/tiles/layergroup)'
+    // Base url for the Inline Maps and Table Maps API
+    ,base_url_legacy: '/tiles/:table'
+
+    // Maximum number of connections for one process
+    // 128 is a good value with a limit of 1024 open file descriptors
+    ,maxConnections:128
+    // Maximum number of templates per user. Unlimited by default.
+    ,maxUserTemplates:1024
+    // Seconds since "last creation" before a detached
+    // or template instance map expires. Or: how long do you want
+    // to be able to navigate the map without a reload ?
+    // Defaults to 7200 (2 hours)
+    ,mapConfigTTL: 7200
+    // idle socket timeout, in milliseconds 
+    ,socket_timeout: 600000
+    ,enable_cors: true
+    ,cache_enabled: false
+    ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
+    // If log_filename is given logs will be written
+    // there, in append mode. Otherwise stdout is used (default).
+    // Log file will be re-opened on receiving the HUP signal
+    ,log_filename: 'logs/node-windshaft.log'
+    // Templated database username for authorized user
+    // Supported labels: 'user_id' (read from redis)
+    ,postgres_auth_user: 'development_cartodb_user_<%= user_id %>'
+    // Templated database password for authorized user
+    // Supported labels: 'user_id', 'user_password' (both read from redis)
+    ,postgres_auth_pass: '<%= user_password %>'
+    ,postgres: {
+        // Parameters to pass to datasource plugin of mapnik
+        // See http://github.com/mapnik/mapnik/wiki/PostGIS
+        type: "postgis",
+        user: "publicuser",
+        password: "public",
+        host: '127.0.0.1',
+        port: 5432,
+        extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
+        /* experimental
+        geometry_field: "the_geom",
+        extent: "-180,-90,180,90",
+        srid: 4326,
+        */
+        row_limit: 65535,
+        simplify_geometries: true,
+        /*
+         * Set persist_connection to false if you want
+         * database connections to be closed on renderer
+         * expiration (1 minute after last use).
+         * Setting to true (the default) would never
+         * close any connection for the server's lifetime
+         */
+        persist_connection: false,
+        max_size: 500
+    }
+    ,mapnik_version: undefined
+    ,mapnik_tile_format: 'png8:m=h'
+    ,statsd: {
+        host: 'localhost',
+        port: 8125,
+        prefix: 'dev.',
+        cacheDns: true
+        // support all allowed node-statsd options
+    }
+    ,renderer: {
+      // Milliseconds since last access before renderer cache item expires
+      cache_ttl: 60000,
+      metatile: 4,
+      bufferSize: 64,
+      statsInterval: 5000 // milliseconds between each report to statsd about number of renderers and mapnik pool status
+    }
+    ,millstone: {
+        // Needs to be writable by server user
+        cache_basedir: '/tmp/cdb-tiler-dev/millstone-dev'
+    }
+    ,redis: {
+        host: '127.0.0.1',
+        port: 6379,
+        // Max number of connections in each pool.
+        // Users will be put on a queue when the limit is hit.
+        // Set to maxConnection to have no possible queues.
+        // There are currently 2 pools involved in serving
+        // windshaft-cartodb requests so multiply this number
+        // by 2 to know how many possible connections will be
+        // kept open by the server. The default is 50.
+        max: 50,
+        returnToHead: true, // defines the behaviour of the pool: false => queue, true => stack
+        idleTimeoutMillis: 1, // idle time before dropping connection
+        reapIntervalMillis: 1, // time between cleanups
+        slowQueries: {
+            log: true,
+            elapsedThreshold: 200
+        },
+        slowPool: {
+            log: true, // whether a slow acquire must be logged or not
+            elapsedThreshold: 25 // the threshold to determine an slow acquire must be reported or not
+        },
+        emitter: {
+            statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd
+        }
+    }
+    ,sqlapi: {
+        protocol: 'http',
+        // If "host" is given, it will be used
+        // to connect to the SQL-API without a
+        // DNS lookup
+        host: '127.0.0.1',
+        port: 8080,
+        // The "domain" part will be appended to 
+        // the cartodb username and passed to
+        // SQL-API requests in the Host HTTP header
+        domain: 'localhost.lan',
+        version: 'v1',
+        // Maximum lenght of SQL query for GET
+        // requests. Longer queries will be sent
+        // using POST. Defaults to 2048
+        max_get_sql_length: 2048,
+        // Maximum time to wait for a response,
+        // in milliseconds. Defaults to 100.
+        timeout: 100
+    }
+    ,varnish: {
+        host: 'localhost',
+        port: 6082,
+        secret: 'xxx',
+        ttl: 86400,
+        layergroupTtl: 86400 // the max-age for cache-control header in layergroup responses
+    }
+    // If useProfiler is true every response will be served with an
+    // X-Tiler-Profile header containing elapsed timing for various 
+    // steps taken for producing the response.
+    ,useProfiler:true
+};
+
+module.exports = config;

config/environments/production.js

@@ -1,35 +0,0 @@
-var config = {
-    environment: 'production'
-    ,port: 8181
-    ,host: '127.0.0.1'
-    ,enable_cors: true
-    ,cache_enabled: true
-    ,postgres_auth_user: 'cartodb_user_<%= user_id %>'
-    ,postgres: {
-        user: "publicuser",
-        host: '127.0.0.1',
-        port: 6432,
-        extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188",
-        simplify: true
-    }
-    ,millstone: {
-        cache_basedir: '/home/ubuntu/tile_assets/'
-    }
-    ,redis: {
-        host: '127.0.0.1',
-        port: 6379
-    }
-    ,sqlapi: {
-        protocol: 'https',
-        host: 'cartodb.com',
-        port: 8080,
-        version: 'v2'
-    }
-    ,varnish: {
-        host: 'localhost',
-        port: 6082,
-        ttl: 86400
-    }
-};
-
-module.exports = config;

config/environments/production.js.example

@@ -0,0 +1,164 @@
+var config = {
+    environment: 'production'
+    ,port: 8181
+    ,host: '127.0.0.1'
+    ,uv_threadpool_size: undefined
+    // Regular expression pattern to extract username
+    // from hostname. Must have a single grabbing block.
+    ,user_from_host: '^(.*)\\.cartodb\\.com$'
+
+    // Base URLs for the APIs
+    //
+    // See http://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API
+    // 
+    // Base url for the Templated Maps API
+    // "/api/v1/map/named" is the new API,
+    // "/tiles/template" is for compatibility with versions up to 1.6.x
+    ,base_url_templated: '(?:/api/v1/map/named|/tiles/template)'
+    // Base url for the Detached Maps API
+    // "maps" is the the new API,
+    // "tiles/layergroup" is for compatibility with versions up to 1.6.x
+    ,base_url_detached: '(?:/api/v1/map|/tiles/layergroup)'
+    // Base url for the Inline Maps and Table Maps API
+    ,base_url_legacy: '/tiles/:table'
+
+    // Maximum number of connections for one process
+    // 128 is a good value with a limit of 1024 open file descriptors
+    ,maxConnections:128
+    // Maximum number of templates per user. Unlimited by default.
+    ,maxUserTemplates:1024
+    // Seconds since "last creation" before a detached
+    // or template instance map expires. Or: how long do you want
+    // to be able to navigate the map without a reload ?
+    // Defaults to 7200 (2 hours)
+    ,mapConfigTTL: 7200
+    // idle socket timeout, in milliseconds
+    ,socket_timeout: 600000 
+    ,enable_cors: true
+    ,cache_enabled: true
+    ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
+    // If log_filename is given logs will be written
+    // there, in append mode. Otherwise stdout is used (default).
+    // Log file will be re-opened on receiving the HUP signal
+    ,log_filename: 'logs/node-windshaft.log'
+    // Templated database username for authorized user
+    // Supported labels: 'user_id' (read from redis)
+    ,postgres_auth_user: 'cartodb_user_<%= user_id %>'
+    // Templated database password for authorized user
+    // Supported labels: 'user_id', 'user_password' (both read from redis)
+    ,postgres_auth_pass: '<%= user_password %>'
+    ,postgres: {
+        // Parameters to pass to datasource plugin of mapnik
+        // See http://github.com/mapnik/mapnik/wiki/PostGIS
+        user: "publicuser",
+        password: "public",
+        host: '127.0.0.1',
+        port: 6432,
+        extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
+        row_limit: 65535,
+        /*
+         * Set persist_connection to false if you want
+         * database connections to be closed on renderer
+         * expiration (1 minute after last use).
+         * Setting to true (the default) would never
+         * close any connection for the server's lifetime
+         */
+        persist_connection: false,
+        simplify_geometries: true,
+        max_size: 500
+    }
+    ,mapnik_version: undefined
+    ,mapnik_tile_format: 'png8:m=h'
+    ,statsd: {
+        host: 'localhost',
+        port: 8125,
+        prefix: ':host.', // could be hostname, better not containing dots
+        cacheDns: true
+        // support all allowed node-statsd options
+    }
+    ,renderer: {
+      // Milliseconds since last access before renderer cache item expires
+      cache_ttl: 60000,
+      metatile: 4,
+      bufferSize: 64,
+      statsInterval: 5000 // milliseconds between each report to statsd about number of renderers and mapnik pool status
+    }
+    ,millstone: {
+        // Needs to be writable by server user
+        cache_basedir: '/home/ubuntu/tile_assets/'
+    }
+    ,redis: {
+        host: '127.0.0.1',
+        port: 6379,
+        // Max number of connections in each pool.
+        // Users will be put on a queue when the limit is hit.
+        // Set to maxConnection to have no possible queues.
+        // There are currently 2 pools involved in serving
+        // windshaft-cartodb requests so multiply this number
+        // by 2 to know how many possible connections will be
+        // kept open by the server. The default is 50.
+        max: 50,
+        returnToHead: true, // defines the behaviour of the pool: false => queue, true => stack
+        idleTimeoutMillis: 30000, // idle time before dropping connection
+        reapIntervalMillis: 1000, // time between cleanups
+        slowQueries: {
+            log: true,
+            elapsedThreshold: 200
+        },
+        slowPool: {
+            log: true, // whether a slow acquire must be logged or not
+            elapsedThreshold: 25 // the threshold to determine an slow acquire must be reported or not
+        },
+        emitter: {
+            statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd
+        }
+    }
+    ,sqlapi: {
+        protocol: 'https',
+        // If "host" is given, it will be used
+        // to connect to the SQL-API without a
+        // DNS lookup
+        //host: '127.0.0.1',
+        port: 8080,
+        // The "domain" part will be appended to 
+        // the cartodb username and passed to
+        // SQL-API requests in the Host HTTP header
+        domain: 'cartodb.com',
+        version: 'v2',
+        // Maximum lenght of SQL query for GET
+        // requests. Longer queries will be sent
+        // using POST. Defaults to 2048
+        max_get_sql_length: 2048,
+        // Maximum time to wait for a response,
+        // in milliseconds. Defaults to 100.
+        timeout: 100
+    }
+    ,varnish: {
+        host: 'localhost',
+        port: 6082,
+        secret: 'xxx',
+        ttl: 86400,
+        layergroupTtl: 86400 // the max-age for cache-control header in layergroup responses
+    }
+    // If useProfiler is true every response will be served with an
+    // X-Tiler-Profile header containing elapsed timing for various 
+    // steps taken for producing the response.
+    ,useProfiler:false
+    ,serverMetadata: {
+      cdn_url: {
+        http: 'api.cartocdn.com',
+        https: 'cartocdn.global.ssl.fastly.net'
+      }
+    }
+    // Optional rollbar support
+    ,rollbar: {
+      token: 'secret',
+      // See http://github.com/rollbar/node_rollbar#configuration-reference
+      options: {
+        endpoint: 'https://api.rollbar.com/api/1/',
+        handler: 'inline'
+      }
+    }
+};
+
+module.exports = config;

config/environments/staging.js

@@ -1,35 +0,0 @@
-var config = {
-    environment: 'production'
-    ,port: 8181
-    ,host: '127.0.0.1'
-    ,enable_cors: true
-    ,cache_enabled: true
-    ,postgres_auth_user: 'cartodb_staging_user_<%= user_id %>'
-    ,postgres: {
-        user: "publicuser",
-        host: '127.0.0.1',
-        port: 6432,
-        extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188",
-        simplify: true
-    }
-    ,millstone: {
-        cache_basedir: '/home/ubuntu/tile_assets/'
-    }
-    ,redis: {
-        host: '127.0.0.1',
-        port: 6379
-    }
-    ,sqlapi: {
-        protocol: 'https',
-        host: 'cartodb.com',
-        port: 8080,
-        version: 'v2'
-    }
-    ,varnish: {
-        host: 'localhost',
-        port: 6082,
-        ttl: 86400
-    }
-};
-
-module.exports = config;

config/environments/staging.js.example

@@ -0,0 +1,164 @@
+var config = {
+    environment: 'production'
+    ,port: 8181
+    ,host: '127.0.0.1'
+    ,uv_threadpool_size: undefined
+    // Regular expression pattern to extract username
+    // from hostname. Must have a single grabbing block.
+    ,user_from_host: '^(.*)\\.cartodb\\.com$'
+
+    // Base URLs for the APIs
+    //
+    // See http://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API
+    // 
+    // Base url for the Templated Maps API
+    // "/api/v1/maps/named" is the new API,
+    // "/tiles/template" is for compatibility with versions up to 1.6.x
+    ,base_url_templated: '(?:/api/v1/maps/named|/tiles/template)'
+    // Base url for the Detached Maps API
+    // "/api/v1/maps" is the the new API,
+    // "/tiles/layergroup" is for compatibility with versions up to 1.6.x
+    ,base_url_detached: '(?:/api/v1/maps|/tiles/layergroup)'
+    // Base url for the Inline Maps and Table Maps API
+    ,base_url_legacy: '/tiles/:table'
+
+    // Maximum number of connections for one process
+    // 128 is a good value with a limit of 1024 open file descriptors
+    ,maxConnections:128
+    // Maximum number of templates per user. Unlimited by default.
+    ,maxUserTemplates:1024
+    // Seconds since "last creation" before a detached
+    // or template instance map expires. Or: how long do you want
+    // to be able to navigate the map without a reload ?
+    // Defaults to 7200 (2 hours)
+    ,mapConfigTTL: 7200
+    // idle socket timeout, in milliseconds
+    ,socket_timeout: 600000 
+    ,enable_cors: true
+    ,cache_enabled: true
+    ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms (:res[X-Tiler-Profiler]) -> :res[Content-Type]'
+    // If log_filename is given logs will be written
+    // there, in append mode. Otherwise stdout is used (default).
+    // Log file will be re-opened on receiving the HUP signal
+    ,log_filename: 'logs/node-windshaft.log'
+    // Templated database username for authorized user
+    // Supported labels: 'user_id' (read from redis)
+    ,postgres_auth_user: 'cartodb_staging_user_<%= user_id %>'
+    // Templated database password for authorized user
+    // Supported labels: 'user_id', 'user_password' (both read from redis)
+    ,postgres_auth_pass: '<%= user_password %>'
+    ,postgres: {
+        // Parameters to pass to datasource plugin of mapnik
+        // See http://github.com/mapnik/mapnik/wiki/PostGIS
+        user: "publicuser",
+        password: "public",
+        host: '127.0.0.1',
+        port: 6432,
+        extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
+        row_limit: 65535,
+        simplify_geometries: true,
+        /*
+         * Set persist_connection to false if you want
+         * database connections to be closed on renderer
+         * expiration (1 minute after last use).
+         * Setting to true (the default) would never
+         * close any connection for the server's lifetime
+         */
+        persist_connection: false,
+        max_size: 500
+    }
+    ,mapnik_version: undefined
+    ,mapnik_tile_format: 'png8:m=h'
+    ,statsd: {
+        host: 'localhost',
+        port: 8125,
+        prefix: 'stage.:host.',
+        cacheDns: true
+        // support all allowed node-statsd options
+    }
+    ,renderer: {
+      // Milliseconds since last access before renderer cache item expires
+      cache_ttl: 60000,
+      metatile: 4,
+      bufferSize: 64,
+      statsInterval: 5000 // milliseconds between each report to statsd about number of renderers and mapnik pool status
+    }
+    ,millstone: {
+        // Needs to be writable by server user
+        cache_basedir: '/home/ubuntu/tile_assets/'
+    }
+    ,redis: {
+        host: '127.0.0.1',
+        port: 6379,
+        // Max number of connections in each pool.
+        // Users will be put on a queue when the limit is hit.
+        // Set to maxConnection to have no possible queues.
+        // There are currently 2 pools involved in serving
+        // windshaft-cartodb requests so multiply this number
+        // by 2 to know how many possible connections will be
+        // kept open by the server. The default is 50.
+        max: 50,
+        returnToHead: true, // defines the behaviour of the pool: false => queue, true => stack
+        idleTimeoutMillis: 30000, // idle time before dropping connection
+        reapIntervalMillis: 1000, // time between cleanups
+        slowQueries: {
+            log: true,
+            elapsedThreshold: 200
+        },
+        slowPool: {
+            log: true, // whether a slow acquire must be logged or not
+            elapsedThreshold: 25 // the threshold to determine an slow acquire must be reported or not
+        },
+        emitter: {
+            statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd
+        }
+    }
+    ,sqlapi: {
+        protocol: 'https',
+        // If "host" is given, it will be used
+        // to connect to the SQL-API without a
+        // DNS lookup
+        //host: '127.0.0.1',
+        port: 8080,
+        // The "domain" part will be appended to 
+        // the cartodb username and passed to
+        // SQL-API requests in the Host HTTP header
+        domain: 'cartodb.com',
+        version: 'v2',
+        // Maximum lenght of SQL query for GET
+        // requests. Longer queries will be sent
+        // using POST. Defaults to 2048
+        max_get_sql_length: 2048,
+        // Maximum time to wait for a response,
+        // in milliseconds. Defaults to 100.
+        timeout: 100
+    }
+    ,varnish: {
+        host: 'localhost',
+        port: 6082,
+        secret: 'xxx',
+        ttl: 86400,
+        layergroupTtl: 86400 // the max-age for cache-control header in layergroup responses
+    }
+    // If useProfiler is true every response will be served with an
+    // X-Tiler-Profile header containing elapsed timing for various 
+    // steps taken for producing the response.
+    ,useProfiler:true
+    ,serverMetadata: {
+      cdn_url: {
+        http: 'api.cartocdn.com',
+        https: 'cartocdn.global.ssl.fastly.net'
+      }
+    }
+    // Optional rollbar support
+    ,rollbar: {
+      token: 'secret',
+      // See http://github.com/rollbar/node_rollbar#configuration-reference
+      options: {
+        endpoint: 'https://api.rollbar.com/api/1/',
+        handler: 'inline'
+      }
+    }
+};
+
+module.exports = config;

config/environments/test.js

@@ -1,38 +0,0 @@
-var config = {
-     environment: 'test'
-    ,port: 8888
-    ,host: '127.0.0.1'
-    ,enable_cors: true
-    ,cache_enabled: false
-    ,postgres_auth_user: 'test_cartodb_user_<%= user_id %>'
-    ,postgres: {
-        user: "publicuser",
-        host: '127.0.0.1',
-        port: 5432,
-        srid: 4326,
-        extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188",
-        simplify: true
-    }
-    ,millstone: {
-        cache_basedir: '/tmp/cdb-tiler-test/millstone'
-    }
-    ,redis: {
-        host: '127.0.0.1',
-        port: 6333,
-        idleTimeoutMillis: 1,
-        reapIntervalMillis: 1
-    }
-    ,sqlapi: {
-        protocol: 'http',
-        host: 'localhost.lan',
-        port: 8080,
-        version: 'v1'
-    }
-    ,varnish: {
-        host: '',
-        port: null,
-        ttl: 86400
-    }
-};
-
-module.exports = config;

config/environments/test.js.example

@@ -0,0 +1,151 @@
+var config = {
+     environment: 'test'
+    ,port: 8888
+    ,host: '127.0.0.1'
+    ,uv_threadpool_size: undefined
+    // Regular expression pattern to extract username
+    // from hostname. Must have a single grabbing block.
+    ,user_from_host: '(.*)'
+
+    // Base URLs for the APIs
+    //
+    // See https://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API
+    // 
+    // Base url for the Templated Maps API
+    // "/api/v1/map/named" is the new API,
+    // "/tiles/template" is for compatibility with versions up to 1.6.x
+    ,base_url_templated: '(?:/api/v1/map/named|/tiles/template)'
+    // Base url for the Detached Maps API
+    // "maps" is the the new API,
+    // "tiles/layergroup" is for compatibility with versions up to 1.6.x
+    ,base_url_detached: '(?:/api/v1/map|/tiles/layergroup)'
+    // Base url for the Inline Maps and Table Maps API
+    ,base_url_legacy: '/tiles/:table'
+
+    // Maximum number of connections for one process
+    // 128 is a good value with a limit of 1024 open file descriptors
+    ,maxConnections:128
+    // Maximum number of templates per user. Unlimited by default.
+    ,maxUserTemplates:1024
+    // Seconds since "last creation" before a detached
+    // or template instance map expires. Or: how long do you want
+    // to be able to navigate the map without a reload ?
+    // Defaults to 7200 (2 hours)
+    ,mapConfigTTL: 7200
+    // idle socket timeout, in milliseconds
+    ,socket_timeout: 600000 
+    ,enable_cors: true
+    ,cache_enabled: false
+    ,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
+    // If log_filename is given logs will be written
+    // there, in append mode. Otherwise stdout is used (default).
+    // Log file will be re-opened on receiving the HUP signal
+    //,log_filename: 'logs/node-windshaft.log'
+    // Templated database username for authorized user
+    // Supported labels: 'user_id' (read from redis)
+    ,postgres_auth_user: 'test_windshaft_cartodb_user_<%= user_id %>'
+    // Templated database password for authorized user
+    // Supported labels: 'user_id', 'user_password' (both read from redis)
+    ,postgres_auth_pass: 'test_windshaft_cartodb_user_<%= user_id %>_pass'
+    ,postgres: {
+        // Parameters to pass to datasource plugin of mapnik
+        // See http://github.com/mapnik/mapnik/wiki/PostGIS
+        user: "testpublicuser",
+        password: "public",
+        host: '127.0.0.1',
+        port: 5432,
+        extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
+        row_limit: 65535,
+        simplify_geometries: true,
+        /*
+         * Set persist_connection to false if you want
+         * database connections to be closed on renderer
+         * expiration (1 minute after last use).
+         * Setting to true (the default) would never
+         * close any connection for the server's lifetime
+         */
+        persist_connection: false,
+        max_size: 500
+    }
+    ,mapnik_version: ''
+    ,mapnik_tile_format: 'png8:m=h'
+    ,statsd: {
+        host: 'localhost',
+        port: 8125,
+        prefix: 'test.:host.',
+        cacheDns: true
+        // support all allowed node-statsd options
+    }
+    ,renderer: {
+      // Milliseconds since last access before renderer cache item expires
+      cache_ttl: 60000,
+      metatile: 4,
+      bufferSize: 64,
+      statsInterval: 5000 // milliseconds between each report to statsd about number of renderers and mapnik pool status
+    }
+    ,millstone: {
+        // Needs to be writable by server user
+        cache_basedir: '/tmp/cdb-tiler-test/millstone'
+    }
+    ,redis: {
+        host: '127.0.0.1',
+        port: 6335,
+        // Max number of connections in each pool.
+        // Users will be put on a queue when the limit is hit.
+        // Set to maxConnection to have no possible queues.
+        // There are currently 2 pools involved in serving
+        // windshaft-cartodb requests so multiply this number
+        // by 2 to know how many possible connections will be
+        // kept open by the server. The default is 50.
+        max: 50,
+        returnToHead: true, // defines the behaviour of the pool: false => queue, true => stack
+        idleTimeoutMillis: 1, // idle time before dropping connection
+        reapIntervalMillis: 1, // time between cleanups
+        slowQueries: {
+            log: true,
+            elapsedThreshold: 200
+        },
+        slowPool: {
+            log: true, // whether a slow acquire must be logged or not
+            elapsedThreshold: 25 // the threshold to determine an slow acquire must be reported or not
+        },
+        emitter: {
+            statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd
+        }
+    }
+    ,sqlapi: {
+        protocol: 'http',
+        // If "host" is given, it will be used
+        // to connect to the SQL-API without a
+        // DNS lookup
+        host: '127.0.0.1',
+        port: 1080,
+        // The "domain" part will be appended to 
+        // the cartodb username and passed to
+        // SQL-API requests in the Host HTTP header
+        domain: 'donot_look_this_up',
+        // This port will be used by "make check" for testing purposes
+        // It must be available
+        version: 'v1',
+        // Maximum lenght of SQL query for GET
+        // requests. Longer queries will be sent
+        // using POST. Defaults to 2048
+        max_get_sql_length: 2048,
+        // Maximum time to wait for a response,
+        // in milliseconds. Defaults to 100.
+        timeout: 100
+    }
+    ,varnish: {
+        host: '',
+        port: null,
+        secret: 'xxx',
+        ttl: 86400,
+        layergroupTtl: 86400 // the max-age for cache-control header in layergroup responses
+    }
+    // If useProfiler is true every response will be served with an
+    // X-Tiler-Profile header containing elapsed timing for various 
+    // steps taken for producing the response.
+    ,useProfiler:true
+};
+
+module.exports = config;

config/settings.js

@@ -1 +0,0 @@
-module.exports.oneDay   = 86400000;

configure

@@ -0,0 +1,92 @@
+#!/bin/sh
+
+#
+# This script creates config/environments/*.js files using
+# config/environments/*.js.example files as input and performing
+# settings substitutions.
+#
+# It relies on a known format of the .js.example files which haven't
+# been made easier to parse to still let humans copy them manually and
+# do further editing or leave them as such to get the same setup as before
+# the introduction of this script.
+#
+# The script is a work in progress. Available switches are printed
+# by invoking with the --help switch. More switches will be added
+# as the need/request for them arises.
+#
+# --strk(2012-07-23)
+#
+
+ENVDIR=config/environments
+
+PGPORT=
+SQLAPI_PORT=
+MAPNIK_VERSION=
+ENVIRONMENT=development
+
+STATUS="$0 $*"
+
+usage() {
+  echo "Usage: $0 [OPTION]"
+  echo 
+  echo "Configuration:"
+  echo "  --help                        display this help and exit"
+  echo "  --with-pgport=NUM             access PostgreSQL server on TCP port NUM [$PGPORT]"
+  echo "  --with-sqlapi-port=NUM        access SQL-API server on TCP port NUM [$SQLAPI_PORT]"
+  echo "  --with-mapnik-version=STRING  set mapnik version string [$MAPNIK_VERSION]"
+  echo "  --environment=STRING          set output environment name [$ENVIRONMENT]"
+}
+
+while test -n "$1"; do
+  case "$1" in
+    --help|-h)
+      usage
+      exit 0
+      ;;
+    --with-pgport=*)
+      PGPORT=`echo "$1" | cut -d= -f2`
+      ;;
+    --with-sqlapi-port=*)
+      SQLAPI_PORT=`echo "$1" | cut -d= -f2`
+      ;;
+    --with-mapnik-version=*)
+      MAPNIK_VERSION=`echo "$1" | cut -d= -f2`
+      ;;
+    --environment=*)
+      ENVIRONMENT=`echo "$1" | cut -d= -f2`
+      ;;
+    *)
+      echo "Unknown option '$1'" >&2
+      usage >&2
+      exit 1
+  esac
+  shift
+done
+
+ENVEX=./${ENVDIR}/${ENVIRONMENT}.js.example
+
+if [ -z "$PGPORT" ]; then
+  PGPORT=`node -e "console.log(require('${ENVEX}').postgres.port)"`
+fi
+if [ -z "$SQLAPI_PORT" ]; then
+  SQLAPI_PORT=`node -e "console.log(require('${ENVEX}').sqlapi.port)"`
+fi
+
+echo "PGPORT: $PGPORT"
+echo "SQLAPI_PORT: $SQLAPI_PORT"
+echo "MAPNIK_VERSION: $MAPNIK_VERSION"
+echo "ENVIRONMENT: $ENVIRONMENT"
+
+o=`dirname "${ENVEX}"`/`basename "${ENVEX}" .example`
+echo "Writing $o"
+
+# See http://austinmatzko.com/2008/04/26/sed-multi-line-search-and-replace/
+sed -n "1h;1!H;\${;g;s/\(,postgres: {[^}]*port: *'\?\)[^',]*\('\?,\)/\1$PGPORT\2/;p;}" < "${ENVEX}" \
+  | sed "s/mapnik_version:.*/mapnik_version: '$MAPNIK_VERSION'/" \
+  | sed -n "1h;1!H;\${;g;s/\(,sqlapi: {[^}]*port: *'\?\)[^',]*\('\?,\)/\1$SQLAPI_PORT\2/;p;}" \
+  > "$o"
+
+STATUSFILE=config.status--${ENVIRONMENT}
+echo "Writing ${STATUSFILE}"
+echo ${STATUS} > ${STATUSFILE} && chmod +x ${STATUSFILE}
+

docs/Map-API-internal.md

@@ -0,0 +1,111 @@
+# Kind of maps
+
+Windshaft-CartoDB supports these kind of maps:
+
+ - [Temporary maps](#temporary-maps) (created by anyone)
+   - [Detached maps](#detached-maps)
+   - [Inline maps](#inline-maps) (legacy)
+ - [Persistent maps](#peristent-maps) (created by CartDB user)
+   - [Template maps](#template-maps)
+   - [Table maps](#table-maps) (legacy, deprecated) 
+
+## Temporary maps
+
+Temporary maps have no owners and are anonymous in nature.
+There are two kind of temporary maps:
+
+ - Detached maps (aka MultiLayer-API)
+ - Inline maps 
+
+### Detached maps
+
+Detached maps are maps which are configured with a request
+obtaining a temporary token and then used by referencing
+the obtained token. The token expires automatically when unused.
+
+Anyone can create detached maps, but users will need read access
+to the data source of the map layers.
+
+The configuration format is a [MapConfig]
+(http://github.com/CartoDB/Windshaft/wiki/MapConfig-specification) document.
+
+The HTTP endpoints for creating the map and using it are described [here]
+(http://github.com/CartoDB/Windshaft-cartodb/wiki/MultiLayer-API)
+
+*TODO* cleanup the referenced document
+
+### Inline maps
+
+Inline maps are maps that only exist for a single request,
+being the request for a specific map resource (tile).
+
+Inline maps are always bound to a table, and can only be
+obtained by those having read access to the that table.
+Additionally, users need to have access to any datasource
+specified as part of the configuration.
+
+Inline maps only support PNG and UTF8GRID tiles.
+
+The configuration consist in a set of parameters, to be
+specified in the query string of the tile request:
+
+ * sql - the query to run as datasource, can be an array
+ * style - the CartoCSS style for the datasource, can be an array
+ * style_version - version of the CartoCSS style, can be an array
+ * interactivity - only for fetching UTF8GRID, 
+
+If the style is not provided, style of the associated table is
+used; if the sql is not provided, all records of the associated
+table are used as the datasource; the two possibilities result
+in a mix between _inline_ maps and [Table maps][].
+
+*TODO* specify (or link) api endpoints
+
+## Persistent maps
+
+Persistent maps can only be created by a CartoDB user who has full
+responsibility over editing and deleting them. There are two
+kind of persistent maps:
+
+ - Template maps 
+ - Table maps (legacy, deprecated) 
+
+### Templated maps
+
+Templated maps are templated [MapConfig]
+(http://github.com/CartoDB/Windshaft/wiki/MapConfig-specification) documents
+associated with an authorization certificate.
+
+The authorization certificate determines who can instanciate the
+template and use the resulting map. Authorized users of the instanciated
+maps will have the same database access privilege of the template owner.
+
+The HTTP endpoints for creating and using templated maps are described [here]
+(http://github.com/CartoDB/Windshaft-cartodb/wiki/Template-maps).
+
+*TODO* cleanup the referenced document
+
+### Table maps
+
+Table maps are maps associated with a table.
+Configuration of such maps is limited to the CartoCSS style.
+
+ * style - the CartoCSS style for the datasource, can be an array
+ * style_version - version of the CartoCSS style, can be an array
+
+You can only fetch PNG or UTF8GRID tiles from these maps.
+
+Access method is the same as the one for [Inline maps](#inline-maps)
+
+# Endpoints description 
+
+- **/api/maps/** (same interface than https://github.com/CartoDB/Windshaft/wiki/Multilayer-API)
+- **/api/maps/named** (same interface than https://github.com/CartoDB/Windshaft-cartodb/wiki/Template-maps)
+
+
+NOTE: in case Multilayer-API does not contain this info yet, the
+      endpoint for fetching attributes is this:
+
+- **/api/maps/:map_id/:layer_index/attributes/:feature_id** 
+   - would return { c: 1, d: 2 }
+

docs/Map-API.md

@@ -0,0 +1,637 @@
+## Maps API
+
+The CartoDB Maps API allows you to generate maps based on data hosted in your CartoDB account and style them using CartoCSS. The API generates a XYZ based URL to fetch Web Mercator projected tiles using web clients like Leaflet, Google Maps, OpenLayers.
+
+You can create two types of maps with the Maps API:
+
+- **Anonymous maps**  
+  Maps that can be created using your CartoDB public data. Any client can change the read-only SQL and CartoCSS parameters that generate the map tiles. These maps can be created from a JavaScript application alone and no authenticated calls are needed. See [this CartoDB.js example]({{ '/cartodb-platform/cartodb-js.html' | prepend: site.baseurl }}).
+
+- **Named maps**  
+  Maps that access to your private data. These maps require an owner to setup and modify any SQL and CartoCSS parameters and are not modifiable without new setup calls. 
+
+## Quickstart
+
+### Anonymous maps
+
+Here is an example of how to create an anonymous map with JavaScript:
+
+{% highlight javascript %}
+var mapconfig = {
+  "version": "1.0.1",
+  "layers": [{
+    "type": "cartodb",
+    "options": {
+      "cartocss_version": "2.1.1",
+      "cartocss": "#layer { polygon-fill: #FFF; }",
+      "sql": "select * from european_countries_e"
+    }
+  }]
+}
+
+$.ajax({
+  crossOrigin: true,
+  type: 'POST',
+  dataType: 'json',
+  contentType: 'application/json',
+  url: 'http://documentation.cartodb.com/api/v1/map',
+  data: JSON.stringify(mapconfig),
+  success: function(data) {
+    var templateUrl = 'http://documentation.cartodb.com/api/v1/map/' + data.layergroupid + '{z}/{x}/{y}.png'
+    console.log(templateUrl);
+  }
+})
+{% endhighlight %}
+
+### Named maps
+
+Let's create a named map using some private tables in a CartoDB account.
+The following API call creates a map of European countries that have a white fill color:
+
+{% highlight javascript %}
+// mapconfig.json
+{
+  "version": "0.0.1",
+  "name": "test",
+  "auth": {
+    "method": "open"
+  },
+  "layergroup": {
+    "layers": [{
+      "type": "cartodb",
+      "options": {
+        "cartocss_version": "2.1.1",
+        "cartocss": "#layer { polygon-fill: #FFF; }",
+        "sql": "select * from european_countries_e"
+      }
+    }]
+  }
+}
+{% endhighlight %}
+
+The map config needs to be sent to CartoDB's Map API using an authenticated call. Here we use a command line tool called `curl`. For more info about this tool see [this blog post](http://quickleft.com/blog/command-line-tutorials-curl) or type ``man curl`` in bash. Using `curl` the call would look like:
+
+<div class="code-title notitle code-request"></div>
+{% highlight bash %}
+curl 'https://{account}.cartodb.com/api/v1/map/named?api_key=APIKEY' -H 'Content-Type: application/json' -d @mapconfig.json
+{% endhighlight %}
+
+To get the `URL` to fetch the tiles you need to instantiate the map.
+
+<div class="code-title notitle code-request"></div>
+{% highlight bash %}
+curl 'http://{account}.cartodb.com/api/v1/map/named/test' -H 'Content-Type: application/json'
+{% endhighlight %}
+
+The response will return JSON with properties for the `layergroupid` and the timestamp (`last_updated`) of the last data modification. 
+
+Here is an example response:
+
+{% highlight javascript %}
+{
+  "layergroupid": "c01a54877c62831bb51720263f91fb33:0",
+  "last_updated": "1970-01-01T00:00:00.000Z"
+}
+{% endhighlight %}
+
+You can use the `layergroupid` to instantiate a URL template for accessing tiles on the client. Here we use the `layergroupid` from the example response above in this URL template:
+
+{% highlight bash %}
+http://documentation.cartodb.com/api/v1/map/c01a54877c62831bb51720263f91fb33:0/{z}/{x}/{y}.png
+{% endhighlight %}
+
+## General Concepts
+
+The following concepts are the same for every endpoint in the API except when it's noted explicitly.
+
+### Auth
+
+By default, users do not have access to private tables in CartoDB. In order to instantiate a map from private table data an API Key is required. Additionally, to include some endpoints an API Key must be included (e.g. creating a named map).
+
+To execute an authorized request, api_key=YOURAPIKEY should be added to the request URL. The param can be also passed as POST param. We **strongly advise** using HTTPS when you are performing requests that include your `api_key`.
+
+### Errors
+
+Errors are reported using standard HTTP codes and extended information encoded in JSON with this format:
+
+{% highlight javascript %}
+{
+  "errors": [
+    "access forbidden to table TABLE"
+  ]
+}
+{% endhighlight %}
+
+If you use JSONP, the 200 HTTP code is always returned so the JavaScript client can receive errors from the JSON object.
+
+### CORS support
+
+All the endpoints which might be accessed using a web browser add CORS headers and allow OPTIONS method.
+
+## Anonymous Maps
+
+Anonymous maps allows you to instantiate a map given SQL and CartoCSS. It also allows you to add interaction capabilities using [UTF Grid.](https://github.com/mapbox/utfgrid-spec)
+
+### Instantiate
+
+#### Definition
+
+<div class="code-title notitle code-request"></div>
+{% highlight html %}
+POST /api/v1/map
+{% endhighlight %}
+
+#### Params
+
+{% highlight javascript %}
+{
+  "version": "1.0.1",
+  "layers": [{
+    "type": "cartodb",
+    "options": {
+      "cartocss_version": "2.1.1", 
+      "cartocss": "#layer { polygon-fill: #FFF; }",
+      "sql": "select * from european_countries_e",
+      "interactivity": ["cartodb_id", "iso3"]
+    }
+  }]
+}
+{% endhighlight %}
+
+Should be a [Mapconfig](https://github.com/CartoDB/Windshaft/blob/0.19.1/doc/MapConfig-1.1.0.md).
+
+#### Response
+
+The response includes:
+
+- **layergroupid**  
+  The ID for that map, used to compose the URL for the tiles. The final URL is:
+
+  {% highlight html %}
+  http://{account}.cartodb.com/api/v1/map/:layergroupid/{z}/{x}/{y}.png
+  {% endhighlight %}
+
+- **updated_at**  
+  The ISO date of the last time the data involved in the query was updated.
+
+- **metadata** *(optional)*  
+  Includes information about the layers. Some layers may not have metadata.
+
+- **cdn_url**  
+  URLs to fetch the data using the best CDN for your zone.
+
+#### Example
+
+<div class="code-title code-request with-result">REQUEST</div>
+{% highlight bash %}
+curl 'http://documentation.cartodb.com/api/v1/map' -H 'Content-Type: application/json' -d @mapconfig.json
+{% endhighlight %}
+
+<div class="code-title">RESPONSE</div>
+{% highlight javascript %}
+{
+  "layergroupid":"c01a54877c62831bb51720263f91fb33:0",
+  "last_updated":"1970-01-01T00:00:00.000Z"
+  "cdn_url": {
+    "http": "http://cdb.com",
+    "https": "https://cdb.com"
+  }
+}
+{% endhighlight %}
+
+The tiles can be accessed using:
+
+{% highlight bash %}
+http://documentation.cartodb.com/api/v1/map/c01a54877c62831bb51720263f91fb33:0/{z}/{x}/{y}.png
+{% endhighlight %}
+
+For UTF grid tiles:
+
+{% highlight bash %}
+http://documentation.cartodb.com/api/v1/map/c01a54877c62831bb51720263f91fb33:0/:layer/{z}/{x}/{y}.grid.json
+{% endhighlight %}
+
+For attributes defined in `attributes` section:
+
+{% highlight bash %}
+http://documentation.cartodb.com/api/v1/map/c01a54877c62831bb51720263f91fb33:0/:layer/attributes/:feature_id
+{% endhighlight %}
+
+Which returns JSON with the attributes defined, like:
+
+{% highlight javascript %}
+{ c: 1, d: 2 }
+{% endhighlight %}
+
+Notice UTF Grid and attributes endpoints need an intenger parameter, ``layer``. That number is the 0-based index of the layer inside the mapconfig. So in this case 0 returns the UTF grid tiles/attributes for layer 0, the only layer in the example mapconfig. If a second layer was available it could be returned with 1, a third layer with 2, etc.
+
+### Create JSONP
+
+The JSONP endpoint is provided in order to allow web browsers access which don't support CORS.
+
+#### Definition
+
+<div class="code-title notitle code-request"></div>
+{% highlight bash %}
+GET /api/v1/map?callback=method
+{% endhighlight %}
+
+#### Params
+
+- **auth_token** *(optional)*  
+  If the named map needs authorization.
+
+- **config**  
+  Encoded JSON with the params for creating named maps (the variables defined in the template).
+
+- **lmza**  
+  This attribute contains the same as config but LZMA compressed. It cannot be used at the same time as `config`.
+
+- **callback**  
+  JSON callback name.
+
+#### Example
+
+<div class="code-title code-request with-result">REQUEST</div>
+{% highlight bash %}
+curl http://...
+{% endhighlight %}
+
+<div class="code-title">RESPONSE</div>
+{% highlight javascript %}
+{
+}
+{% endhighlight %}
+
+### Remove
+
+Anonymous maps cannot be removed by an API call. They will expire after about five minutes but sometimes longer. If an anonymous map expires and tiles are requested from it, an error will be raised. This could happen if a user leaves a map open and after time returns to the map an attempts to interact with it in a way that requires new tiles (e.g. zoom). The client will need to go through the steps of creating the map again to fix the problem.
+
+## Named Maps
+
+Named maps are essentially the same as anonymous maps but the mapconfig is stored in the server and given a unique name. Two other big differences are that you can created named maps from private data and that users without an API Key can see them even though they are from that private data. 
+
+The main two differences compared to anonymous maps are:
+
+- **auth layer**  
+  This allows you to control who is able to see the map based on a token auth
+
+- **templates**  
+  Since the mapconfig is static it can contain some variables so the client con modify the map appearance using those variables.
+
+Template maps are persistent with no preset expiration. They can only be created or deleted by a CartoDB user with a valid API_KEY (see auth section).
+
+### Create
+
+#### Definition
+
+<div class="code-title notitle code-request"></div>
+{% highlight html %}
+POST /api/v1/map/named
+{% endhighlight %}
+
+#### Params
+
+<div class="code-title">template.json</div>
+{% highlight javascript %}
+{
+  "version": "0.0.1",
+  "name": "template_name",
+  "auth": {
+    "method": "token",
+    "valid_tokens": [
+      "auth_token1",
+      "auth_token2"
+    ]
+  },
+  "placeholders": {
+    "color": {
+      "type": "css_color",
+      "default": "red"
+    },
+    "cartodb_id": {
+      "type": "number",
+      "default": 1
+    }
+  },
+  "layergroup": {
+    "version": "1.0.1",
+    "layers": [
+      {
+        "type": "cartodb",
+        "options": {
+          "cartocss_version": "2.1.1",
+          "cartocss": "#layer { polygon-fill: <%= color %>; }",
+          "sql": "select * from european_countries_e WHERE cartodb_id = <%= cartodb_id %>"
+        }
+      }
+    ]
+  }
+}
+{% endhighlight %}
+
+##### Arguments
+
+- **name**: there can be at most 1 template with the same name for any user valid names start with a letter and only contains letter, numbers or underscores
+- **auth**:
+  - **method** `"token"` or `"open"` (the default if no `"method"` is given)
+- **placeholders**: Variables not listed here are not substituted. Variable not provided at instantiation time trigger an error. A default is required for optional variables. Type specification is used for quoting, to avoid injections see template format section below.
+- **layergroup**: the layer list definition. This is the MapConfig explained in anonymous maps see https://github.com/CartoDB/Windshaft/blob/master/doc/MapConfig-1.1.0.md
+
+#### Template Format
+
+A templated `layergroup` allows using placeholders in the "cartocss" and "sql" elements of the "option" object in any "layer" of a layergroup configuration
+
+Valid placeholder names start with a letter and can only contain letters, numbers or underscores. They have to be written between `<%=` and `%>` strings in order to be replaced.
+
+##### Example
+
+{% highlight javascript %}
+<%= my_color %>
+{% endhighlight %}
+
+The set of supported placeholders for a template will need to be explicitly defined with a specific type and default value for each.
+
+#### Placeholder Types
+
+The placeholder type will determine the kind of escaping for the associated value. Supported types are:
+
+- **sql_literal** internal single-quotes will be sql-escaped
+- **sql_ident** internal double-quotes will be sql-escaped
+- **number** can only contain numerical representation
+- **css_color** can only contain color names or hex-values
+
+Placeholder default values will be used whenever new values are not provided as options at the time of creation on the client. They can also be used to test the template by creating a default version with now options provided.
+
+When using templates, be very careful about your selections as they can give broad access to your data if they are defined losely.
+
+<div class="code-title code-request with-result">REQUEST</div>
+{% highlight html %}
+curl -X POST \
+   -H 'Content-Type: application/json' \
+   -d @template.json \
+   'https://documentation.cartodb.com/api/v1/map/named?api_key=APIKEY'
+{% endhighlight %}
+
+<div class="code-title">RESPONSE</div>
+{% highlight javascript %}
+{
+  "templateid":"name",
+}
+{% endhighlight %}
+
+### Instantiate
+
+Instantiating a map allows you to get the information needed to fetch tiles. That temporal map is an anonymous map.
+
+#### Definition
+
+<div class="code-title notitle code-request"></div>
+{% highlight html %}
+POST /api/v1/map/named/:template_name
+{% endhighlight %}
+
+#### Param
+
+{% highlight javascript %}
+// params.json
+{
+ color: "#ff0000",
+ cartodb_id: 3
+}
+{% endhighlight %}
+
+The fields you pass as `params.json` depend on the variables allowed by the named map. If there are variables missing it will raise an error (HTTP 400)
+
+- **auth_token** *optional* if the named map needs auth
+
+#### Example
+
+You can initialize a template map by passing all of the required parameters in a POST to `/api/v1/map/named/:template_name`.
+
+Valid credentials will be needed if required by the template.
+
+<div class="code-title code-request with-result">REQUEST</div>
+{% highlight bash %}
+curl -X POST \
+  -H 'Content-Type: application/json' \
+  -d @params.json \
+  'https://documentation.cartodb.com/api/v1/map/named/@template_name?auth_token=AUTH_TOKEN'
+{% endhighlight %}
+
+<div class="code-title">Response</div>
+{% highlight javascript %}
+{
+  "layergroupid": "docs@fd2861af@c01a54877c62831bb51720263f91fb33:123456788",
+  "last_updated": "2013-11-14T11:20:15.000Z"
+}
+{% endhighlight %}
+
+<div class="code-title">Error</div>
+{% highlight javascript %}
+{
+  "error": "Some error string here"
+}
+{% endhighlight %}
+
+You can then use the `layergroupid` for fetching tiles and grids as you would normally (see anonymous map section).  However, you'll need to show the `auth_token`, if required by the template.
+
+### Using JSONP
+
+There is also a special endpoint to be able to initialize a map using JSONP (for old browsers).
+
+#### Definition
+
+<div class="code-title notitle code-request"></div>
+{% highlight bash %}
+GET /api/v1/map/named/:template_name/jsonp
+{% endhighlight %}
+
+#### Params
+
+- **auth_token** *(optional)* If the named map needs auth
+- **config** Encoded JSON with the params for creating named maps (the variables defined in the template)
+- **lmza** This attribute contains the same as config but LZMA compressed. It cannot be used at the same time than `config`.
+- **callback:** JSON callback name
+
+<div class="code-title code-request with-result">REQUEST</div>
+{% highlight bash %}
+curl 'https://documentation.cartodb.com/api/v1/map/named/:template_name/jsonp?auth_token=AUTH_TOKEN&callback=callback&config=template_params_json'
+{% endhighlight %}
+
+<div class="code-title">RESPONSE</div>
+{% highlight javascript %}
+callback({
+  "layergroupid":"c01a54877c62831bb51720263f91fb33:0",
+  "last_updated":"1970-01-01T00:00:00.000Z"
+  "cdn_url": {
+    "http": "http://cdb.com",
+    "https": "https://cdb.com"
+  }
+})
+{% endhighlight %}
+
+This takes the `callback` function (required), `auth_token` if the template needs auth, and `config` which is the variable for the template (in cases where it has variables). 
+
+{% highlight javascript %}
+url += "config=" + encodeURIComponent(
+JSON.stringify({ color: 'red' });
+{% endhighlight %}
+
+The response is in this format:
+
+{% highlight javascript %}
+callback({
+  layergroupid: "dev@744bd0ed9b047f953fae673d56a47b4d:1390844463021.1401",
+  last_updated: "2014-01-27T17:41:03.021Z"
+})
+{% endhighlight %}
+
+### Update
+
+#### Definition
+
+<div class="code-title notitle code-request"></div>
+{% highlight bash %}
+PUT /api/v1/map/named/:template_name
+{% endhighlight %}
+
+#### Params
+
+Same params used to create a map.
+
+#### Response
+
+Same as updating a map.
+
+#### Other Info
+
+Updating a named map removes all the named map instances so they need to be initialized again.
+
+#### Example
+
+<div class="code-title code-request with-result">REQUEST</div>
+{% highlight bash %}
+curl -X PUT \
+  -H 'Content-Type: application/json' \
+  -d @template.json \
+  'https://documentation.cartodb.com/api/v1/map/named/:template_name?api_key=APIKEY'
+{% endhighlight %}
+
+<div class="code-title">RESPONSE</div>
+{% highlight javascript %}
+{
+  "template_id": "@template_name"
+}
+{% endhighlight %}
+
+If any template has the same name, it will be updated.
+
+If a template with the same name does NOT exist, a 400 HTTP response is generated with an error in this format:
+
+{% highlight javascript %}
+{
+  "error": "error string here"
+}
+{% endhighlight %}
+
+Updating a template map will also remove all signatures from previously initialized maps. 
+
+### Delete 
+
+Delete the specified template map from the server and disables any previously initialized versions of the map.
+
+#### Definition
+
+<div class="code-title notitle code-request"></div>
+{% highlight bash %}
+DELETE /api/v1/map/named/:template_name
+{% endhighlight %}
+
+#### Example
+
+<div class="code-title code-request">REQUEST</div>
+{% highlight bash %}
+curl -X DELETE 'https://documentation.cartodb.com/api/v1/map/named/:template_name?auth_token=AUTH_TOKEN'
+{% endhighlight %}
+
+<div class="code-title">RESPONSE</div>
+{% highlight javascript %}
+{
+  "error": "Some error string here"
+}
+{% endhighlight %}
+
+On success, a 204 (No Content) response would be issued. Otherwise a 4xx response with with an error will be returned:
+
+### Listing Available Templates
+
+This allows you to get a list of all available templates. 
+
+#### Definition
+
+<div class="code-title notitle code-request"></div>
+{% highlight bash %}
+GET /api/v1/map/named/
+{% endhighlight %}
+
+#### Params
+
+- **api_key** is required
+
+#### Example
+
+<div class="code-title code-request with-result">REQUEST</div>
+{% highlight bash %}
+curl -X GET 'https://documentation.cartodb.com/api/v1/map/named?api_key=APIKEY'
+{% endhighlight %}
+
+<div class="code-title with-result">RESPONSE</div>
+{% highlight javascript %}
+{
+   "template_ids": ["@template_name1","@template_name2"]
+}
+{% endhighlight %}
+
+<div class="code-title">ERROR</div>
+{% highlight javascript %}
+{
+   "error": "Some error string here"
+}
+{% endhighlight %}
+
+### Getting a Specific Template
+
+This gets the definition of a template
+
+#### Definition
+
+<div class="code-title notitle code-request"></div>
+{% highlight bash %}
+GET /api/v1/map/named/:template_name
+{% endhighlight %}
+
+#### Params
+
+- **api_key** is required
+
+#### Example
+
+<div class="code-title code-request with-result">REQUEST</div>
+{% highlight bash %}
+curl -X GET 'https://documentation.cartodb.com/api/v1/map/named/:template_name?auth_token=AUTH_TOKEN'
+{% endhighlight %}
+
+<div class="code-title with-result">RESPONSE</div>
+{% highlight javascript %}
+{
+  "template": {...} // see template.json above
+}
+{% endhighlight %}
+
+<div class="code-title">ERROR</div>
+{% highlight javascript %}
+{
+  "error": "Some error string here"
+}
+{% endhighlight %}

docs/MultiLayer-API.md

@@ -0,0 +1,28 @@
+The Windshaft-CartoDB MultiLayer API extends the [Windshaft MultiLayer API](https://github.com/Vizzuality/Windshaft/wiki/Multilayer-API) in a few ways.
+
+## Last modification timestamp embedded in the token
+
+It encodes a timestamp of 'last modification time' into the map token (token:EPOCH) returned to the client.
+It accepts tokens with encoded timestamp from the client considering the token suffix as a cache_buster value.
+
+Clients don't need to be aware of the extension but rather use the API as they would use the base one.
+The only difference will be that the _same_ layergroup configuration may result in different tokens if source data was modified between the mapview requests. 
+
+## Additional attributes in the response object
+
+Windshaft-CartoDB adds the following attributes in the response object
+
+- ``last_update`` field with ISO format (2013-11-30T12:23:10).
+- ``cdn_url`` object containing CDN url client should use (not mandatory) to access the tiles. It's in the form:
+
+   ```json
+   {
+     http: 'http://cdn_url.com/'
+     https: 'https://secure.cdn_url.com/'
+   }
+   ```
+
+
+## Stats tag
+
+Windshaft-CartoDB adds support for a ``stat_tag`` element in the multilayer configuration to help [stats](Redis-stats-format) gathering.

docs/Template-maps.md

@@ -0,0 +1,293 @@
+Template maps are layergroup configurations that rather than being
+fully defined contain variables that can be set to produce a different
+layergroup configurations (instantiation).
+
+Template maps are persistent, can only be created and deleted by the
+CartoDB user showing a valid API_KEY.
+
+Instantiating a signed template map would result in a [signed
+map](https://github.com/CartoDB/Windshaft-cartodb/wiki/Signed-maps)
+instance that would be signed with the same signature as the template.
+
+Deleting a signed template results in deletion of all signatures created
+as a result of instantiation.
+
+
+# Template format
+
+A templated layergroup would allow using placeholders
+in the "cartocss" and "sql" elements in the "option"
+field of any "layer" of a layergroup configuration
+(see https://github.com/CartoDB/Windshaft/wiki/MapConfig-specification).
+
+Valid placeholder names start with a letter and can only
+contain letters, numbers or underscores. They have to be
+written between ``<%= `` and `` %>`` strings in order to be 
+replaced. Example: ``<%= my_color %>``.
+
+The set of supported placeholders for a template will need to be
+explicitly defined specifying type and default value for each.
+
+**placeholder types**
+
+Placeholder type will determine the kind of escaping for the
+associated value. Supported types are:
+
+ * sql_literal (internal single-quotes will be sql-escaped)
+ * sql_ident (internal double-quotes will be sql-escaped)
+ * number (can only contain numerical representation)
+ * css_color (can only contain color names or hex-values)
+ * ... (add more as need arises)
+
+Placeholder default value will be used when not provided at
+instantiation time and could be used to test validity of the
+template by creating a default instance.
+
+Additionally you'll be able to embed an authorization
+certificate that would be used to sign any instance of the template.
+
+```js
+// template.json 
+{
+  version: "0.0.1",
+  // there can be at most 1 template with the same name for any user 
+  // valid names start with a letter and only contains letter, numbers
+  // or underscores
+  name: "template_name", 
+  // embedded authorization certificate
+  auth: {
+   // See https://github.com/CartoDB/Windshaft-cartodb/wiki/Signed-maps
+   method: "token", // or "open" (the default if no "method" is given)
+   // only (required and non empty) for "token" method
+   valid_tokens: ["auth_token1","auth_token2"]
+  },
+  // Variables not listed here are not substituted
+  // Variable not provided at instantiation time trigger an error
+  // A default is required for optional variables
+  // Type specification is used for quoting, to avoid injections
+  placeholders: {
+      color: {
+        type:"css_color",
+        default:"red"
+      },
+      cartodb_id: {
+        type:"number",
+        default: 1
+      }
+  },
+  layergroup: {
+   // see https://github.com/CartoDB/Windshaft/wiki/MapConfig-specification
+   "version": "1.0.1",
+   "layers": [{
+    "type": "cartodb",
+    "options": {
+      "cartocss_version": "2.1.1", 
+      "cartocss": "#layer { polygon-fill: <%= color %>; }",
+      "sql": "select * from european_countries_e WHERE cartodb_id = <%= cartodb_id %>"
+    }
+   }]
+  } 
+}
+```
+
+# Creating a templated map
+
+You can create a signed template map with a single call (for simplicity).
+You'd use a POST sending JSON data:
+
+```sh
+curl -X POST \
+   -H 'Content-Type: application/json' \
+   -d @template.json \
+   'https://docs.cartodb.com/tiles/template?api_key=APIKEY'
+```
+
+The response would be like this:
+```js
+{
+   "template_id":"@template_name"
+}
+```
+
+If a template with the same name exists in the user storage,
+a 400 response is generated.
+
+Errors are in this form:
+```js
+{
+   "error":"Some error string here"
+}
+```
+
+# Updating an existing template
+
+Update of a template map implies removal all signatures from previous
+map instances.
+
+You can update a signed template map with a PUT:
+
+```sh
+curl -X PUT \
+   -H 'Content-Type: application/json' \
+   -d @template.json \
+   'https://docs.cartodb.com/tiles/template/:template_name?api_key=APIKEY'
+```
+A template with the same name will be updated, if any.
+
+The response would be like this:
+```js
+{
+   "template_id":"@template_name"
+}
+```
+
+If a template with the same name does NOT exist,
+a 400 HTTP response is generated with an error in this format:
+
+```js
+{
+   "error":"Some error string here"
+}
+```
+
+
+# Listing available templates
+
+You can get a list of available templates with a GET to ``/template``.
+A valid api_key is required.
+
+```sh
+curl -X GET 'https://docs.cartodb.com/tiles/template?api_key=APIKEY'
+```
+
+The response would be like this:
+```js
+{
+   "template_ids": ["@template_name1","@template_name2"]
+}
+```
+
+Or, on error:
+
+```js
+{
+   "error":"Some error string here"
+}
+```
+
+# Getting a specific template
+
+You can get the definition of a template with a
+GET to ``/template/:template_name``.
+A valid api_key is required.
+
+Example:
+
+```sh
+curl -X GET 'https://docs.cartodb.com/tiles/template/@template_name?auth_token=AUTH_TOKEN'
+```
+
+The response would be like this:
+```js
+{
+   "template": {...}  // see template.json above
+}
+```
+
+Or, on error:
+
+```js
+{
+   "error":"Some error string here"
+}
+```
+
+# Instantiating a template map
+
+You can instantiate a template map passing all required parameters with
+a POST to ``/template/:template_name``.
+
+Valid credentials will be needed, if required by the template.
+
+```js
+// params.js
+{
+ color: '#ff0000',
+ cartodb_id: 3
+}
+```
+
+```sh
+curl -X POST \
+  -H 'Content-Type: application/json' \
+  -d @params.js \
+  'https://docs.cartodb.com/tiles/template/@template_name?auth_token=AUTH_TOKEN'
+
+```
+
+The response would be like this:
+```js
+{
+   "layergroupid":"docs@fd2861af@c01a54877c62831bb51720263f91fb33:123456788",
+   "last_updated":"2013-11-14T11:20:15.000Z"
+}
+```
+
+or, on error:
+
+```js
+{
+   "error":"Some error string here"
+}
+```
+
+You can then use the ``layergroupid`` for fetching tiles and grids as you do
+normally ( see https://github.com/CartoDB/Windshaft/wiki/Multilayer-API).
+But you'll still have to show the ``auth_token``, if required by the template
+(see https://github.com/CartoDB/Windshaft-cartodb/wiki/Signed-maps)
+
+Instances of a signed template map will be signed with the same signature
+certificate associated with the template. Such certificate would contain
+a reference to the template identifier, so that it can be revoked every
+time the template is updated or deleted. 
+
+### using JSONP
+There is also a special endpoint to be able to instanciate using JSONP (for old browsers)
+
+```
+curl 'https://docs.cartodb.com/tiles/template/@template_name/jsonp?auth_token=AUTH_TOKEN&callback=function_name&config=template_params_json'
+```
+
+it takes the ``callback`` function (required), ``auth_token`` in case the template needs auth and ``config`` which is the variabñes for the template (in case it has variables). For example config may be created (using javascript)
+```
+url += "config=" + encodeURIComponent(
+JSON.stringify({ color: 'red' });
+```
+
+the response it's in this format:
+```
+jQuery17205720721024554223_1390996319118(
+{
+layergroupid: "dev@744bd0ed9b047f953fae673d56a47b4d:1390844463021.1401",
+last_updated: "2014-01-27T17:41:03.021Z"
+}
+)
+```
+# Deleting a template map
+
+Deletion of a template map will imply removal all instance signatures
+
+You can delete a templated map with a DELETE to ``/template/:template_name``:
+
+```sh
+curl -X DELETE 'https://docs.cartodb.com/tiles/template/@template_name?auth_token=AUTH_TOKEN'
+```
+
+On success, a 204 (No Content) response would be issued.
+Otherwise a 4xx response with this format:
+
+```js
+{
+   "error":"Some error string here"
+}
+```

docs/metrics.md

@@ -0,0 +1,46 @@
+Windshaft-cartodb metrics
+=========================
+See [Windshaft metrics documentation](https://github.com/CartoDB/Windshaft/blob/master/doc/metrics.md) to understand the full picture.
+
+The next list includes the API endpoints, each endpoint may have several inner timers, some of them are displayed within this list as subitems. Find the description for them in the Inner timers section.
+## Timers
+- **windshaft-cartodb.flush_cache**: time to flush the tile and sql cache
+- **windshaft-cartodb.get_template**: time to retrieve an specific template
+- **windshaft-cartodb.delete_template**: time to delete an specific template
+- **windshaft-cartodb.get_template_list**: time to retrieve the list of owned templates
+- **windshaft-cartodb.instance_template_post**: time to create a template via HTTP POST
+- **windshaft-cartodb.instance_template_get**: time to create a template via HTTP GET
+    + TemplateMaps_instance
+    + createLayergroup
+
+There are some endpoints that are not being tracked:
+- Adding a template
+- Updating a template
+
+### Inner timers
+Again, each inner timer may have several inner timers.
+
+- **addCacheChannel**: time to add X-Cache-Channel header based on table last modifications
+- **LZMA decompress**: time to decompress request params with LZMA
+- **TemplateMaps_instance**: time to retrieve a map template instance, see *getTemplate* and *authorizedByCert*
+- **affectedTables**: time to check what are the affected tables for adding the cache channel, see *addCacheChannel*
+- **authorize**: time to authorize a request, see *authorizedByAPIKey*, *authorizedByCert*, *authorizedBySigner*
+- **authorizedByAPIKey**: time to authorize using an API KEY
+- **authorizedByCert**: time to authorize a request by a cert, see [signed map](https://github.com/CartoDB/Windshaft-cartodb/wiki/Signed-maps)
+- **authorizedBySigner**: time to authorize a request for a [signed map](https://github.com/CartoDB/Windshaft-cartodb/wiki/Signed-maps)
+- **findLastUpdated**: time to retrieve the last update time for a list of tables, see *affectedTables*
+- **fingerPrint**: time to create a fingerprint for a signed map
+- **generateCacheChannel**: time to generate the headers for the cache channel based on the request, see *addCacheChannel*
+- **getSignerMapKey**: time to retrieve from redis the authorized key for a signed map
+- **getTablePrivacy**: time to retrieve from redis the privacy of a table
+- **getTemplate**: time to retrieve from redis the template for a map
+- **getUserMapKey**: time to retrieve from redis the user key for a map
+- **incMapviewCount**: time to incremenent in redis the map views
+- **mapStore_load**: time to retrieve from redis a map configuration
+- **req2params.setup**: time to prepare the params from a request, see *req2params* in Windshaft documentation
+- **setDBAuth**: time to retrieve from redis and set db user and db password from a user
+- **setDBConn**: time to retrieve from redis and set db host and db name from a user
+- **setDBParams**: time to prepare all db params to be able to connect/query a database, see *setDBAuth* and *setDBConn*
+- **signMap**: time to sign in redis layergroup for a map, see signed maps
+- **tablePrivacy_getUserDBName**: time to retrieve from redis the database for a user
+

lib/cartodb/api/query_tables_api.js

@@ -0,0 +1,122 @@
+var sqlApi = require('../sql/sql_api'),
+    PSQL   = require('cartodb-psql');
+
+function QueryTablesApi() {
+}
+
+var affectedTableRegexCache = {
+    bbox: /!bbox!/g,
+    scale_denominator: /!scale_denominator!/g,
+    pixel_width: /!pixel_width!/g,
+    pixel_height: /!pixel_height!/g
+};
+
+module.exports = QueryTablesApi;
+
+QueryTablesApi.prototype.getLastUpdatedTime = function (username, api_key, tableNames, callback) {
+    var sql = 'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max FROM CDB_TableMetadata m WHERE m.tabname = any (ARRAY['+
+        tableNames.map(function(t) { return "'" + t + "'::regclass"; }).join(',') +
+        '])';
+
+    // call sql api
+    sqlApi.query(username, api_key, sql, function(err, rows){
+        if (err){
+            var msg = err.message ? err.message : err;
+            callback(new Error('could not find last updated timestamp: ' + msg));
+            return;
+        }
+        // when the table has not updated_at means it hasn't been changed so a default last_updated is set
+        var last_updated = 0;
+        if(rows.length !== 0) {
+            last_updated = rows[0].max || 0;
+        }
+
+        callback(null, last_updated*1000);
+    });
+};
+
+QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, options, sql, callback) {
+
+    var query = 'SELECT CDB_QueryTables($windshaft$' + prepareSql(sql) + '$windshaft$)';
+
+    runQuery(username, options, query, handleAffectedTablesInQueryRows, callback);
+};
+
+function handleAffectedTablesInQueryRows(err, rows, callback) {
+    if (err){
+        var msg = err.message ? err.message : err;
+        callback(new Error('could not fetch source tables: ' + msg));
+        return;
+    }
+    var qtables = rows[0].cdb_querytables;
+    var tableNames = qtables.split(/^\{(.*)\}$/)[1];
+    tableNames = tableNames ? tableNames.split(',') : [];
+    callback(null, tableNames);
+}
+
+QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (username, options, sql, callback) {
+
+    var query = [
+        'WITH querytables AS (',
+            'SELECT * FROM CDB_QueryTables($windshaft$' + prepareSql(sql) + '$windshaft$) as tablenames',
+        ')',
+        'SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max',
+        'FROM CDB_TableMetadata m',
+        'WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])'
+    ].join(' ');
+
+    runQuery(username, options, query, handleAffectedTablesAndLastUpdatedTimeRows, callback);
+};
+
+function handleAffectedTablesAndLastUpdatedTimeRows(err, rows, callback) {
+    if (err || rows.length === 0) {
+        var msg = err.message ? err.message : err;
+        callback(new Error('could not fetch affected tables and last updated time: ' + msg));
+        return;
+    }
+
+    var result = rows[0];
+
+    var tableNames = result.tablenames.split(/^\{(.*)\}$/)[1];
+    tableNames = tableNames ? tableNames.split(',') : [];
+
+    var lastUpdatedTime = result.max || 0;
+
+    callback(null, {
+        affectedTables: tableNames,
+        lastUpdatedTime: lastUpdatedTime * 1000
+    });
+}
+
+
+function runQuery(username, options, query, queryHandler, callback) {
+    if (shouldQueryPostgresDirectly()) {
+        var psql = new PSQL(options);
+        psql.query(query, function(err, resultSet) {
+            resultSet = resultSet || {};
+            var rows = resultSet.rows || [];
+            queryHandler(err, rows, callback);
+        });
+    } else {
+        sqlApi.query(username, options.api_key, query, function(err, rows) {
+            queryHandler(err, rows, callback);
+        });
+    }
+}
+
+
+function prepareSql(sql) {
+    return sql
+        .replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)')
+        .replace(affectedTableRegexCache.scale_denominator, '0')
+        .replace(affectedTableRegexCache.pixel_width, '1')
+        .replace(affectedTableRegexCache.pixel_height, '1')
+    ;
+}
+
+
+function shouldQueryPostgresDirectly() {
+    return global.environment
+        && global.environment.enabledFeatures
+        && global.environment.enabledFeatures.cdbQueryTablesFromPostgres;
+}

lib/cartodb/cache_validator.js

@@ -1,89 +1,26 @@
 var _             = require('underscore'),
     Varnish       = require('node-varnish'),
-    request       = require('request'),
-    crypto      = require('crypto'),
-    channelCache  = {},
     varnish_queue = null;
 
-function init(host, port) {
-    varnish_queue = new Varnish.VarnishQueue(host, port);
+function init(host, port, secret) {
+    varnish_queue = new Varnish.VarnishQueue(host, port, secret);
+    varnish_queue.on('error', function(e) {
+        console.log("[CACHE VALIDATOR ERROR] " + e);
+    });
 }
 
 function invalidate_db(dbname, table) {
+    var cmd = 'purge obj.http.X-Cache-Channel ~ "^' + dbname +
+              ':(.*'+ table +'.*)|(table)$"';
     try{
-        varnish_queue.run_cmd('purge obj.http.X-Cache-Channel ~ "^' + dbname + ':(.*'+ table +'.*)|(table)$"');
-        console.log('[SUCCESS FLUSHING CACHE]');
+        varnish_queue.run_cmd(cmd, false);
     } catch (e) {
-        console.log("[ERROR FLUSHING CACHE] Is enable_cache set to true? Failed for: " + 'purge obj.http.X-Cache-Channel ~ "^' + dbname + ':(.*'+ table +'.*)|(table)$"');
+        console.log("[CACHE VALIDATOR ERROR] could not queue command " +
+                    cmd + " -- " + e);
     }
 }
 
-function generateCacheChannel(req, callback){
-    var cacheChannel = "";
-
-    // use key to call sql api with sql request if present, else just return dbname and table name
-    // base key
-    var tableNames = req.params.table;
-    var dbName     = req.params.dbname;
-    var username   = req.headers.host.split('.')[0];
-
-    // replace tableNames with the results of the explain if present
-    if (_.isString(req.params.sql) && req.params.sql != ''){
-        // initialise MD5 key of sql for cache lookups
-        var sql_md5 = generateMD5(req.params.sql);
-        var api = global.environment.sqlapi;
-        var qs  = {};
-
-        // use cache if present
-        if (!_.isNull(channelCache[sql_md5]) && !_.isUndefined(channelCache[sql_md5])) {
-            callback(channelCache[sql_md5]);
-        } else{
-            // strip out windshaft/mapnik inserted sql if present
-            var sql = req.params.sql.match(/^\((.*)\)\sas\scdbq$/);
-            sql = (sql != null) ? sql[1] : req.params.sql;
-
-            // build up api string
-            var sqlapi = api.protocol + '://' + username + '.' + api.host + ':' + api.port + '/api/' + api.version + '/sql'
-
-            // add query to querystring
-            qs.q = 'SELECT CDB_QueryTables($windshaft$' + sql + '$windshaft$)';
-
-            // add api_key if present in tile request (means table is private)
-            if (_.isString(req.params.map_key) && req.params.map_key != ''){
-                qs.api_key = req.params.map_key;
-            }
-
-            // call sql api
-            request.get({url:sqlapi, qs:qs, json:true}, function(err, response, body){
-                if (!err && response.statusCode == 200) {
-                    tableNames = body.rows[0].cdb_querytables.split(/^\{(.*)\}$/)[1];
-                } else {
-                    //oops, no SQL API. Just cache using fallback 'table' key
-                    tableNames = 'table';
-                }
-                cacheChannel = buildCacheChannel(dbName,tableNames);
-                channelCache[sql_md5] = cacheChannel; // store for caching
-                callback(cacheChannel);
-            });
-        }
-    } else {
-        cacheChannel = buildCacheChannel(dbName,tableNames);
-        callback(cacheChannel);
-    }
-}
-
-function buildCacheChannel(dbName, tableNames){
-    return dbName + ':' + tableNames;
-}
-
-function generateMD5(data){
-    var hash = crypto.createHash('md5');
-    hash.update(data);
-    return hash.digest('hex');
-}
-
 module.exports = {
     init: init,
-    invalidate_db: invalidate_db,
-    generateCacheChannel: generateCacheChannel
+    invalidate_db: invalidate_db
 }

lib/cartodb/carto_data.js

@@ -1,223 +0,0 @@
-/**
- * User: simon
- * Date: 30/08/2011
- * Time: 21:10
- * Desc: CartoDB helper.
- *       Retrieves dbname (based on subdomain/username)
- *       and geometry type from the redis stores of cartodb
- */
-
-var   RedisPool = require("./redis_pool")
-    , _ = require('underscore')
-    , Step = require('step');
-
-module.exports = function() {
-    var redis_pool = new RedisPool(global.environment.redis);
-
-
-    var me = {
-        user_metadata_db: 5,
-        table_metadata_db: 0,
-        user_key:  "rails:users:<%= username %>",
-        table_key: "rails:<%= database_name %>:<%= table_name %>"
-    };
-
-
-    /**
-     * Get the database name for this particular subdomain/username
-     *
-     * @param req - standard express req object. importantly contains host information
-     * @param callback
-     */
-    me.getDatabase = function(req, callback) {
-        // strip subdomain from header host
-        var username = req.headers.host.split('.')[0]
-        var redisKey = _.template(this.user_key, {username: username});
-
-        this.retrieve(this.user_metadata_db, redisKey, 'database_name', callback);
-    };
-
-
-
-    /**
-     * Get the user id for this particular subdomain/username
-     *
-     * @param req - standard express req object. importantly contains host information
-     * @param callback
-     */
-    me.getId= function(req, callback) {
-        // strip subdomain from header host
-        var username = req.headers.host.split('.')[0];
-        var redisKey = _.template(this.user_key, {username: username});
-
-        this.retrieve(this.user_metadata_db, redisKey, 'id', callback);
-    };
-
-    /**
-     * Check the user map key for this particular subdomain/username
-     *
-     * @param req - standard express req object. importantly contains host information
-     * @param callback
-     */
-    me.checkMapKey = function(req, callback) {
-        // strip subdomain from header host
-        var username = req.headers.host.split('.')[0];
-        var redisKey = "rails:users:" + username; 
-        this.retrieve(this.user_metadata_db, redisKey, "map_key", function(err, val) {
-            var valid = 0;
-            if ( val && val == req.query.map_key ) valid = 1;
-            callback(err, valid);
-        });
-    };
-
-    /**
-     * Get privacy for cartodb table
-     *
-     * @param req - standard req object. Importantly contains table and host information
-     * @param callback - is the table private or not?
-     */
-    me.authorize= function(req, callback) {
-        var that = this;
-
-        Step(
-            function(){
-                that.checkMapKey(req, this);
-            },
-            function checkIfInternal(err, check_result){
-                if (err) throw err;
-                if (check_result === 1) {
-                    // authorized by key, login as db owner 
-                    that.getId(req, function(err, user_id) {
-                        if (err) throw new Error(err);
-                        var dbuser = _.template(global.settings.postgres_auth_user, {user_id: user_id});
-                        _.extend(req, {dbuser:dbuser});
-                        callback(err, true); 
-                    });
-                } else {
-                    return true; // continue to check if the table is public/private
-                }
-            }
-            ,function (err, data){
-                if (err) throw err;
-                that.getDatabase(req, this);
-            },
-            function(err, data){
-                if (err) throw err;
-                var redisKey = _.template(that.table_key, {database_name: data, table_name: req.params.table});
-
-                that.retrieve(that.table_metadata_db, redisKey, 'privacy', this);
-            },
-            function(err, data){
-                if (err) throw err;
-                callback(err, data);
-             }
-        );
-    };
-
-
-    /**
-     * Get the geometry type for this particular table;
-     * @param req - standard req object. Importantly contains table and host information
-     * @param callback
-     */
-    me.getGeometryType = function(req, callback){
-        var that = this;
-
-        Step(
-            function(){
-                that.getDatabase(req, this)
-            },
-            function(err, data){
-                if (err) throw err;
-                var redisKey = _.template(that.table_key, {database_name: data, table_name: req.params.table});
-
-                that.retrieve(that.table_metadata_db, redisKey, 'the_geom_type', this);
-            },
-            function(err, data){
-                if (err) throw err;
-                callback(err, data);
-            }
-        );
-    };
-
-
-    me.getInfowindow = function(req, callback){
-        var that = this;
-
-        Step(
-            function(){
-                that.getDatabase(req, this);
-            },
-            function(err, data) {
-                if (err) throw err;
-                var redisKey = _.template(that.table_key, {database_name: data, table_name: req.params.table});
-                that.retrieve(that.table_metadata_db, redisKey, 'infowindow', this);
-            },
-            function(err, data){
-                if (err) throw err;
-                callback(err, data);
-            }
-        );
-    };
-
-
-    me.getMapMetadata = function(req, callback){
-        var that = this;
-
-        Step(
-            function(){
-                that.getDatabase(req, this);
-            },
-            function(err, data) {
-                if (err) throw err;
-                var redisKey = _.template(that.table_key, {database_name: data, table_name: req.params.table});
-
-                that.retrieve(that.table_metadata_db, redisKey, 'map_metadata', this);
-            },
-            function(err, data){
-                if (err) throw err;
-                callback(err, data);
-            }
-        );
-    };
-
-    // Redis Hash lookup
-    me.retrieve = function(db, redisKey, hashKey, callback) {
-        this.redisCmd(db,'HGET',[redisKey, hashKey], callback);
-    };
-
-    // Redis Set member check
-    me.inSet = function(db, setKey, member, callback) {
-        this.redisCmd(db,'SISMEMBER',[setKey, member], callback);
-    };
-
-    /**
-     * Use Redis
-     *
-     * @param db - redis database number
-     * @param redisFunc - the redis function to execute
-     * @param redisArgs - the arguments for the redis function in an array
-     * @param callback - function to pass results too.
-     */
-    me.redisCmd = function(db, redisFunc, redisArgs, callback) {
-        var redisClient;
-
-        Step(
-            function getRedisClient() {
-                redis_pool.acquire(db, this);
-            },
-            function executeQuery(err, data) {
-                redisClient = data;
-                redisArgs.push(this);
-                redisClient[redisFunc.toUpperCase()].apply(redisClient, redisArgs);
-            },
-            function releaseRedisClient(err, data) {
-                if (err) throw err;
-                redis_pool.release(db, redisClient);
-                callback(err, data);
-            }
-        );
-    };
-
-    return me;
-}();

lib/cartodb/cartodb_windshaft.js

@@ -1,25 +1,35 @@
-
 var _ = require('underscore')
     , Step       = require('step')
     , Windshaft = require('windshaft')
-    , Cache = require('./cache_validator');
+    , SignedMaps = require('./signed_maps.js')
+    , TemplateMaps = require('./template_maps.js')
+    , Cache = require('./cache_validator')
+    , os = require('os')
+;
+
+if ( ! process.env['PGAPPNAME'] )
+  process.env['PGAPPNAME']='cartodb_tiler';
 
 var CartodbWindshaft = function(serverOptions) {
+   var debug = global.environment.debug;
 
-    // set the cache chanel info to invalidate the cache on the frontend server
-    serverOptions.afterTileRender = function(req, res, tile, headers, callback) {
-        var ttl = global.environment.varnish.ttl || 86400;
-        Cache.generateCacheChannel(req, function(channel){
-            res.header('X-Cache-Channel', channel);
-            res.header('Last-Modified', new Date().toUTCString());
-            res.header('Cache-Control', 'no-cache,max-age='+ttl+',must-revalidate, public');
-            callback(null, tile, headers);
-        });
-    };
+    // Perform keyword substitution in statsd
+    // See https://github.com/CartoDB/Windshaft-cartodb/issues/153
+    if ( global.environment.statsd ) {
+      if ( global.environment.statsd.prefix ) {
+        var host_token = os.hostname().split('.').reverse().join('.');
+        global.environment.statsd.prefix = global.environment.statsd.prefix.replace(/:host/, host_token);
+      }
+    }
+
+    var redisPool = serverOptions.redis.pool
+        || require('redis-mpool')(_.extend(global.environment.redis, {name: 'windshaft:cartodb'}));
+
+    var cartoData = require('cartodb-redis')({pool: redisPool});
 
     if(serverOptions.cache_enabled) {
         console.log("cache invalidation enabled, varnish on ", serverOptions.varnish_host, ' ', serverOptions.varnish_port);
-        Cache.init(serverOptions.varnish_host, serverOptions.varnish_port);
+        Cache.init(serverOptions.varnish_host, serverOptions.varnish_port, serverOptions.varnish_secret);
         serverOptions.afterStateChange = function(req, data, callback) {
             Cache.invalidate_db(req.params.dbname, req.params.table);
             callback(null, data);
@@ -28,15 +38,102 @@ var CartodbWindshaft = function(serverOptions) {
 
     serverOptions.beforeStateChange = function(req, callback) {
         var err = null;
-        if ( ! req.hasOwnProperty('dbuser') ) {
+        if ( ! req.params.hasOwnProperty('_authorizedByApiKey') ) {
           err = new Error("map state cannot be changed by unauthenticated request!");
         }
         callback(err, req);
-    }
+    };
+
+    // This is for Templated maps
+    //
+    // "named" is the official, "template" is for backward compatibility up to 1.6.x
+    //
+    var template_baseurl = global.environment.base_url_templated || '(?:/maps/named|/tiles/template)';
+
+    serverOptions.signedMaps = new SignedMaps(redisPool);
+    var templateMapsOpts = {
+      max_user_templates: global.environment.maxUserTemplates
+    };
+    var templateMaps = new TemplateMaps(redisPool, serverOptions.signedMaps, templateMapsOpts);
 
     // boot
     var ws = new Windshaft.Server(serverOptions);
 
+    // Override getVersion to include cartodb-specific versions
+    var wsversion = ws.getVersion;
+    ws.getVersion = function() {
+      var version = wsversion();
+      version.windshaft_cartodb = require('../../package.json').version;
+      return version;
+    };
+
+    var ws_sendResponse = ws.sendResponse;
+    // GET routes for which we don't want to request any caching.
+    // POST/PUT/DELETE requests are never cached anyway.
+    var noCacheGETRoutes = [
+      '/',
+      // See https://github.com/CartoDB/Windshaft-cartodb/issues/176
+      serverOptions.base_url_mapconfig,
+      template_baseurl + '/:template_id/jsonp'
+    ];
+    ws.sendResponse = function(res, args) {
+      var that = this;
+      var thatArgs = arguments;
+      var statusCode;
+      if ( res._windshaftStatusCode ) {
+        // Added by our override of sendError
+        statusCode = res._windshaftStatusCode;
+      } else {
+        if ( args.length > 2 ) statusCode = args[2];
+        else {
+          statusCode = args[1] || 200;
+        }
+      }
+      var req = res.req;
+      Step (
+        function addCacheChannel() {
+          if ( ! req ) {
+            // having no associated request can happen when
+            // using fake response objects for testing layergroup
+            // creation
+            return false;
+          }
+          if ( ! req.params ) {
+            // service requests (/version, /) 
+            // have no need for an X-Cache-Channel
+            return false;
+          }
+          if ( statusCode != 200 ) {
+            // We do not want to cache
+            // unsuccessful responses
+            return false;
+          }
+          if ( _.contains(noCacheGETRoutes, req.route.path) ) {
+//console.log("Skipping cache channel in route:\n" + req.route.path);
+            return false;
+          }
+//console.log("Adding cache channel to route\n" + req.route.path + " not matching any in:\n" + mapCreateRoutes.join("\n"));
+          serverOptions.addCacheChannel(that, req, this);
+        },
+        function sendResponse(err, added) {
+          if ( err ) console.log(err + err.stack);
+          ws_sendResponse.apply(that, thatArgs);
+          return null;
+        },
+        function finish(err) {
+          if ( err ) console.log(err + err.stack);
+        }
+      );
+    };
+
+    var ws_sendError = ws.sendError;
+    ws.sendError = function() {
+      var res = arguments[0];
+      var statusCode = arguments[2];
+      res._windshaftStatusCode = statusCode;
+      ws_sendError.apply(this, arguments);
+    };
+
     /**
      * Helper to allow access to the layer to be used in the maps infowindow popup.
      */
@@ -48,9 +145,10 @@ var CartodbWindshaft = function(serverOptions) {
             },
             function(err, data){
                 if (err){
-                    res.send({error: err.message}, 500);
+                    ws.sendError(res, {error: err.message}, 500, 'GET INFOWINDOW', err);
+                    //ws.sendResponse(res, [{error: err.message}, 500]);
                 } else {
-                    res.send({infowindow: data}, 200);
+                    ws.sendResponse(res, [{infowindow: data}, 200]);
                 }
             }
         );
@@ -68,9 +166,10 @@ var CartodbWindshaft = function(serverOptions) {
             },
             function(err, data){
                 if (err){
-                    res.send(err.message, 500);
+                    ws.sendError(res, {error: err.message}, 500, 'GET MAP_METADATA', err);
+                    //ws.sendResponse(res, [err.message, 500]);
                 } else {
-                    res.send({map_metadata: data}, 200);
+                    ws.sendResponse(res, [{map_metadata: data}, 200]);
                 }
             }
         );
@@ -81,21 +180,492 @@ var CartodbWindshaft = function(serverOptions) {
      * TODO: Move?
      */
     ws.del(serverOptions.base_url + '/flush_cache', function(req, res){
+        if ( req.profiler && req.profiler.statsd_client ) {
+          req.profiler.start('windshaft-cartodb.flush_cache');
+        }
         ws.doCORS(res);
         Step(
-            function(){
-                serverOptions.flushCache(req, Cache, this);
+            function flushCache(){
+                serverOptions.flushCache(req, serverOptions.cache_enabled ? Cache : null, this);
             },
-            function(err, data){
+            function sendResponse(err, data){
                 if (err){
-                    res.send(500);
+                    ws.sendError(res, {error: err.message}, 500, 'DELETE CACHE', err);
+                    //ws.sendResponse(res, [500]);
                 } else {
-                    res.send({status: 'ok'}, 200);
+                    ws.sendResponse(res, [{status: 'ok'}, 200]);
                 }
             }
         );
     });
+
+    // ---- Template maps interface starts @{
+
+    ws.userByReq = function(req) {
+        return serverOptions.userByReq(req);
+    };
+
+    // Add a template
+    ws.post(template_baseurl, function(req, res) {
+      ws.doCORS(res);
+      var that = this;
+      var response = {};
+      var cdbuser = ws.userByReq(req);
+      Step(
+        function checkPerms(){
+            serverOptions.authorizedByAPIKey(req, this);
+        },
+        function addTemplate(err, authenticated) {
+          if ( err ) throw err;
+          if (authenticated !== 1) {
+            err = new Error("Only authenticated user can create templated maps");
+            err.http_status = 403;
+            throw err;
+          }
+          var next = this;
+          if ( ! req.headers['content-type'] || req.headers['content-type'].split(';')[0] != 'application/json' )
+            throw new Error('template POST data must be of type application/json');
+          var cfg = req.body;
+          templateMaps.addTemplate(cdbuser, cfg, this);
+        },
+        function prepareResponse(err, tpl_id){
+          if ( err ) throw err;
+          // NOTE: might omit "cdbuser" if == dbowner ...
+          return { template_id: cdbuser + '@' + tpl_id };
+        },
+        function finish(err, response){
+            if ( req.profiler ) {
+              res.header('X-Tiler-Profiler', req.profiler.toJSONString());
+            }
+            if (err){
+                response = { error: ''+err };
+                var statusCode = 400;
+                if ( ! _.isUndefined(err.http_status) ) {
+                  statusCode = err.http_status;
+                }
+                ws.sendError(res, response, statusCode, 'POST TEMPLATE', err);
+            } else {
+              ws.sendResponse(res, [response, 200]);
+            }
+        }
+      );
+    });
+
+    // Update a template
+    ws.put(template_baseurl + '/:template_id', function(req, res) {
+      ws.doCORS(res);
+      var that = this;
+      var response = {};
+      var cdbuser = ws.userByReq(req);
+      var template;
+      var tpl_id;
+      Step(
+        function checkPerms(){
+            serverOptions.authorizedByAPIKey(req, this);
+        },
+        function updateTemplate(err, authenticated) {
+          if ( err ) throw err;
+          if (authenticated !== 1) {
+            err = new Error("Only authenticated user can list templated maps");
+            err.http_status = 403;
+            throw err;
+          }
+          if ( ! req.headers['content-type'] || req.headers['content-type'].split(';')[0] != 'application/json' )
+            throw new Error('template PUT data must be of type application/json');
+          template = req.body;
+          tpl_id = req.params.template_id.split('@');
+          if ( tpl_id.length > 1 ) {
+            if ( tpl_id[0] != cdbuser ) {
+              err = new Error("Invalid template id '"
+                + req.params.template_id + "' for user '" + cdbuser + "'");
+              err.http_status = 404;
+              throw err;
+            }
+            tpl_id = tpl_id[1];
+          }
+          templateMaps.updTemplate(cdbuser, tpl_id, template, this);
+        },
+        function prepareResponse(err){
+          if ( err ) throw err;
+          return { template_id: cdbuser + '@' + tpl_id };
+        },
+        function finish(err, response){
+            if ( req.profiler ) {
+              res.header('X-Tiler-Profiler', req.profiler.toJSONString());
+            }
+            if (err){
+                var statusCode = 400;
+                response = { error: ''+err };
+                if ( ! _.isUndefined(err.http_status) ) {
+                  statusCode = err.http_status;
+                }
+                ws.sendError(res, response, statusCode, 'PUT TEMPLATE', err);
+            } else {
+              ws.sendResponse(res, [response, 200]);
+            }
+        }
+      );
+    });
+
+    // Get a specific template
+    ws.get(template_baseurl + '/:template_id', function(req, res) {
+      if ( req.profiler && req.profiler.statsd_client ) {
+        req.profiler.start('windshaft-cartodb.get_template');
+      }
+      ws.doCORS(res);
+      var that = this;
+      var response = {};
+      var cdbuser = ws.userByReq(req);
+      var template;
+      var tpl_id;
+      Step(
+        function checkPerms(){
+            serverOptions.authorizedByAPIKey(req, this);
+        },
+        function updateTemplate(err, authenticated) {
+          if ( err ) throw err;
+          if (authenticated !== 1) {
+            err = new Error("Only authenticated users can get template maps");
+            err.http_status = 403;
+            throw err;
+          }
+          tpl_id = req.params.template_id.split('@');
+          if ( tpl_id.length > 1 ) {
+            if ( tpl_id[0] != cdbuser ) {
+              var err = new Error("Cannot get template id '"
+                + req.params.template_id + "' for user '" + cdbuser + "'");
+              err.http_status = 404;
+              throw err;
+            }
+            tpl_id = tpl_id[1];
+          }
+          templateMaps.getTemplate(cdbuser, tpl_id, this);
+        },
+        function prepareResponse(err, tpl_val){
+          if ( err ) throw err;
+          if ( ! tpl_val ) {
+            err = new Error("Cannot find template '" + tpl_id + "' of user '" + cdbuser + "'");
+            err.http_status = 404;
+            throw err;
+          }
+          // auth_id was added by ourselves,
+          // so we remove it before returning to the user
+          delete tpl_val.auth_id;
+          return { template: tpl_val };
+        },
+        function finish(err, response){
+            if (err){
+                var statusCode = 400;
+                response = { error: ''+err };
+                if ( ! _.isUndefined(err.http_status) ) {
+                  statusCode = err.http_status;
+                }
+                ws.sendError(res, response, statusCode, 'GET TEMPLATE', err);
+            } else {
+              ws.sendResponse(res, [response, 200]);
+            }
+        }
+      );
+    });
+
+    // Delete a specific template
+    ws.del(template_baseurl + '/:template_id', function(req, res) {
+      if ( req.profiler && req.profiler.statsd_client ) {
+        req.profiler.start('windshaft-cartodb.delete_template');
+      }
+      ws.doCORS(res);
+      var that = this;
+      var response = {};
+      var cdbuser = ws.userByReq(req);
+      var template;
+      var tpl_id;
+      Step(
+        function checkPerms(){
+            serverOptions.authorizedByAPIKey(req, this);
+        },
+        function updateTemplate(err, authenticated) {
+          if ( err ) throw err;
+          if (authenticated !== 1) {
+            err = new Error("Only authenticated users can delete template maps");
+            err.http_status = 403;
+            throw err;
+          }
+          tpl_id = req.params.template_id.split('@');
+          if ( tpl_id.length > 1 ) {
+            if ( tpl_id[0] != cdbuser ) {
+              var err = new Error("Cannot find template id '"
+                + req.params.template_id + "' for user '" + cdbuser + "'");
+              err.http_status = 404;
+              throw err;
+            }
+            tpl_id = tpl_id[1];
+          }
+          templateMaps.delTemplate(cdbuser, tpl_id, this);
+        },
+        function prepareResponse(err, tpl_val){
+          if ( err ) throw err;
+          return { status: 'ok' };
+        },
+        function finish(err, response){
+            if (err){
+                var statusCode = 400;
+                response = { error: ''+err };
+                if ( ! _.isUndefined(err.http_status) ) {
+                  statusCode = err.http_status;
+                }
+                ws.sendError(res, response, statusCode, 'DELETE TEMPLATE', err);
+            } else {
+              ws.sendResponse(res, ['', 204]);
+            }
+        }
+      );
+    });
+
+    // Get a list of owned templates 
+    ws.get(template_baseurl, function(req, res) {
+      if ( req.profiler && req.profiler.statsd_client ) {
+        req.profiler.start('windshaft-cartodb.get_template_list');
+      }
+      ws.doCORS(res);
+      var that = this;
+      var response = {};
+      var cdbuser = ws.userByReq(req);
+      Step(
+        function checkPerms(){
+            serverOptions.authorizedByAPIKey(req, this);
+        },
+        function listTemplates(err, authenticated) {
+          if ( err ) throw err;
+          if (authenticated !== 1) {
+            err = new Error("Only authenticated user can list templated maps");
+            err.http_status = 403;
+            throw err;
+          }
+          templateMaps.listTemplates(cdbuser, this);
+        },
+        function prepareResponse(err, tpl_ids){
+          if ( err ) throw err;
+          // NOTE: might omit "cbduser" if == dbowner ...
+          var ids = _.map(tpl_ids, function(id) { return cdbuser + '@' + id; });
+          return { template_ids: ids };
+        },
+        function finish(err, response){
+            var statusCode = 200;
+            if (err){
+                response = { error: ''+err };
+                if ( ! _.isUndefined(err.http_status) ) {
+                  statusCode = err.http_status;
+                }
+                ws.sendError(res, response, statusCode, 'GET TEMPLATE LIST', err);
+            } else {
+              ws.sendResponse(res, [response, statusCode]);
+            }
+        }
+      );
+    });
+
+    ws.setDBParams = function(cdbuser, params, callback) {
+      Step(
+        function setAuth() {
+          serverOptions.setDBAuth(cdbuser, params, this);
+        },
+        function setConn(err) {
+          if ( err ) throw err;
+          serverOptions.setDBConn(cdbuser, params, this);
+        },
+        function finish(err) {
+          callback(err);
+        }
+      );
+    };
+
+    ws.options(template_baseurl + '/:template_id', function(req, res) {
+      ws.doCORS(res, "Content-Type");
+      return next();
+    });
+
+    // Instantiate a template
+    function instanciateTemplate(req, res, template_params, callback) {
+      ws.doCORS(res);
+      var that = this;
+      var response = {};
+      var template;
+      var signedMaps = serverOptions.signedMaps;
+      var layergroup;
+      var layergroupid;
+      var fakereq; // used for call to createLayergroup
+      var cdbuser = ws.userByReq(req);
+      // Format of template_id: [<template_owner>]@<template_id>
+      var tpl_id = req.params.template_id.split('@');
+      if ( tpl_id.length > 1 ) {
+        if ( tpl_id[0] && tpl_id[0] != cdbuser ) {
+          var err = new Error('Cannot instanciate map of user "'
+                              + tpl_id[0] + '" on database of user "'
+                              + cdbuser + '"');
+          err.http_status = 403;
+          callback(err);
+          return;
+        }
+        tpl_id = tpl_id[1];
+      }
+      var auth_token = req.query.auth_token;
+      Step(
+        function getTemplate(){
+          templateMaps.getTemplate(cdbuser, tpl_id, this);
+        },
+        function checkAuthorized(err, data) {
+          if ( req.profiler ) req.profiler.done('getTemplate');
+          if ( err ) throw err;
+          if ( ! data ) {
+            err = new Error("Template '" + tpl_id + "' of user '" + cdbuser + "' not found");
+            err.http_status = 404;
+            throw err;
+          }
+          template = data;
+          var cert = templateMaps.getTemplateCertificate(template);
+          var authorized = false;
+          try {
+            // authorizedByCert will throw if unauthorized
+            authorized = signedMaps.authorizedByCert(cert, auth_token);
+          } catch (err) {
+            // we catch to add http_status
+            err.http_status = 403;
+            throw err;
+          }
+          if ( ! authorized ) {
+            err = new Error('Unauthorized template instanciation');
+            err.http_status = 403;
+            throw err;
+          }
+          /*if ( (! req.headers['content-type'] || req.headers['content-type'].split(';')[0] != 'application/json') && req.query.callback === undefined) {
+            throw new Error('template POST data must be of type application/json, it is instead ');
+          }*/
+          //var template_params = req.body;
+          if ( req.profiler ) req.profiler.done('authorizedByCert');
+          return templateMaps.instance(template, template_params);
+        },
+        function prepareParams(err, instance){
+          if ( req.profiler ) req.profiler.done('TemplateMaps_instance');
+          if ( err ) throw err;
+          layergroup = instance;
+          fakereq = { query: {}, params: {}, headers: _.clone(req.headers),
+            method: req.method,
+            res: res,
+            profiler: req.profiler
+          };
+          ws.setDBParams(cdbuser, fakereq.params, this);
+        },
+        function setApiKey(err){
+          if ( req.profiler ) req.profiler.done('setDBParams');
+          if ( err ) throw err;
+          cartoData.getUserMapKey(cdbuser, this);
+        },
+        function createLayergroup(err, val) {
+          if ( req.profiler ) req.profiler.done('getUserMapKey');
+          if ( err ) throw err;
+          fakereq.params.api_key = val;
+          ws.createLayergroup(layergroup, fakereq, this);
+        },
+        function signLayergroup(err, resp) {
+          // NOTE: createLayergroup uses profiler.start()/end() internally
+          //if ( req.profiler ) req.profiler.done('createLayergroup');
+          if ( err ) throw err;
+          response = resp;
+          var signer = cdbuser;
+          var map_id = response.layergroupid.split(':')[0]; // dropping last_updated 
+          var crt_id = template.auth_id; // check ?
+          if ( ! crt_id ) {
+            var errmsg = "Template '" + tpl_id + "' of user '" + cdbuser + "' has no signature";
+            // Is this really illegal ?
+            // Maybe we could just return an unsigned layergroupid
+            // in this case...
+            err = new Error(errmsg);
+            err.http_status = 403; // Forbidden, we refuse to respond to this
+            throw err;
+          }
+          signedMaps.signMap(signer, map_id, crt_id, this);
+        },
+        function prepareResponse(err) {
+          if ( req.profiler ) req.profiler.done('signMap');
+          if ( err ) throw err;
+          //console.log("Response from createLayergroup: "); console.dir(response);
+          // Add the signature part to the token!
+          var tplhash = templateMaps.fingerPrint(template).substring(0,8);
+          if ( req.profiler ) req.profiler.done('fingerPrint');
+          response.layergroupid = cdbuser + '@' + tplhash + '@' + response.layergroupid;
+          return response;
+        },
+        callback
+      );
+    }
+
+    function finish_instanciation(err, response, res, req) {
+        if ( req.profiler ) {
+          res.header('X-Tiler-Profiler', req.profiler.toJSONString());
+        }
+        if (err) {
+            var statusCode = 400;
+            response = { error: ''+err };
+            if ( ! _.isUndefined(err.http_status) ) {
+              statusCode = err.http_status;
+            }
+            if(debug) {
+              response.stack = err.stack;
+            }
+            ws.sendError(res, response, statusCode, 'POST INSTANCE TEMPLATE', err);
+        } else {
+          ws.sendResponse(res, [response, 200]);
+        }
+    }
+
+    ws.post(template_baseurl + '/:template_id', function(req, res) {
+      if ( req.profiler && req.profiler.statsd_client) {
+        req.profiler.start('windshaft-cartodb.instance_template_post');
+      }
+      Step(
+        function() {
+          if ( ! req.headers['content-type'] || req.headers['content-type'].split(';')[0] != 'application/json') {
+            throw new Error('template POST data must be of type application/json, it is instead ');
+          }
+          instanciateTemplate(req, res, req.body, this);
+        }, function(err, response) {
+          finish_instanciation(err, response, res, req);
+        }
+      );
+    });
+
+    /**
+     * jsonp endpoint, allows to instanciate a template with a json call.
+     * callback query argument is mandartoy
+     */
+    ws.get(template_baseurl + '/:template_id/jsonp', function(req, res) {
+      if ( req.profiler && req.profiler.statsd_client) {
+        req.profiler.start('windshaft-cartodb.instance_template_get');
+      }
+      Step(
+        function() {
+          if ( req.query.callback === undefined || req.query.callback.length === 0) {
+            throw new Error('callback parameter should be present and be a function name');
+          }
+          var config = {};
+          if(req.query.config) {
+            try {
+              config = JSON.parse(req.query.config);
+            } catch(e) {
+              throw new Error('badformed config parameter, should be a valid JSON');
+            }
+          }
+          instanciateTemplate(req, res, config, this);
+        }, function(err, response) {
+          finish_instanciation(err, response, res, req);
+        }
+      );
+    });
+
+
+    // ---- Template maps interface ends @}
+
     return ws;
-}
+};
 
 module.exports = CartodbWindshaft;

lib/cartodb/log4js_rollbar.js

@@ -0,0 +1,49 @@
+var rollbar = require("rollbar");
+
+/**
+ * Rollbar Appender. Sends logging events to Rollbar using node-rollbar 
+ *
+ * @param config object with rollbar configuration data
+ * {
+ *   token: 'your-secret-token',
+ *   options: node-rollbar options
+ * }
+ */
+function rollbarAppender(config) {
+
+  var opt = config.options;
+	rollbar.init(opt.token, opt.options);
+	
+	return function(loggingEvent) {
+/*
+For logger.trace('one','two','three'):
+{ startTime: Wed Mar 12 2014 16:27:40 GMT+0100 (CET),
+  categoryName: '[default]',
+  data: [ 'one', 'two', 'three' ],
+  level: { level: 5000, levelStr: 'TRACE' },
+  logger: { category: '[default]', _events: { log: [Object] } } }
+*/
+
+    // Levels:
+    // TRACE  5000
+    // DEBUG 10000
+    // INFO  20000
+    // WARN  30000
+    // ERROR 40000
+    // FATAL 50000
+    //
+    // We only log error and higher errors
+    //
+    if ( loggingEvent.level.level < 40000 ) return;
+
+    rollbar.reportMessage(loggingEvent.data);
+	};
+}
+
+function configure(config) {
+	return rollbarAppender(config);
+}
+
+exports.name      = "rollbar";
+exports.appender  = rollbarAppender;
+exports.configure = configure;

lib/cartodb/redis_pool.js

@@ -1,77 +0,0 @@
-/**
- * RedisPool. A database specific redis pooling lib
- *
- */
-
-var redis  = require('redis')
-  , _      = require('underscore')
-  , Pool   = require('generic-pool').Pool;
-
-// constructor.
-// 
-// - `opts` {Object} optional config for redis and pooling
-var RedisPool = function(opts){
-  var opts = opts || {};
-  var defaults = {
-    host: '127.0.0.1', 
-    port: '6379', 
-    max: 50, 
-    idleTimeoutMillis: 10000, 
-    reapIntervalMillis: 1000, 
-    log: false
-  };    
-  var options = _.defaults(opts, defaults)
-
-  var me = {
-    pools: {} // cached pools by DB name
-  };
-  
-  // Acquire resource.
-  //
-  // - `database` {String} redis database name
-  // - `callback` {Function} callback to call once acquired. Takes the form
-  //   `callback(err, resource)`  
-  me.acquire = function(database, callback) {
-      if (!this.pools[database]) {
-        this.pools[database] = this.makePool(database);            
-      }
-      this.pools[database].acquire(function(err,resource) {
-        callback(err, resource);
-      });
-  };
-  
-  // Release resource.
-  //
-  // - `database` {String} redis database name
-  // - `resource` {Object} resource object to release
-  me.release = function(database, resource) {
-      this.pools[database] && this.pools[database].release(resource);
-  };
-    
-  // Factory for pool objects.
-  me.makePool = function(database) {
-    return Pool({
-      name: database,
-      create: function(callback){
-        var client = redis.createClient(options.port, options.host);          
-        client.on('connect', function () {
-          client.send_anyway = true;
-          client.select(database);  
-          client.send_anyway = false;
-        });    
-        return callback(null, client);
-      },
-      destroy: function(client) { 
-        return client.quit(); 
-      },
-      max: options.max, 
-      idleTimeoutMillis: options.idleTimeoutMillis, 
-      reapIntervalMillis: options.reapIntervalMillis, 
-      log: options.log 
-    });
-  };
-      
-  return me;
-};
-
-module.exports = RedisPool;

lib/cartodb/server_options.js

@@ -1,20 +1,623 @@
 var   _          = require('underscore')
     , Step       = require('step')
-    , cartoData  = require('./carto_data');
+    , Cache      = require('./cache_validator')
+    , QueryTablesApi   = require('./api/query_tables_api')
+    , crypto     = require('crypto')
+    , LZMA       = require('lzma').LZMA
+  ;
+
+// This is for backward compatibility with 1.3.3
+if ( _.isUndefined(global.environment.sqlapi.domain) ) {
+  // Only use "host" as "domain" if it contains alphanumeric characters
+  var host = global.environment.sqlapi.host;
+  if ( host && host.match(/[a-zA-Z]/) ) {
+    global.environment.sqlapi.domain = host;
+  }
+}
+
+module.exports = function(redisPool) {
+    var redisOpts = redisPool ? {pool: redisPool} : global.environment.redis;
+    var cartoData = require('cartodb-redis')(redisOpts),
+        lzmaWorker = new LZMA(),
+        queryTablesApi = new QueryTablesApi();
+
+    var rendererConfig = _.defaults(global.environment.renderer || {}, {
+        cache_ttl: 60000, // milliseconds
+        metatile: 4,
+        bufferSize: 64,
+        statsInterval: 60000
+    });
 
-module.exports = function(){
     var me = {
-        base_url: '/tiles/:table',
+        // This is for inline maps and table maps
+        base_url: global.environment.base_url_legacy || '/tiles/:table',
+
+        /// @deprecated with Windshaft-0.17.0
+        ///base_url_notable: '/tiles',
+
+        // This is for Detached maps
+        //
+        // "maps" is the official, while
+        // "tiles/layergroup" is for backward compatibility up to 1.6.x
+        //
+        base_url_mapconfig: global.environment.base_url_detached || '(?:/maps|/tiles/layergroup)',
+
         grainstore: {
+          map: {
+            // TODO: allow to specify in configuration
+            srid: 3857
+          },
           datasource: global.environment.postgres,
-          cachedir: global.environment.millstone.cache_basedir
+          cachedir: global.environment.millstone.cache_basedir,
+          mapnik_version: global.environment.mapnik_version,
+          mapnik_tile_format: global.environment.mapnik_tile_format || 'png',
+          default_layergroup_ttl: global.environment.mapConfigTTL || 7200, 
+          gc_prob: 0.01 // @deprecated since Windshaft-1.8.0
+        },
+        mapnik: {
+          poolSize: rendererConfig.poolSize,
+          metatile: rendererConfig.metatile,
+          bufferSize: rendererConfig.bufferSize
+        },
+        statsd: global.environment.statsd,
+        renderCache: {
+            ttl: rendererConfig.cache_ttl,
+            statsInterval: rendererConfig.statsInterval
         },
         redis: global.environment.redis,
         enable_cors: global.environment.enable_cors,
         varnish_host: global.environment.varnish.host,
         varnish_port: global.environment.varnish.port,
+        varnish_secret: global.environment.varnish.secret,
         cache_enabled: global.environment.cache_enabled,
-        log_format: '[:date] :req[X-Real-IP] \033[90m:method\033[0m \033[36m:req[Host]:url\033[0m \033[90m:status :response-time ms -> :res[Content-Type]\033[0m'
+        log_format: global.environment.log_format,
+        useProfiler: global.environment.useProfiler
+    };
+    
+    // Do not send unwatch on release
+    // See http://github.com/CartoDB/Windshaft-cartodb/issues/161
+    me.redis.unwatchOnRelease = false;
+
+    // Re-use redisPool
+    me.redis.pool = redisPool;
+
+/* This whole block is about generating X-Cache-Channel { */
+
+    // TODO: review lifetime of elements of this cache
+    // NOTE: by-token indices should only be dropped when
+    //       the corresponding layegroup is dropped, because
+    //       we have no SQL after layer creation.
+    me.channelCache = {};
+
+    me.buildCacheChannel = function (dbName, tableNames){
+      return dbName + ':' + tableNames.join(',');
+    };
+
+    me.generateMD5 = function(data){
+        var hash = crypto.createHash('md5');
+        hash.update(data);
+        return hash.digest('hex');
+    };
+
+    me.generateCacheChannel = function(app, req, callback){
+
+      // Build channelCache key
+      var dbName = req.params.dbname;
+      var cacheKey = [ dbName ];
+      if ( req.params.token ) cacheKey.push(req.params.token);
+      else if ( req.params.sql ) cacheKey.push( me.generateMD5(req.params.sql) );
+      cacheKey = cacheKey.join(':');
+
+      var that = this;
+
+      Step (
+        function checkCached() {
+          if ( me.channelCache.hasOwnProperty(cacheKey) ) {
+            callback(null, me.channelCache[cacheKey]);
+            return;
+          }
+          return null;
+        },
+        function extractSQL(err) {
+          if ( err ) throw err;
+
+          if ( req.params.token ) {
+            // TODO: cached cache channel for token-based access should
+            //       be constructed at renderer cache creation time
+            // See http://github.com/CartoDB/Windshaft-cartodb/issues/152
+            if ( ! app.mapStore ) {
+              throw new Error('missing channel cache for token ' + req.params.token);
+            }
+            var next = this;
+            var mapStore = app.mapStore;
+            Step(
+              function loadFromStore() {
+                mapStore.load(req.params.token, this);
+              },
+              function getSQL(err, mapConfig) {
+                if (req.profiler) req.profiler.done('mapStore_load');
+                if ( err ) throw err;
+                var sql = [];
+                _.each(mapConfig.obj().layers, function(lyr) {
+                  sql.push(lyr.options.sql);
+                });
+                sql = sql.join(';');
+                return sql;
+              },
+              function finish(err, sql) {
+                next(err, sql);
+              }
+            );
+            return;
+          }
+
+          if ( ! req.params.sql ) {
+            return null; // no sql
+          }
+
+          // We have sql, and no token...
+
+          // strip out windshaft/mapnik inserted sql if present
+          var sql = req.params.sql.match(/^\((.*)\)\sas\scdbq$/);
+          sql = (sql != null) ? sql[1] : req.params.sql;
+
+          return sql;
+        },
+        function findAffectedTables(err, sql) {
+          if ( err ) throw err;
+          if ( ! sql ) {
+            if ( ! req.params.table ) {
+              throw new Error("this request doesn't need an X-Cache-Channel generated");
+            }
+            return [req.params.table];
+          }
+          var user, key;
+          var next = this;
+          Step (
+            function findUserKey() {
+              if ( req.params.hasOwnProperty('_authorizedBySigner') ) {
+                user = req.params._authorizedBySigner;
+                cartoData.getUserMapKey(user, this);
+              } else {
+                user = that.userByReq(req);
+                key = req.params.map_key || req.params.api_key;
+                return null;
+              }
+            },
+            function getAffected(err, data) {
+              if ( err ) throw err;
+              if ( data ) {
+                if ( req.profiler ) req.profiler.done('getSignerMapKey');
+                key = data;
+              }
+              queryTablesApi.getAffectedTablesInQuery(user, {
+                  user: req.params.dbuser,
+                  pass: req.params.dbpass,
+                  host: req.params.dbhost,
+                  port: req.params.dbport,
+                  dbname: req.params.dbname,
+                  api_key: key
+              }, sql, this); // in addCacheChannel
+            },
+            function finish(err, data) {
+              next(err,data);
+            }
+          );
+        },
+        function buildCacheChannel(err, tableNames) {
+          if ( err ) throw err;
+          if (req.profiler && ! req.params.table ) {
+            req.profiler.done('affectedTables');
+          }
+
+          var dbName     = req.params.dbname;
+          var cacheChannel = me.buildCacheChannel(dbName,tableNames);
+          // store for caching from me.generateCacheChannel
+          // (not worth when table was specified in params)
+          if ( ! req.params.table ) {
+            me.channelCache[cacheKey] = cacheChannel;
+          }
+          return cacheChannel;
+        },
+        function finish(err, cacheChannel) {
+          callback(err, cacheChannel);
+        }
+      );
+    };
+
+    // Set the cache chanel info to invalidate the cache on the frontend server
+    //
+    // @param req The request object.
+    //            The function will have no effect unless req.res exists.
+    //            It is expected that req.params contains 'table' and 'dbname'
+    //
+    // @param cb function(err, channel) will be called when ready.
+    //           the channel parameter will be null if nothing was added
+    //
+    me.addCacheChannel = function(app, req, cb) {
+        // skip non-GET requests, or requests for which there's no response
+        if ( req.method != 'GET' || ! req.res ) { cb(null, null); return; }
+        if (req.profiler) req.profiler.start('addCacheChannel');
+        var res = req.res;
+        var cache_policy = req.query.cache_policy;
+        if ( req.params.token ) cache_policy = 'persist';
+        if ( cache_policy == 'persist' ) {
+          res.header('Cache-Control', 'public,max-age=31536000'); // 1 year
+        } else {
+          var ttl = global.environment.varnish.ttl || 86400;
+          res.header('Cache-Control', 'no-cache,max-age='+ttl+',must-revalidate, public');
+        }
+
+        // Set Last-Modified header
+        var lastUpdated;
+        if ( req.params.cache_buster ) {
+          // Assuming cache_buster is a timestamp
+          // FIXME: store lastModified in the cache channel instead
+          lastUpdated = new Date(parseInt(req.params.cache_buster));
+        } else {
+          lastUpdated = new Date();
+        }
+        res.header('Last-Modified', lastUpdated.toUTCString());
+
+        me.generateCacheChannel(app, req, function(err, channel){
+            if (req.profiler) req.profiler.done('generateCacheChannel');
+            if (req.profiler) req.profiler.end();
+            if ( ! err ) {
+              res.header('X-Cache-Channel', channel);
+              cb(null, channel);
+            } else {
+              console.log('ERROR generating cache channel: ' + ( err.message ? err.message : err ));
+              // TODO: evaluate if we should bubble up the error instead
+              cb(null, 'ERROR');
+            }
+        });
+    };
+
+    me.afterLayergroupCreate = function(req, mapconfig, response, callback) {
+        var token = response.layergroupid;
+
+        var username = this.userByReq(req);
+
+        var tasksleft = 2; // redis key and affectedTables
+        var errors = [];
+
+        var done = function(err) {
+          if ( err ) {
+            errors.push('' + err);
+          }
+          if ( ! --tasksleft ) {
+            err = errors.length ? new Error(errors.join('\n')) : null;
+            callback(err);
+          }
+        };
+
+        // include in layergroup response the variables in serverMedata
+        // those variables are useful to send to the client information
+        // about how to reach this server or information about it
+        var serverMetadata = global.environment.serverMetadata;
+        if (serverMetadata) {
+          _.extend(response, serverMetadata);
+        }
+
+        // Don't wait for the mapview count increment to
+        // take place before proceeding. Error will be logged
+        // asyncronously
+        cartoData.incMapviewCount(username, mapconfig.stat_tag, function(err) {
+          if (req.profiler) req.profiler.done('incMapviewCount');
+          if ( err ) console.log("ERROR: failed to increment mapview count for user '" + username + "': " + err);
+          done();
+        });
+
+        var sql = [];
+        _.each(mapconfig.layers, function(lyr) {
+          sql.push(lyr.options.sql);
+        });
+        sql = sql.join(';');
+
+        var dbName = req.params.dbname;
+        var usr    = this.userByReq(req);
+        var key    = req.params.map_key || req.params.api_key;
+
+        var cacheKey = dbName + ':' + token;
+
+        Step(
+            function getAffectedTablesAndLastUpdatedTime() {
+                queryTablesApi.getAffectedTablesAndLastUpdatedTime(usr, {
+                    user: req.params.dbuser,
+                    pass: req.params.dbpass,
+                    host: req.params.dbhost,
+                    port: req.params.dbport,
+                    dbname: req.params.dbname,
+                    api_key: key
+                }, sql, this);
+            },
+            function handleAffectedTablesAndLastUpdatedTime(err, result) {
+                if (req.profiler) req.profiler.done('queryTablesAndLastUpdated');
+                if ( err ) throw err;
+                var cacheChannel = me.buildCacheChannel(dbName, result.affectedTables);
+                me.channelCache[cacheKey] = cacheChannel;
+
+                if (req.res && req.method == 'GET') {
+                    var res = req.res;
+                    if ( req.query && req.query.cache_policy == 'persist' ) {
+                        res.header('Cache-Control', 'public,max-age=31536000'); // 1 year
+                    } else {
+                        var ttl = global.environment.varnish.layergroupTtl || 86400;
+                        res.header('Cache-Control', 'public,max-age='+ttl+',must-revalidate');
+                    }
+                    res.header('Last-Modified', (new Date()).toUTCString());
+                    res.header('X-Cache-Channel', cacheChannel);
+                }
+
+                // last update for layergroup cache buster
+                response.layergroupid = response.layergroupid + ':' + result.lastUpdatedTime;
+                response.last_updated = new Date(result.lastUpdatedTime).toISOString();
+                return null;
+            },
+            function finish(err) {
+                done(err);
+            }
+        );
+    };
+
+/* X-Cache-Channel generation } */
+
+    me.re_userFromHost = new RegExp(
+      global.environment.user_from_host ||
+      '^([^\\.]+)\\.' // would extract "strk" from "strk.cartodb.com"
+    );
+
+    me.userByReq = function(req) {
+      var host = req.headers.host;
+      var mat = host.match(this.re_userFromHost);
+      if ( ! mat ) {
+        console.error("ERROR: user pattern '" + this.re_userFromHost
+          + "' does not match hostname '" + host + "'");
+        return;
+      }
+      // console.log("Matches: "); console.dir(mat);
+      if ( ! mat.length === 2 ) {
+        console.error("ERROR: pattern '" + this.re_userFromHost
+          + "' gave unexpected matches against '" + host + "': " + mat);
+        return;
+      }
+      return mat[1];
+    };
+
+    // Set db authentication parameters to those of the given username
+    //
+    // @param username the cartodb username, mapped to a database username
+    //                 via CartodbRedis metadata records
+    //
+    // @param params the parameters to set auth options into
+    //               added params are: "dbuser" and "dbpassword"
+    //
+    // @param callback function(err) 
+    //
+    me.setDBAuth = function(username, params, callback) {
+
+      var user_params = {};
+      var auth_user = global.environment.postgres_auth_user;
+      var auth_pass = global.environment.postgres_auth_pass;
+      Step(
+        function getId() {
+          cartoData.getUserId(username, this);
+        },
+        function(err, user_id) {
+          if (err) throw err;
+          user_params['user_id'] = user_id;
+          var dbuser = _.template(auth_user, user_params);
+          _.extend(params, {dbuser:dbuser});
+
+          // skip looking up user_password if postgres_auth_pass
+          // doesn't contain the "user_password" label 
+          if (!auth_pass || ! auth_pass.match(/\buser_password\b/) ) return null;
+
+          cartoData.getUserDBPass(username, this);
+        },
+        function(err, user_password) {
+          if (err) throw err;
+          user_params['user_password'] = user_password;
+          if ( auth_pass ) {
+            var dbpass = _.template(auth_pass, user_params);
+            _.extend(params, {dbpassword:dbpass});
+          }
+          return true;
+        },
+        function finish(err) {
+          callback(err); 
+        }
+      );
+    };
+
+    // Set db connection parameters to those for the given username
+    //
+    // @param dbowner cartodb username of database owner,
+    //                mapped to a database username
+    //                via CartodbRedis metadata records
+    //
+    // @param params the parameters to set connection options into
+    //               added params are: "dbname", "dbhost"
+    //
+    // @param callback function(err) 
+    //
+    me.setDBConn = function(dbowner, params, callback) {
+      // Add default database connection parameters
+      // if none given
+      _.defaults(params, {
+        dbuser: global.environment.postgres.user,
+        dbpassword: global.environment.postgres.password,
+        dbhost: global.environment.postgres.host,
+        dbport: global.environment.postgres.port
+      });
+      Step(
+        function getConnectionParams() {
+            cartoData.getUserDBConnectionParams(dbowner, this);
+        },
+        function extendParams(err, dbParams){
+            if (err) throw err;
+            // we don't want null values or overwrite a non public user
+            if (params.dbuser != 'publicuser' || !dbParams.dbuser) {
+                delete dbParams.dbuser;
+            }
+            if ( dbParams ) _.extend(params, dbParams);
+            return null;
+        },
+        function finish(err) {
+          callback(err);
+        }
+      );
+    };
+
+
+    // Check if a request is authorized by a signer
+    //
+    // Any existing signature for the given request will verified
+    // for authorization to this specific request (may require auth_token)
+    // See https://github.com/CartoDB/Windshaft-cartodb/wiki/Signed-maps
+    //
+    // @param req express request object
+    // @param callback function(err, signed_by) signed_by will be
+    //                 null if the request is not signed by anyone
+    //                 or will be a string cartodb username otherwise.
+    //                 
+    me.authorizedBySigner = function(req, callback)
+    {
+      if ( ! req.params.token || ! req.params.signer ) {
+        //console.log("No signature provided"); // debugging
+        callback(null, null); // no signer requested
+        return;
+      }
+
+      var signer = req.params.signer;
+      var layergroup_id = req.params.token;
+      var auth_token = req.params.auth_token;
+
+      //console.log("Checking authorization from signer " + signer + " for resource " + layergroup_id + " with auth_token " + auth_token);
+
+      me.signedMaps.isAuthorized(signer, layergroup_id, auth_token,
+        function(err, authorized) {
+          callback(err, authorized ? signer : null);
+        });
+    };
+
+    // Check if a request is authorized by api_key
+    //
+    // @param req express request object
+    // @param callback function(err, authorized) 
+    //        NOTE: authorized is expected to be 0 or 1 (integer)
+    //                 
+    me.authorizedByAPIKey = function(req, callback)
+    {
+        var givenKey = req.query.api_key || req.query.map_key;
+        if ( ! givenKey && req.body ) {
+          // check also in request body
+          givenKey = req.body.api_key || req.body.map_key;
+        }
+        if ( ! givenKey ) {
+          callback(null, 0); // no api key, no authorization...
+          return;
+        }
+        //console.log("given ApiKey: " + givenKey);
+        var user = me.userByReq(req);
+        Step(
+          function (){
+              cartoData.getUserMapKey(user, this);
+          },
+          function checkApiKey(err, val){
+              if (err) throw err;
+              return ( val && givenKey == val ) ? 1 : 0;
+          },
+          function finish(err, authorized) {
+              callback(err, authorized);
+          }
+        );
+    };
+
+    /**
+     * Check access authorization
+     *
+     * @param req - standard req object. Importantly contains table and host information
+     * @param callback function(err, allowed) is access allowed not?
+     */
+    me.authorize = function(req, callback) {
+        var that = this;
+        var user = me.userByReq(req);
+
+        Step(
+            function (){
+                that.authorizedByAPIKey(req, this);
+            },
+            function checkApiKey(err, authorized){
+                if (req.profiler) req.profiler.done('authorizedByAPIKey');
+                if (err) throw err;
+
+                // if not authorized by api_key, continue 
+                if (authorized !== 1)  {
+                  // not authorized by api_key, 
+                  // check if authorized by signer
+                  that.authorizedBySigner(req, this);
+                  return;
+                }
+
+                _.extend(req.params, { _authorizedByApiKey: true });
+
+                // authorized by api key, login as the given username and stop
+                that.setDBAuth(user, req.params, function(err) {
+                  callback(err, true); // authorized (or error)
+                });
+            },
+            function checkSignAuthorized(err, signed_by){
+                if (err) throw err;
+                if (req.profiler) {
+                  if ( req.params._authorizedByApiKey ) {
+                    req.profiler.done('setDBAuth');
+                  } else {
+                    req.profiler.done('authorizedBySigner');
+                  }
+                }
+
+                if ( ! signed_by ) {
+                  // request not authorized by signer.
+  
+                  // if table was given, continue to check table privacy
+                  if ( req.params.table ) return null;
+
+                  // if no signer name was given, let dbparams and
+                  // PostgreSQL do the rest.
+                  // 
+                  if ( ! req.params.signer ) {
+                    callback(null, true); // authorized so far
+                    return;
+                  }
+
+                  // if signer name was given, return no authorization
+                  callback(null, false); 
+                  return;
+                }
+
+                // Authorized by "signed_by" !
+                _.extend(req.params, { _authorizedBySigner: signed_by });
+                that.setDBAuth(signed_by, req.params, function(err) {
+                  if (req.profiler) req.profiler.done('setDBAuth');
+                  callback(err, true); // authorized (or error)
+                });
+            },
+            function getDatabase(err){
+                if (err) throw err;
+                // NOTE: only used to get to table privacy
+                cartoData.getUserDBName(user, this);
+            },
+            function getPrivacy(err, dbname){
+                if (err) throw err;
+                if (req.profiler) req.profiler.done('tablePrivacy_getUserDBName');
+                cartoData.getTablePrivacy(dbname, req.params.table, this);
+            },
+            function(err, privacy){
+                if (req.profiler) req.profiler.done('getTablePrivacy');
+                callback(err, privacy !== "0");
+            }
+        );
     };
 
     /**
@@ -25,51 +628,116 @@ module.exports = function(){
      */
     me.req2params = function(req, callback){
 
+        if ( req.query.lzma ) {
+
+          // TODO: check ?
+          //console.log("type of req.query.lzma is " + typeof(req.query.lzma));
+
+          // Decode (from base64)
+          var lzma = (new Buffer(req.query.lzma, 'base64').toString('binary')).split('').map(function(c) { return c.charCodeAt(0) - 128 });
+
+          // Decompress
+          lzmaWorker.decompress(
+            lzma,
+            function(result) {
+              if (req.profiler) req.profiler.done('LZMA decompress');
+              try {
+                delete req.query.lzma;
+                _.extend(req.query, JSON.parse(result));
+                me.req2params(req, callback);
+              } catch (err) {
+                callback(new Error('Error parsing lzma as JSON: ' + err));
+              }
+            },
+            function(percent) { // progress
+              //console.log("LZMA decompression " + percent + "%");
+            }
+          );
+          return;
+        }
+
         // Whitelist query parameters and attach format
-        var good_query = ['sql', 'geom_type', 'cache_buster','callback', 'interactivity', 'map_key', 'style'];
+        var good_query = ['sql', 'geom_type', 'cache_buster', 'cache_policy', 'callback', 'interactivity', 'map_key', 'api_key', 'auth_token', 'style', 'style_version', 'style_convert', 'config' ];
         var bad_query  = _.difference(_.keys(req.query), good_query);
 
         _.each(bad_query, function(key){ delete req.query[key]; });
         req.params =  _.extend({}, req.params); // shuffle things as request is a strange array/object
 
+        var user = me.userByReq(req);
+
+        if ( req.params.token ) {
+          //console.log("Request parameters include token " + req.params.token);
+          var tksplit = req.params.token.split(':');
+          req.params.token = tksplit[0];
+          if ( tksplit.length > 1 ) req.params.cache_buster= tksplit[1];
+          tksplit = req.params.token.split('@');
+          if ( tksplit.length > 1 ) {
+            req.params.signer = tksplit.shift();
+            if ( ! req.params.signer ) req.params.signer = user;
+            else if ( req.params.signer != user ) {
+              var err = new Error('Cannot use map signature of user "' + req.params.signer + '" on database of user "' + user + '"');
+              err.http_status = 403;
+              callback(err);
+              return;
+            }
+            if ( tksplit.length > 1 ) {
+              var template_hash = tksplit.shift(); // unused
+            }
+            req.params.token = tksplit.shift(); 
+            //console.log("Request for token " + req.params.token + " with signature from " + req.params.signer);
+          }
+        }
+
         // bring all query values onto req.params object
         _.extend(req.params, req.query);
 
         // for cartodb, ensure interactivity is cartodb_id or user specified
         req.params.interactivity = req.params.interactivity || 'cartodb_id';
 
-        req.params.processXML = function(req, xml, callback) {
-          var dbuser = req.dbuser ? req.dbuser : global.settings.postgres.user;
-          if ( ! me.rx_dbuser ) me.rx_dbuser = /(<Parameter name="user"><!\[CDATA\[)[^\]]*(]]><\/Parameter>)/;
-          xml = xml.replace(me.rx_dbuser, "$1" + dbuser + "$2");
-          callback(null, xml);
-        }
+        var that = this;
+
+        if (req.profiler) req.profiler.done('req2params.setup');
 
         Step(
             function getPrivacy(){
-                cartoData.authorize(req, this);
+                me.authorize(req, this);
             },
-            function gatekeep(err, data){
+            function gatekeep(err, authorized){
+                if (req.profiler) req.profiler.done('authorize');
                 if(err) throw err;
-                if(data === "0") throw new Error("Sorry, you are unauthorized (permission denied)");
-                return data;
+                if(!authorized) {
+                  err = new Error("Sorry, you are unauthorized (permission denied)");
+                  err.http_status = 403;
+                  throw err;
+                }
+                return null;
             },
-            function getDatabase(err, data){
+            function getDatabase(err){
                 if(err) throw err;
-
-                cartoData.getDatabase(req, this);
+                that.setDBConn(user, req.params, this);
             },
-            function getGeometryType(err, data){
+            function getGeometryType(err){
+                if (req.profiler) req.profiler.done('setDBConn');
                 if (err) throw err;
-                _.extend(req.params, {dbname:data});
-
-                cartoData.getGeometryType(req, this);
+                if ( ! req.params.table ) return null;
+                cartoData.getTableGeometryType(req.params.dbname, req.params.table, this);
             },
             function finishSetup(err, data){
+                if ( err ) { callback(err, req); return; }
+
                 if (!_.isNull(data))
                     _.extend(req.params, {geom_type: data});
 
-                callback(err, req);
+                // Add default database connection parameters
+                // if none given
+                _.defaults(req.params, {
+                  dbuser: global.environment.postgres.user,
+                  dbpassword: global.environment.postgres.password,
+                  dbhost: global.environment.postgres.host,
+                  dbport: global.environment.postgres.port
+                });
+
+                callback(null, req);
             }
         );
     };
@@ -81,14 +749,23 @@ module.exports = function(){
      */
     me.getInfowindow = function(req, callback){
         var that = this;
+        var user = me.userByReq(req);
 
         Step(
             function(){
+                // TODO: if this step really needed ?
                 that.req2params(req, this);
             },
+            function getDatabase(err){
+                if (err) throw err;
+                cartoData.getUserDBName(user, this);
+            },
+            function getInfowindow(err, dbname){
+                if (err) throw err;
+                cartoData.getTableInfowindow(dbname, req.params.table, this);
+            },
             function(err, data){
-                if (err) callback(err, null);
-                else cartoData.getInfowindow(data, callback);
+                callback(err, data);
             }
         );
     };
@@ -100,14 +777,23 @@ module.exports = function(){
      */
     me.getMapMetadata = function(req, callback){
         var that = this;
+        var user = me.userByReq(req);
 
         Step(
             function(){
+                // TODO: if this step really needed ?
                 that.req2params(req, this);
             },
-            function(err, data){
+            function getDatabase(err){
                 if (err) throw err;
-                cartoData.getMapMetadata(data, callback);
+                cartoData.getUserDBName(user, this);
+            },
+            function getMapMetadata(err, dbname){
+                if (err) throw err;
+                cartoData.getTableMapMetadata(dbname, req.params.table, this);
+            },
+            function(err, data){
+                callback(err, data);
             }
         );
     };
@@ -121,16 +807,24 @@ module.exports = function(){
         var that = this;
 
         Step(
-            function(){
+            function getParams(){
+                // this is mostly to compute req.params.dbname
                 that.req2params(req, this);
             },
-            function(err, data){
-                if (err) throw err;
-                Cache.invalidate_db(req.params.dbname, req.params.table);
+            function flushInternalCache(err){
+                // TODO: implement this, see
+                // http://github.com/Vizzuality/Windshaft-cartodb/issues/73
+                return true;
+            },
+            function flushVarnishCache(err){
+                if (err) { callback(err); return; }
+                if(Cache) {
+                  Cache.invalidate_db(req.params.dbname, req.params.table);
+                }
                 callback(null, true);
             }
         );
     };
 
     return me;
-}();
+};

lib/cartodb/signed_maps.js

@@ -0,0 +1,397 @@
+var crypto    = require('crypto');
+var Step      = require('step');
+var _         = require('underscore');
+
+var debug = global.environment ? global.environment.debug : undefined;
+
+// Class handling map signatures and user certificates
+//
+// See https://github.com/CartoDB/Windshaft-cartodb/wiki/Signed-maps
+//
+// @param redis_pool an instance of a "redis-mpool"
+//        See https://github.com/CartoDB/node-redis-mpool
+//        Needs version 0.x.x of the API.
+//
+function SignedMaps(redis_pool) {
+  this.redis_pool = redis_pool;
+
+  // Database containing signatures
+  // TODO: allow configuring ?
+  // NOTE: currently it is the same as
+  //       the one containing layergroups
+  this.db_signatures = 0;
+
+  //
+  // Map signatures in redis are reference to signature certificates
+  // We have the following datastores:
+  //
+  //  1. User certificates: set of per-user authorization certificates
+  //  2. Map signatures: set of per-map certificate references 
+  //  3. Certificate applications: set of per-certificate signed maps
+
+  // User certificates (HASH:crt_id->crt_val)
+  this.key_map_crt = "map_crt|<%= signer %>";
+
+  // Map signatures (SET:crt_id)
+  this.key_map_sig = "map_sig|<%= signer %>|<%= map_id %>";
+  
+  // Certificates applications (SET:map_id)
+  //
+  // Everytime a map is signed, the map identifier (layergroup_id)
+  // is added to this set. The purpose of this set is to drop
+  // all map signatures when a certificate is removed
+  //
+  this.key_crt_sig = "crt_sig|<%= signer %>|<%= crt_id %>";
+
+};
+
+var o = SignedMaps.prototype;
+
+//--------------- PRIVATE METHODS --------------------------------
+
+o._acquireRedis = function(callback) {
+  this.redis_pool.acquire(this.db_signatures, callback);
+};
+
+o._releaseRedis = function(client) {
+  this.redis_pool.release(this.db_signatures, client);
+};
+
+/**
+ * Internal function to communicate with redis
+ *
+ * @param redisFunc - the redis function to execute
+ * @param redisArgs - the arguments for the redis function in an array
+ * @param callback - function to pass results too.
+ */
+o._redisCmd = function(redisFunc, redisArgs, callback) {
+  var redisClient;
+  var that = this;
+  var db = that.db_signatures;
+
+  Step(
+    function getRedisClient() {
+      that.redis_pool.acquire(db, this);
+    },
+    function executeQuery(err, data) {
+      if ( err ) throw err;
+      redisClient = data;
+      redisArgs.push(this);
+      redisClient[redisFunc.toUpperCase()].apply(redisClient, redisArgs);
+    },
+    function releaseRedisClient(err, data) {
+      if ( ! _.isUndefined(redisClient) ) that.redis_pool.release(db, redisClient);
+      callback(err, data);
+    }
+  );
+};
+
+o._getAuthMethod = function(auth) {
+  return auth.method || 'open';
+};
+
+//--------------- PUBLIC API -------------------------------------
+
+/// Check formal validity of a certificate
+//
+/// Return an Error instance if invalid, null otherwise
+///
+o.checkInvalidCertificate = function(cert) {
+  //console.log("Checking cert: "); console.dir(cert);
+  if ( cert.version !== "0.0.1" ) {
+    return new Error("Unsupported certificate version " + cert.version);
+  }
+
+  if ( ! cert.auth ) {
+console.log("Cert is : "); console.dir(cert);
+    return new Error("No certificate authorization");
+  }
+
+  var method = this._getAuthMethod(cert.auth);
+
+  switch ( method ) {
+    case 'open':
+      break;
+    case 'token':
+      if ( ! _.isArray(cert.auth.valid_tokens) )
+        return new Error("Invalid 'token' authentication: missing valid_tokens");
+      if ( ! cert.auth.valid_tokens.length )
+        return new Error("Invalid 'token' authentication: no valid_tokens");
+      break;
+    default:
+      return new Error("Unsupported authentication method: " + cert.auth.method);
+      break;
+  }
+
+  return null; // all valid
+}
+
+// Check if the given certificate authorizes waiver of "auth"
+o.authorizedByCert = function(cert, auth) {
+  auth = _.isArray(auth) ? auth : [auth];
+
+  var err = this.checkInvalidCertificate(cert);
+  if ( err ) throw err;
+
+  var method = this._getAuthMethod(cert.auth);
+
+  // Open authentication certificates are always authorized
+  if ( method === 'open' ) return true;
+
+  // Token based authentication requires valid token
+  if ( method === 'token' ) {
+    return _.intersection(cert.auth.valid_tokens, auth).length > 0;
+  }
+
+  throw new Error("Unsupported authentication method: " + cert.auth.method);
+};
+
+// Check if shown credential are authorized to access a map
+// by the given signer.
+//
+// @param signer a signer name (cartodb username)
+// @param map_id a layergroup_id
+// @param auth an authentication token, or undefined if none
+//                    (can still be authorized by signature)
+//
+// @param callback function(Error, Boolean)
+//
+o.isAuthorized = function(signer, map_id, auth, callback) {
+  var that = this;
+  var redisClient;
+  var db = that.db_signatures;
+  var authorized = false;
+  var certificate_id_list;
+  var missing_certificates = [];
+  if ( debug ) {
+    console.log("Check auth from signer '" + signer + "' on map '" + map_id + "' with auth '" + auth + "'");
+  }
+  Step(
+    function getRedisClient() {
+      that.redis_pool.acquire(db, this);
+    },
+    function getMapSignatures(err, client) {
+      if ( err ) throw err;
+      redisClient = client;
+      var map_sig_key = _.template(that.key_map_sig, {signer:signer, map_id:map_id});
+      redisClient.SMEMBERS(map_sig_key, this);
+      //that._redisCmd('SMEMBERS', [ map_sig_key ], this);
+    },
+    function getCertificates(err, crt_lst) {
+      if ( err ) throw err;
+      if ( debug ) {
+        console.log("Map '" + map_id + "' is signed by " + crt_lst.length + " certificates of user '" + signer);
+      }
+      certificate_id_list = crt_lst;
+      if ( ! crt_lst.length ) {
+        // No certs, avoid calling redis with short args list.
+        // Next step expects a list of certificate values so
+        // we directly send the empty list.
+        return crt_lst;
+      }
+      var map_crt_key = _.template(that.key_map_crt, {signer:signer});
+      //that._redisCmd('HMGET', [ map_crt_key ].concat(crt_lst), this);
+      redisClient.HMGET(map_crt_key, crt_lst, this);
+    },
+    function checkCertificates(err, certs) {
+      if ( err ) throw err;
+      for (var i=0; i<certs.length; ++i) {
+        var crt_id = certificate_id_list[i];
+        if ( _.isNull(certs[i]) ) {
+          missing_certificates.push(crt_id);
+          continue;
+        }
+        var cert;
+        try {
+          //console.log("cert " + crt_id + ": " + certs[i]);
+          cert = JSON.parse(certs[i]);
+          authorized = that.authorizedByCert(cert, auth);
+        } catch (err) {
+          console.log("Certificate " + certificate_id_list[i] + " by user '" + signer + "' is malformed: " + err);
+          continue;
+        }
+        if ( authorized ) {
+          if ( debug ) {
+            console.log("Access to map '" + map_id + "' authorized by cert '"
+                        + certificate_id_list[i] + "' of user '" + signer + "'");
+          }
+          //console.dir(cert);
+          break; // no need to further check certs
+        }
+      }
+      return null;
+    },
+    function finish(err) {
+      if ( missing_certificates.length ) {
+        console.log("WARNING: map '" + map_id + "' is signed by '" + signer
+          + "' with " + missing_certificates.length
+          + " missing certificates: "
+          + missing_certificates + " (TODO: give cleanup instructions)");
+      }
+      if ( redisClient ) that.redis_pool.release(db, redisClient);
+      callback(err, authorized);
+    }
+  );
+};
+
+// Add an authorization certificate from a user.
+//
+// @param signer a signer name (cartodb username)
+// @param cert certificate object, see
+//             http://github.com/CartoDB/Windshaft-cartodb/wiki/Signed-maps
+//
+// @param callback function(err, crt_id) return certificate id
+//
+// TODO: allow for requesting error when certificate already exists ?
+//
+o.addCertificate = function(signer, cert, callback) {
+  var crt_val = JSON.stringify(cert);
+  var crt_id = crypto.createHash('md5').update(crt_val).digest('hex');
+
+  var usr_crt_key = _.template(this.key_map_crt, {signer:signer});
+  this._redisCmd('HSET', [ usr_crt_key, crt_id, crt_val ], function(err, created) {
+    // NOTE: created would be 0 if the field already existed, 1 otherwise
+    callback(err, crt_id);
+  });
+};
+
+// Remove an authorization certificate of a user, also removing
+// any signature made with the certificate.
+//
+// @param signer a signer name (cartodb username)
+// @param crt_id certificate identifier, as returned by addCertificate
+// @param callback function(err)
+//
+o.delCertificate = function(signer, crt_id, callback) {
+  var db = this.db_signatures;
+  var crt_sig_key = _.template(this.key_crt_sig, {signer:signer, crt_id:crt_id});
+  var signed_map_list;
+  var redis_client;
+  var that = this;
+  Step (
+    function getRedisClient() {
+      that._acquireRedis(this);
+    },
+    function removeCertificate(err, data) {
+      if ( err ) throw err;
+      redis_client = data;
+      // Remove the certificate (would be enough to stop authorizing uses)
+      var usr_crt_key = _.template(that.key_map_crt, {signer:signer});
+      redis_client.HDEL(usr_crt_key, crt_id, this);
+    },
+    function getMapSignatures(err, deleted) {
+      if ( err ) throw err;
+      if ( ! deleted ) {
+        // debugging (how can this be possible?)
+        console.log("WARNING: authorization certificate '" + crt_id
+          + "' by user '" + signer + "' did not exist on delete request");
+      }
+      // Get all signatures by this certificate
+      redis_client.SMEMBERS(crt_sig_key, this);
+    },
+    function delMapSignaturesReference(err, map_id_list) {
+      if ( err ) throw err;
+      signed_map_list = map_id_list;
+      if ( debug ) {
+        console.log("Certificate '" + crt_id + "' from user '" + signer
+          + "' was used to sign " + signed_map_list.length + " maps");
+      }
+      redis_client.DEL(crt_sig_key, this);
+    },
+    function delMapSignatures(err) {
+      if ( err ) throw err;
+      var crt_sig_key = _.template(that.key_crt_sig, {signer:signer, crt_id:crt_id});
+      var tx = redis_client.MULTI();
+      for (var i=0; i<signed_map_list.length; ++i) {
+        var map_id = signed_map_list[i];
+        var map_sig_key = _.template(that.key_map_sig, {signer:signer, map_id:map_id});
+        //console.log("Queuing removal of '" + crt_id + "' from '" + map_sig_key + "'");
+        tx.SREM( map_sig_key, crt_id )
+      }
+      tx.EXEC(this);
+    },
+    function reportTransaction(err, rets) {
+      if ( err ) throw err;
+      if ( debug ) {
+        for (var i=0; i<signed_map_list.length; ++i) {
+          var ret = rets[i];
+          if ( ! ret ) {
+            console.log("No signature with certificate '" + crt_id
+              + "' of user '" + signer + "' found in map '"
+              + signed_map_list[i] + "'");
+          } else {
+            console.log("Signature with certificate '" + crt_id
+              + "' of user '" + signer + "' removed from map '"
+              + signed_map_list[i] + "'");
+          }
+        }
+      }
+      return null;
+    },
+    function finish(err) {
+      if ( ! _.isUndefined(redis_client) ) {
+        that._releaseRedis(redis_client);
+      }
+      callback(err);
+    }
+  );
+};
+
+// Sign a map with a certificate reference
+//
+// @param signer a signer name (cartodb username)
+// @param map_id a layergroup_id
+// @param crt_id signature certificate identifier
+//
+// @param callback function(Error) 
+//
+o.signMap = function(signer, map_id, crt_id, callback) {
+  var that = this;
+  Step(
+    function addMapSignature() {
+      var map_sig_key = _.template(that.key_map_sig, {signer:signer, map_id:map_id});
+      if ( debug ) {
+        console.log("Adding " + crt_id + " to " + map_sig_key);
+      }
+      that._redisCmd('SADD', [ map_sig_key, crt_id ], this);
+    },
+    function addCertificateUsage(err) {
+      // Add the map to the set of maps signed by the given cert
+      if ( err ) throw err;
+      var crt_sig_key = _.template(that.key_crt_sig, {signer:signer, crt_id:crt_id});
+      that._redisCmd('SADD', [ crt_sig_key, map_id ], this);
+    },
+    function finish(err) {
+      callback(err);
+    }
+  );
+};
+
+// Sign a map with a full certificate
+//
+// @param signer a signer name (cartodb username)
+// @param map_id a layergroup_id
+// @param cert_id signature certificate identifier
+//
+// @param callback function(Error, String) return certificate id
+//
+o.addSignature = function(signer, map_id, cert, callback) {
+  var that = this;
+  var certificate_id;
+  Step(
+    function addCertificate() {
+      that.addCertificate(signer, cert, this);
+    },
+    function signMap(err, cert_id) {
+      if ( err ) throw err;
+      if ( ! cert_id ) throw new Error("addCertificate returned no certificate id");
+      certificate_id = cert_id;
+      that.signMap(signer, map_id, cert_id, this);
+    },
+    function finish(err) {
+      callback(err, certificate_id);
+    }
+  );
+};
+
+module.exports = SignedMaps;

lib/cartodb/sql/sql_api.js

@@ -0,0 +1,66 @@
+var _       = require('underscore'),
+    request = require('request');
+
+module.exports.query = function (username, api_key, sql, callback) {
+    var api = global.environment.sqlapi;
+
+    // build up api string
+    var sqlapihostname = username;
+    if ( api.domain ) sqlapihostname += '.' + api.domain;
+
+    var sqlapi = api.protocol + '://';
+    if ( api.host && api.host != api.domain ) sqlapi += api.host;
+    else sqlapi += sqlapihostname;
+    sqlapi += ':' + api.port + '/api/' + api.version + '/sql';
+
+    var qs  = { q: sql };
+
+    // add api_key if given
+    if (_.isString(api_key) && api_key != '') { qs.api_key = api_key; }
+
+    // call sql api
+    //
+    // NOTE: using POST to avoid size limits:
+    // See http://github.com/CartoDB/Windshaft-cartodb/issues/111
+    //
+    // NOTE: uses "host" header to allow IP based specification
+    //       of sqlapi address (and avoid a DNS lookup)
+    //
+    // NOTE: allows for keeping up to "maxConnections" concurrent
+    //       sockets opened per SQL-API host.
+    // See http://nodejs.org/api/http.html#http_agent_maxsockets
+    //
+    var maxSockets = global.environment.maxConnections || 128;
+    var maxGetLen = api.max_get_sql_length || 2048;
+    var maxSQLTime = api.timeout || 100; // 1/10 of a second by default
+    var reqSpec = {
+        url:sqlapi,
+        json:true,
+        headers:{host: sqlapihostname}
+        // http://nodejs.org/api/http.html#http_agent_maxsockets
+        ,pool:{maxSockets:maxSockets}
+        // timeout in milliseconds
+        ,timeout:maxSQLTime
+    };
+    if ( sql.length > maxGetLen ) {
+        reqSpec.method = 'POST';
+        reqSpec.body = qs;
+    } else {
+        reqSpec.method = 'GET';
+        reqSpec.qs = qs;
+    }
+    request(reqSpec, function(err, res, body) {
+        if (err){
+            console.log('ERROR connecting to SQL API on ' + sqlapi + ': ' + err);
+            callback(err);
+            return;
+        }
+        if (res.statusCode != 200) {
+            var msg = res.body.error ? res.body.error : res.body;
+            callback(new Error(msg));
+            console.log('unexpected response status (' + res.statusCode + ') for sql query: ' + sql + ': ' + msg);
+            return;
+        }
+        callback(null, body.rows);
+    });
+};

lib/cartodb/template_maps.js

@@ -0,0 +1,608 @@
+var crypto  = require('crypto'),
+    Step    = require('step'),
+    _       = require('underscore'),
+    dot     = require('dot');
+
+// Class handling map templates
+//
+// See http://github.com/CartoDB/Windshaft-cartodb/wiki/Template-maps
+//
+// @param redis_pool an instance of a "redis-mpool"
+//        See https://github.com/CartoDB/node-redis-mpool
+//        Needs version 0.x.x of the API.
+//
+// @param signed_maps an instance of a "signed_maps" class,
+//        See signed_maps.js
+//
+// @param opts TemplateMap options. Supported elements:
+//    'max_user_templates' limit on the number of per-user
+//
+// 
+function TemplateMaps(redis_pool, signed_maps, opts) {
+  this.redis_pool = redis_pool;
+  this.signed_maps = signed_maps;
+  this.opts = opts || {};
+
+  // Database containing templates
+  // TODO: allow configuring ?
+  // NOTE: currently it is the same as
+  //       the one containing layergroups
+  this.db_signatures = 0;
+
+  //
+  // Map templates are owned by a user that specifies access permissions
+  // for their instances.
+  // 
+  // We have the following datastores:
+  //
+  //  1. User templates: set of per-user map templates
+  //     NOTE: each template would have an associated auth
+  //           reference, see signed_maps.js
+
+  // User templates (HASH:tpl_id->tpl_val)
+  this.key_usr_tpl = dot.template("map_tpl|{{=it.owner}}");
+
+  // User template locks (HASH:tpl_id->ctime)
+  this.key_usr_tpl_lck = dot.template("map_tpl|{{=it.owner}}|locks");
+
+  this.lock_ttl = this.opts['lock_ttl'] || 5000;
+}
+
+var o = TemplateMaps.prototype;
+
+//--------------- PRIVATE METHODS --------------------------------
+
+o._userTemplateLimit = function() {
+  return this.opts['max_user_templates'] || 0;
+};
+
+o._acquireRedis = function(callback) {
+  this.redis_pool.acquire(this.db_signatures, callback);
+};
+
+o._releaseRedis = function(client) {
+  this.redis_pool.release(this.db_signatures, client);
+};
+
+/**
+ * Internal function to communicate with redis
+ *
+ * @param redisFunc - the redis function to execute
+ * @param redisArgs - the arguments for the redis function in an array
+ * @param callback - function to pass results too.
+ */
+o._redisCmd = function(redisFunc, redisArgs, callback) {
+  var redisClient;
+  var that = this;
+  var db = that.db_signatures;
+
+  Step(
+    function getRedisClient() {
+      that.redis_pool.acquire(db, this);
+    },
+    function executeQuery(err, data) {
+      if ( err ) throw err;
+      redisClient = data;
+      redisArgs.push(this);
+      redisClient[redisFunc.toUpperCase()].apply(redisClient, redisArgs);
+    },
+    function releaseRedisClient(err, data) {
+      if ( ! _.isUndefined(redisClient) ) that.redis_pool.release(db, redisClient);
+      callback(err, data);
+    }
+  );
+};
+
+// @param callback function(err, obtained)
+o._obtainTemplateLock = function(owner, tpl_id, callback) {
+    var that = this,
+        lockKey = this.key_usr_tpl_lck({owner:owner});
+    Step (
+        function obtainLock() {
+            that._redisCmd('HGET', [lockKey, tpl_id], this);
+        },
+        function checkLock(err, lockTime) {
+            if (err) { throw err; }
+
+            var _newLockTime = Date.now();
+            if (!lockTime || ((_newLockTime - lockTime) > that.lock_ttl)) {
+                that._redisCmd('HSET', [lockKey, tpl_id, _newLockTime], this);
+            } else {
+                throw new Error("Template '" + tpl_id + "' of user '" + owner + "' is locked");
+            }
+        },
+        function finish(err, hsetValue) {
+            callback(err, !!hsetValue);
+        }
+    );
+};
+
+// @param callback function(err, deleted)
+o._releaseTemplateLock = function(owner, tpl_id, callback) {
+    this._redisCmd('HDEL', [this.key_usr_tpl_lck({owner:owner}), tpl_id], callback);
+};
+
+var _reValidIdentifier = /^[a-zA-Z][0-9a-zA-Z_]*$/;
+o._checkInvalidTemplate = function(template) {
+  if ( template.version != '0.0.1' ) {
+    return new Error("Unsupported template version " + template.version);
+  }
+  var tplname = template.name;
+  if ( ! tplname ) {
+    return new Error("Missing template name");
+  }
+  if ( ! tplname.match(_reValidIdentifier) ) {
+    return new Error("Invalid characters in template name '" + tplname + "'");
+  }
+
+  var placeholders = template.placeholders || {};
+
+  var placeholderKeys = Object.keys(placeholders);
+  for (var i = 0, len = placeholderKeys.length; i < len; i++) {
+      var placeholderKey = placeholderKeys[i];
+
+      if (!placeholderKey.match(_reValidIdentifier)) {
+          return new Error("Invalid characters in placeholder name '" + placeholderKey + "'");
+      }
+      if ( ! placeholders[placeholderKey].hasOwnProperty('default') ) {
+          return new Error("Missing default for placeholder '" + placeholderKey + "'");
+      }
+      if ( ! placeholders[placeholderKey].hasOwnProperty('type') ) {
+          return new Error("Missing type for placeholder '" + placeholderKey + "'");
+      }
+  }
+
+  // Check certificate validity
+  var cert = this.getTemplateCertificate(template);
+  var err = this.signed_maps.checkInvalidCertificate(cert);
+  if ( err ) return err;
+
+  // TODO: run more checks over template format ?
+};
+
+//--------------- PUBLIC API -------------------------------------
+
+// Extract a signature certificate from a template
+//
+// The certificate will be ready to be passed to
+// SignedMaps.addCertificate or SignedMaps.authorizedByCert
+//
+o.getTemplateCertificate = function(template) {
+  return {
+      version: '0.0.1',
+      template_id: template.name,
+      auth: template.auth
+  };
+};
+
+// Add a template
+//
+// NOTE: locks user+template_name or fails
+//
+// @param owner cartodb username of the template owner
+//
+// @param template layergroup template, see
+//        http://github.com/CartoDB/Windshaft-cartodb/wiki/Template-maps#template-format
+//
+// @param callback function(err, tpl_id) 
+//        Return template identifier (only valid for given user)
+//
+o.addTemplate = function(owner, template, callback) {
+  var invalidError = this._checkInvalidTemplate(template);
+  if ( invalidError ) {
+    callback(invalidError);
+    return;
+  }
+  var tplname = template.name;
+
+  // Procedure:
+  //
+  // - Check against limit
+  // 0. Obtain a lock for user+template_name, fail if impossible
+  // 1. Check no other template exists with the same name
+  // 2. Install certificate extracted from template, extending
+  //    it to contain a name to properly salt things out.
+  // 3. Modify the template object to reference certificate by id
+  // 4. Install template
+  // 5. Release lock
+  //
+  //
+
+  var usr_tpl_key = this.key_usr_tpl({owner:owner});
+  var gotLock = false;
+  var that = this;
+  var limit = that._userTemplateLimit();
+  Step(
+    function checkLimit() {
+      if ( ! limit ) return 0;
+      that._redisCmd('HLEN', [ usr_tpl_key ], this);
+    },
+    // try to obtain a lock
+    function obtainLock(err, len) {
+      if ( err ) throw err;
+      if ( limit && len >= limit ) {
+        throw new Error("User '" + owner + "' reached limit on number of templates (" + len + "/" + limit + ")");
+      }
+      that._obtainTemplateLock(owner, tplname, this);
+    },
+    function getExistingTemplate(err, locked) {
+      if ( err ) throw err;
+      if ( ! locked ) {
+        // Already locked
+        throw new Error("Template '" + tplname + "' of user '" + owner + "' is locked");
+      }
+      gotLock = true;
+      that._redisCmd('HEXISTS', [ usr_tpl_key, tplname ], this);
+    },
+    function installCertificate(err, exists) {
+      if ( err ) throw err;
+      if ( exists ) {
+        throw new Error("Template '" + tplname + "' of user '" + owner + "' already exists"); 
+      }
+      var cert = that.getTemplateCertificate(template);
+      that.signed_maps.addCertificate(owner, cert, this);
+    },
+    function installTemplate(err, crt_id) {
+      if ( err ) throw err;
+      template.auth_id = crt_id;
+      var tpl_val = JSON.stringify(template);
+      that._redisCmd('HSET', [ usr_tpl_key, tplname, tpl_val ], this);
+    },
+    function releaseLock(err, newfield) {
+      if ( ! err && ! newfield ) {
+        console.log("ERROR: addTemplate overridden existing template '"
+          + tplname + "' of '" + owner
+          + "' -- HSET returned " + overridden + ": someone added it without locking ?");
+        // TODO: how to recover this ?!
+      }
+
+      if ( err && ! gotLock ) throw err;
+
+      // release the lock
+      var next = this;
+      that._releaseTemplateLock(owner, tplname, function(e, d) {
+        if ( e ) {
+          console.log("Error removing lock on template '" + tplname
+            + "' of user '" + owner + "': " + e);
+        } else if ( ! d ) {
+          console.log("ERROR: lock on template '" + tplname
+            + "' of user '" + owner + "' externally removed during insert!");
+        }
+        next(err);
+      });
+    },
+    function finish(err) {
+      callback(err, tplname);
+    }
+  );
+};
+
+// Delete a template
+//
+// NOTE: locks user+template_name or fails
+//
+// Also deletes associated authentication certificate, which
+// in turn deletes all instance signatures
+//
+// @param owner cartodb username of the template owner
+//
+// @param tpl_id template identifier as returned
+//        by addTemplate or listTemplates
+//
+// @param callback function(err)
+//
+o.delTemplate = function(owner, tpl_id, callback) {
+  var usr_tpl_key = this.key_usr_tpl({owner:owner});
+  var gotLock = false;
+  var that = this;
+  Step(
+    // try to obtain a lock
+    function obtainLock() {
+      that._obtainTemplateLock(owner, tpl_id, this);
+    },
+    function getExistingTemplate(err, locked) {
+      if ( err ) throw err;
+      if ( ! locked ) {
+        // Already locked
+        throw new Error("Template '" + tpl_id + "' of user '" + owner + "' is locked");
+      }
+      gotLock = true;
+      that._redisCmd('HGET', [ usr_tpl_key, tpl_id ], this);
+    },
+    function delCertificate(err, tplval) {
+      if ( err ) throw err;
+      if ( ! tplval ) {
+        throw new Error("Template '" + tpl_id + "' of user '" + owner + "' does not exist"); 
+      }
+      var tpl = JSON.parse(tplval);
+      if ( ! tpl.auth_id ) {
+        // not sure this is an error, in case we'll ever
+        // allow unsigned templates...
+        console.log("ERROR: installed template '" + tpl_id
+            + "' of user '" + owner + "' has no auth_id reference: "); console.dir(tpl);
+        return null;
+      }
+      var next = this;
+      that.signed_maps.delCertificate(owner, tpl.auth_id, function(err) {
+        if ( err ) {
+          var msg = "ERROR: could not delete certificate '"
+                  + tpl.auth_id + "' associated with template '"
+                  + tpl_id + "' of user '" + owner + "': " + err;
+          // I'm actually not sure we want this event to be fatal
+          // (avoiding a deletion of the template itself) 
+          next(new Error(msg));
+        } else {
+          next();
+        }
+      });
+    },
+    function delTemplate(err) {
+      if ( err ) throw err;
+      that._redisCmd('HDEL', [ usr_tpl_key, tpl_id ], this);
+    },
+    function releaseLock(err, deleted) {
+      if ( ! err && ! deleted ) {
+          console.log("ERROR: template '" + tpl_id
+            + "' of user '" + owner + "' externally removed during delete!");
+      }
+
+      if ( ! gotLock ) {
+        if ( err ) throw err;
+        return null;
+      }
+
+      // release the lock
+      var next = this;
+      that._releaseTemplateLock(owner, tpl_id, function(e, d) {
+        if ( e ) {
+          console.log("Error removing lock on template '" + tpl_id
+            + "' of user '" + owner + "': " + e);
+        } else if ( ! d ) {
+          console.log("ERROR: lock on template '" + tpl_id
+            + "' of user '" + owner + "' externally removed during delete!");
+        }
+        next(err);
+      });
+    },
+    function finish(err) {
+      callback(err);
+    }
+  );
+};
+
+// Update a template
+//
+// NOTE: locks user+template_name or fails
+//
+// Also deletes and re-creates associated authentication certificate,
+// which in turn deletes all instance signatures
+//
+// @param owner cartodb username of the template owner
+//
+// @param tpl_id template identifier as returned by addTemplate
+//
+// @param template layergroup template, see
+//        http://github.com/CartoDB/Windshaft-cartodb/wiki/Template-maps#template-format
+//
+// @param callback function(err)
+//        
+o.updTemplate = function(owner, tpl_id, template, callback) {
+
+  var invalidError = this._checkInvalidTemplate(template);
+  if ( invalidError ) {
+    callback(invalidError);
+    return;
+  }
+
+  var tplname = template.name;
+
+  if ( tpl_id != tplname ) {
+    callback(new Error("Cannot update name of a map template ('" + tpl_id + "' != '" + tplname + "')"));
+    return;
+  }
+
+  var usr_tpl_key = this.key_usr_tpl({owner:owner});
+  var gotLock = false;
+  var that = this;
+  Step(
+    // try to obtain a lock
+    function obtainLock() {
+      that._obtainTemplateLock(owner, tpl_id, this);
+    },
+    function getExistingTemplate(err, locked) {
+      if ( err ) throw err;
+      if ( ! locked ) {
+        // Already locked
+        throw new Error("Template '" + tpl_id + "' of user '" + owner + "' is locked");
+      }
+      gotLock = true;
+      that._redisCmd('HGET', [ usr_tpl_key, tpl_id ], this);
+    },
+    function delOldCertificate(err, tplval) {
+      if ( err ) throw err;
+      if ( ! tplval ) {
+        throw new Error("Template '" + tpl_id + "' of user '"
+                         + owner +"' does not exist");
+      }
+      var tpl = JSON.parse(tplval);
+      if ( ! tpl.auth_id ) {
+        // not sure this is an error, in case we'll ever
+        // allow unsigned templates...
+        console.log("ERROR: installed template '" + tpl_id
+            + "' of user '" + owner + "' has no auth_id reference: "); console.dir(tpl);
+        return null;
+      }
+      var next = this;
+      that.signed_maps.delCertificate(owner, tpl.auth_id, function(err) {
+        if ( err ) {
+          var msg = "ERROR: could not delete certificate '"
+                  + tpl.auth_id + "' associated with template '"
+                  + tpl_id + "' of user '" + owner + "': " + err;
+          // I'm actually not sure we want this event to be fatal
+          // (avoiding a deletion of the template itself) 
+          next(new Error(msg));
+        } else {
+          next();
+        }
+      });
+    },
+    function installNewCertificate(err) {
+      if ( err ) throw err;
+      var cert = that.getTemplateCertificate(template);
+      that.signed_maps.addCertificate(owner, cert, this);
+    },
+    function updTemplate(err, crt_id) {
+      if ( err ) throw err;
+      template.auth_id = crt_id;
+      var tpl_val = JSON.stringify(template);
+      that._redisCmd('HSET', [ usr_tpl_key, tplname, tpl_val ], this);
+    },
+    function releaseLock(err, newfield) {
+      if ( ! err && newfield ) {
+          console.log("ERROR: template '" + tpl_id
+            + "' of user '" + owner + "' externally removed during update!");
+      }
+
+      if ( ! gotLock ) {
+        if ( err ) throw err;
+        return null;
+      }
+
+      // release the lock
+      var next = this;
+      that._releaseTemplateLock(owner, tpl_id, function(e, d) {
+        if ( e ) {
+          console.log("Error removing lock on template '" + tpl_id
+            + "' of user '" + owner + "': " + e);
+        } else if ( ! d ) {
+          console.log("ERROR: lock on template '" + tpl_id
+            + "' of user '" + owner + "' externally removed during update!");
+        }
+        next(err);
+      });
+    },
+    function finish(err) {
+      callback(err);
+    }
+  );
+};
+
+// List user templates
+//
+// @param owner cartodb username of the templates owner
+//
+// @param callback function(err, tpl_id_list)
+//        Returns a list of template identifiers
+// 
+o.listTemplates = function(owner, callback) {
+    this._redisCmd('HKEYS', [ this.key_usr_tpl({owner:owner}) ], callback);
+};
+
+// Get a templates
+//
+// @param owner cartodb username of the template owner
+//
+// @param tpl_id template identifier as returned
+//        by addTemplate or listTemplates
+//
+// @param callback function(err, template)
+//        Return full template definition
+//
+o.getTemplate = function(owner, tpl_id, callback) {
+  var that = this;
+  Step(
+    function getTemplate() {
+      that._redisCmd('HGET', [ that.key_usr_tpl({owner:owner}), tpl_id ], this);
+    },
+    function parseTemplate(err, tpl_val) {
+      if ( err ) throw err;
+      // Should we strip auth_id ?
+      return JSON.parse(tpl_val);
+    },
+    function finish(err, tpl) {
+      callback(err, tpl);
+    }
+  );
+};
+
+// Perform placeholder substitutions on a template
+//
+// @param template a template object (will not be modified)
+//
+// @param params an object containing named subsitution parameters
+//        Only the ones found in the template's placeholders object
+//        will be used, with missing ones taking default values.
+//
+// @returns a layergroup configuration 
+//
+// @throws Error on malformed template or parameter
+//
+var _reNumber = /^([-+]?[\d\.]?\d+([eE][+-]?\d+)?)$/,
+    _reCSSColorName = /^[a-zA-Z]+$/,
+    _reCSSColorVal = /^#[0-9a-fA-F]{3,6}$/;
+
+_replaceVars = function(str, params) {
+  //return _.template(str, params); // lazy way, possibly dangerous 
+  // Construct regular expressions for each param
+    Object.keys(params).forEach(function(k) {
+        str = str.replace(new RegExp("<%=\\s*" + k + "\\s*%>", "g"), params[k]);
+    });
+    return str;
+};
+o.instance = function(template, params) {
+  var all_params = {};
+  var phold = template.placeholders || {};
+  Object.keys(phold).forEach(function(k) {
+    var val = params.hasOwnProperty(k) ? params[k] : phold[k].default;
+    var type = phold[k].type;
+    // properly escape
+    if ( type === 'sql_literal' ) {
+      // duplicate any single-quote
+      val = val.replace(/'/g, "''");
+    }
+    else if ( type === 'sql_ident' ) {
+      // duplicate any double-quote
+      val = val.replace(/"/g, '""');
+    }
+    else if ( type === 'number' ) {
+      // check it's a number
+      if ( typeof(val) !== 'number' && ! val.match(_reNumber) ) {
+        throw new Error("Invalid number value for template parameter '"
+          + k + "': " + val);
+      }
+    }
+    else if ( type === 'css_color' ) {
+      // check it only contains letters or
+      // starts with # and only contains hexdigits
+      if ( ! val.match(_reCSSColorName) && ! val.match(_reCSSColorVal) ) {
+        throw new Error("Invalid css_color value for template parameter '"
+          + k + "': " + val);
+      }
+    }
+    else {
+      // NOTE: should be checked at template create/update time
+      throw new Error("Invalid placeholder type '" + type + "'");
+    }
+    all_params[k] = val;
+  });
+
+  // NOTE: we're deep-cloning the layergroup here
+  var layergroup = JSON.parse(JSON.stringify(template.layergroup));
+  for (var i=0; i<layergroup.layers.length; ++i) {
+    var lyropt = layergroup.layers[i].options;
+    if ( lyropt.cartocss ) lyropt.cartocss = _replaceVars(lyropt.cartocss, all_params);
+    if ( lyropt.sql) lyropt.sql = _replaceVars(lyropt.sql, all_params);
+    // Anything else ?
+  }
+  return layergroup;
+};
+
+// Return a fingerPrint of the object
+o.fingerPrint = function(template) {
+  return crypto.createHash('md5')
+    .update(JSON.stringify(template))
+    .digest('hex')
+  ;
+};
+
+module.exports = TemplateMaps;

package.json

@@ -1,39 +1,51 @@
 {
     "private": true,
     "name": "windshaft-cartodb",
-    "version": "0.1.0",
+    "version": "1.21.0",
     "description": "A map tile server for CartoDB",
-    "url": "https://github.com/Vizzuality/Windshaft-cartodb",
+    "keywords": [
+      "cartodb"
+    ],
+    "url": "https://github.com/CartoDB/Windshaft-cartodb",
     "licenses": [{
         "type": "BSD",
-        "url": "https://github.com/Vizzuality/Windshaft-cartodb/blob/master/LICENCE"
+        "url": "https://github.com/CartoDB/Windshaft-cartodb/blob/master/LICENCE"
     }],
-    "repositories": [{
+    "repository": {
         "type": "git",
-        "url": "git://github.com/Vizzuality/Windshaft-cartodb.git"
-    }],
-    "author": {
-      "name": "Simon Tokumine, Javi Santana, Vizzuality",
-      "url": "http://vizzuality.com",
-      "email": "simon@vizzuality.com"
+        "url": "git://github.com/CartoDB/Windshaft-cartodb.git"
     },
+    "author": "Vizzuality <contact@vizzuality.com> (http://vizzuality.com)",
+    "contributors": [
+      "Simon Tokumine <simon@vizzuality.com>",
+      "Javi Santana <jsantana@vizzuality.com>",
+      "Sandro Santilli <strk@vizzuality.com>"
+    ],
     "dependencies": {
-        "connect": "1.8.7",
-        "cluster": "0.6.4",
-        "node-varnish": "0.1.1",
-        "underscore"  : "1.1.x",
-        "grainstore"  : "~0.6.2",
-        "windshaft"  : "~0.4.14",
-        "step": "0.0.x",
-        "generic-pool": "1.0.x",
-        "redis": "0.7.2",
-        "hiredis": "~0.1.12",
-        "request": "2.9.202"
+        "node-varnish": "https://github.com/Vizzuality/node-varnish/tarball/0.3.0",
+        "underscore"  : "~1.6.0",
+        "dot": "~1.0.2",
+        "windshaft": "https://github.com/CartoDB/Windshaft/tarball/0.31.0",
+        "step": "~0.0.5",
+        "request": "~2.9.203",
+        "cartodb-redis": "https://github.com/CartoDB/node-cartodb-redis/tarball/0.11.0",
+        "cartodb-psql": "https://github.com/CartoDB/node-cartodb-psql/tarball/0.4.0",
+        "redis-mpool": "https://github.com/CartoDB/node-redis-mpool/tarball/0.3.0",
+        "lzma": "~1.3.7",
+        "log4js": "~0.6.17",
+        "rollbar": "~0.3.13"
     },
     "devDependencies": {
-        "mocha": "1.2.1"
+        "mocha": "~1.21.4",
+        "redis": "~0.8.6",
+        "strftime": "~0.8.2",
+        "semver": "~1.1.4"
     },
     "scripts": {
         "test": "make check"
+    },
+    "engines": {
+        "node": ">=0.8 <0.11",
+        "npm": ">=1.2.1"
     }
 }

run_tests.sh

@@ -1,11 +1,35 @@
 #!/bin/sh
 
-# Must match config.redis_pool.port in test/support/config.js
-REDIS_PORT=6333
+OPT_CREATE_REDIS=yes # create the redis test environment
+OPT_CREATE_PGSQL=yes # create the PostgreSQL test environment
+OPT_DROP_REDIS=yes   # drop the redis test environment
+OPT_DROP_PGSQL=yes   # drop the PostgreSQL test environment
+
+export PGAPPNAME=cartodb_tiler_tester
+
+cd $(dirname $0)
+BASEDIR=$(pwd)
+cd -
+
+REDIS_PORT=`node -e "console.log(require('${BASEDIR}/config/environments/test.js').redis.port)"`
+export REDIS_PORT
 
 cleanup() {
-	echo "Cleaning up"
-	kill ${PID_REDIS}
+  if test x"$OPT_DROP_REDIS" = xyes; then
+    if test x"$PID_REDIS" = x; then
+      PID_REDIS=$(cat ${BASEDIR}/redis.pid)
+      if test x"$PID_REDIS" = x; then
+        echo "Could not find a test redis pid to kill it"
+        return;
+      fi
+    fi
+    echo "Killing test redis pid ${PID_REDIS}"
+    kill ${PID_REDIS}
+  fi
+  if test x"$OPT_DROP_PGSQL" = xyes; then
+    # TODO: drop postgresql ?
+    echo "Dropping PostgreSQL test database isn't implemented yet"
+  fi
 }
 
 cleanup_and_exit() {
@@ -22,21 +46,76 @@ die() {
 
 trap 'cleanup_and_exit' 1 2 3 5 9 13
 
-echo "Starting redis on port ${REDIS_PORT}"
-echo "port ${REDIS_PORT}" | redis-server - > test.log &
-PID_REDIS=$!
+while [ -n "$1" ]; do
+        # This is kept for backward compatibility
+        if test "$1" = "--nodrop"; then
+                OPT_DROP_REDIS=no
+                OPT_DROP_PGSQL=no
+                shift
+                continue
+        elif test "$1" = "--nodrop-pg"; then
+                OPT_DROP_PGSQL=no
+                shift
+                continue
+        elif test "$1" = "--nodrop-redis"; then
+                OPT_DROP_REDIS=no
+                shift
+                continue
+        elif test "$1" = "--nocreate-pg"; then
+                OPT_CREATE_PGSQL=no
+                shift
+                continue
+        elif test "$1" = "--nocreate-redis"; then
+                OPT_CREATE_REDIS=no
+                shift
+                continue
+        # This is kept for backward compatibility
+        elif test "$1" = "--nocreate"; then
+                OPT_CREATE_REDIS=no
+                OPT_CREATE_PGSQL=no
+                shift
+                continue
+        else
+                break
+        fi
+done
 
-echo "Preparing the database"
-cd test; sh prepare_db.sh >> test.log || die "database preparation failure (see test.log)"; cd -;
+if [ -z "$1" ]; then
+        echo "Usage: $0 [<options>] <test> [<test>]" >&2
+        echo "Options:" >&2
+        echo " --nocreate   do not create the test environment on start" >&2
+        echo " --nodrop     do not drop the test environment on exit" >&2
+        exit 1
+fi
+
+TESTS=$@
+
+if test x"$OPT_CREATE_REDIS" = xyes; then
+  echo "Starting redis on port ${REDIS_PORT}"
+  echo "port ${REDIS_PORT}" | redis-server - > ${BASEDIR}/test.log &
+  PID_REDIS=$!
+  echo ${PID_REDIS} > ${BASEDIR}/redis.pid
+fi
+
+PREPARE_DB_OPTS=
+if test x"$OPT_CREATE_PGSQL" != xyes; then
+  PREPARE_DB_OPTS="$PREPARE_DB_OPTS --skip-pg"
+fi
+if test x"$OPT_CREATE_REDIS" != xyes; then
+  PREPARE_DB_OPTS="$PREPARE_DB_OPTS --skip-redis"
+fi
+
+echo "Preparing the environment"
+cd ${BASEDIR}/test/support
+sh prepare_db.sh ${PREPARE_DB_OPTS} || die "database preparation failure"
+cd -
 
 PATH=node_modules/.bin/:$PATH
 
 echo "Running tests"
-mocha -u tdd \
-  test/unit/cartodb/redis_pool.test.js \
-  test/unit/cartodb/req2params.test.js \
-  test/acceptance/cache_validator.js \
-  test/acceptance/server.js
-
+mocha -t 10000 -u tdd ${MOCHA_OPTS} ${TESTS}
+ret=$?
 
 cleanup
+
+exit $ret

test/acceptance/cache_validator.js

@@ -1,31 +1,8 @@
 var assert      = require('../support/assert');
-var net = require('net');
 require(__dirname + '/../support/test_helper');
 var CacheValidator = require(__dirname + '/../../lib/cartodb/cache_validator');
-var tests = module.exports = {};
 
-function VarnishEmu(on_cmd_recieved, test_callback) {
-    var self = this;
-    var welcome_msg = 'hi, im a varnish emu, right?';
-
-    self.commands_recieved = [];
-
-    var server = net.createServer(function (socket) {
-      var command = '';
-      socket.write("200 " + welcome_msg.length + "\n");
-      socket.write(welcome_msg);
-      socket.on('data', function(data) {        
-        self.commands_recieved.push(data);
-        on_cmd_recieved && on_cmd_recieved(self.commands_recieved);
-        socket.write('200 0\n');
-      });
-    });
-    server.listen(1337, "127.0.0.1");
-    
-    server.on('listening', function(){
-      test_callback();
-    });    
-}
+var VarnishEmu = require('../support/VarnishEmu');
 
 suite('cache_validator', function() {
 

test/fixtures/test_multilayer_bbox.grid.json

@@ -0,0 +1 @@
+{"grid":["                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","        !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"],"keys":["","1"],"data":{"1":{"cartodb_id":1}}}

test/fixtures/test_table_0_0_0_multilayer1.layer0.grid.json

@@ -0,0 +1 @@
+{"grid":["                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                      !!!                       ","                                     !!!!!!                     ","                                    !!!!!!!                     ","                                    !!!!!!!!                    ","                                    !!!!!!!!                    ","                                    !!!!!!!!                    ","                                    !!!!!!!                     ","                                     !!!!!!                     ","                                      !!!                       ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                "],"keys":["","2"],"data":{"2":{"cartodb_id":2}}}

test/fixtures/test_table_0_0_0_multilayer1.layer1.grid.json

@@ -0,0 +1 @@
+{"grid":["                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                       !!                                       ","                      !!!                                       ","                       !!                                       ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                ","                                                                "],"keys":["","2"],"data":{"2":{"cartodb_id":4}}}

test/prepare_db.sh

@@ -1,36 +0,0 @@
-#!/bin/sh
-
-# this script prepare database and redis instance to run accpetance test
-#
-# NOTE: assumes existance of a "template_postgis"
-# NOTE2: use PG* environment variables to control who and where
-#
-# NOTE3: a side effect of the db preparation is the persistent creation
-#        of two database roles which will be valid for the whole cluster
-#        TODO: fix that
-#
-
-die() {
-        msg=$1
-        echo "${msg}" >&2
-        exit 1
-}
-
-TEST_DB="cartodb_test_user_1_db"
-REDIS_PORT=6333
-
-echo "preparing postgres..."
-dropdb "${TEST_DB}"
-createdb -Ttemplate_postgis -EUTF8 "${TEST_DB}" || die "Could not create test database"
-psql "${TEST_DB}" < ./sql/windshaft.test.sql
-psql "${TEST_DB}" < ./sql/gadm4.sql
-
-echo "preparing redis..."
-echo "HSET rails:users:vizzuality id 1" | redis-cli -p ${REDIS_PORT} -n 5
-echo 'HSET rails:users:vizzuality database_name "'"${TEST_DB}"'"' | redis-cli -p ${REDIS_PORT} -n 5
-echo "HSET rails:users:vizzuality map_key 1234" | redis-cli -p ${REDIS_PORT} -n 5
-echo "SADD rails:users:vizzuality:map_key 1235" | redis-cli -p ${REDIS_PORT} -n 5
-echo 'HSET rails:'"${TEST_DB}"':my_table infowindow "this, that, the other"' | redis-cli -p ${REDIS_PORT} -n 0
-echo 'HSET rails:'"${TEST_DB}"':test_table_private_1 privacy "0"' | redis-cli -p ${REDIS_PORT} -n 0
-
-echo "Finished preparing data. Run tests with expresso."

test/support/SQLAPIEmu.js

@@ -0,0 +1,81 @@
+var http        = require('http');
+var url         = require('url');
+var _ = require('underscore');
+
+var SQLAPIEmulator = function(port, cb) {
+
+  this.queries = [];
+  var that = this;
+  this.requests = [];
+
+  this.sqlapi_server = http.createServer(function(req,res) {
+      //console.log("server got request with method " + req.method);
+      var query;
+
+      that.requests.push(req);
+
+      if ( req.method == 'GET' ) {
+        query = url.parse(req.url, true).query;
+        that.handleQuery(query, res);
+      }
+      else if ( req.method == 'POST') {
+        var data = '';
+        req.on('data', function(chunk) {
+          //console.log("GOT Chunk  " + chunk);
+          data += chunk;
+        });
+        req.on('end', function() {
+          //console.log("Data is: "); console.dir(data);
+          query = JSON.parse(data);
+          //console.log("handleQuery is " + that.handleQuery);
+          that.handleQuery(query, res);
+        });
+      }
+      else {
+        that.handleQuery('SQLAPIEmu does not support method' + req.method, res);
+      }
+   }).listen(port, cb);
+};
+
+SQLAPIEmulator.prototype.handleQuery = function(query, res) {
+    this.queries.push(query);
+    if ( query.q.match('SQLAPIERROR') ) {
+      res.statusCode = 400;
+      res.write(JSON.stringify({'error':'Some error occurred'}));
+    } else if ( query.q.match('SQLAPINOANSWER') ) {
+        console.log("SQLAPIEmulator will never respond, on request");
+        return;
+    } else if (query.q.match('tablenames')) {
+        var tableNames = JSON.stringify(query);
+        res.write(queryResult({tablenames: '{' + tableNames + '}', max: 1234567890.123}));
+    } else if ( query.q.match('EPOCH.* as max') ) {
+      // This is the structure of the known query sent by tiler
+        res.write(queryResult({max: 1234567890.123}));
+    } else {
+      if ( query.q.match('_private_') && query.api_key === undefined) {
+        res.statusCode = 403;
+        res.write(JSON.stringify({'error':'forbidden: ' + JSON.stringify(query)}));
+      } else {
+        var qs = JSON.stringify(query);
+        res.write(queryResult({cdb_querytables: '{' + qs + '}', max: 1234567890.123}));
+      }
+    }
+    res.end();
+  };
+
+SQLAPIEmulator.prototype.close = function(cb) {
+  this.sqlapi_server.close(cb);
+};
+
+SQLAPIEmulator.prototype.getLastRequest = function() {
+  return this.requests.pop();
+};
+
+function queryResult(row) {
+    return JSON.stringify({
+        rows: [row]
+    });
+}
+
+module.exports = SQLAPIEmulator;
+

test/support/VarnishEmu.js

@@ -0,0 +1,25 @@
+var net = require('net');
+
+module.exports = function(on_cmd_recieved, test_callback) {
+    var self = this;
+    var welcome_msg = 'hi, im a varnish emu, right?';
+
+    self.commands_recieved = [];
+
+    var server = net.createServer(function (socket) {
+      var command = '';
+      socket.write("200 " + welcome_msg.length + "\n");
+      socket.write(welcome_msg);
+      socket.on('data', function(data) {
+        self.commands_recieved.push(data);
+        on_cmd_recieved && on_cmd_recieved(self.commands_recieved);
+        socket.write('200 0\n');
+      });
+    });
+    server.listen(1337, "127.0.0.1");
+
+    server.on('listening', function(){
+      test_callback();
+    });
+};
+

test/support/assert.js

@@ -1,36 +1,117 @@
 // Cribbed from the ever prolific Konstantin Kaefer
 // https://github.com/mapbox/tilelive-mapnik/blob/master/test/support/assert.js
 
-var fs = require('fs');
-var http = require('http');
-var path = require('path');
-var exec = require('child_process').exec;
+var exec = require('child_process').exec,
+    fs = require('fs'),
+    http = require('http'),
+    path = require('path'),
+    util = require('util');
 
 var assert = module.exports = exports = require('assert');
 
-assert.imageEqualsFile = function(buffer, file_b, callback) {
+// @param tolerance number of tolerated grid cell differences
+assert.utfgridEqualsFile = function(buffer, file_b, tolerance, callback) {
+    fs.writeFileSync('/tmp/grid.json', buffer, 'binary'); // <-- to debug/update
+    var expected_json = JSON.parse(fs.readFileSync(file_b, 'utf8'));
+
+    var err = null;
+
+    var Celldiff = function(x, y, ev, ov) {
+      this.x = x;
+      this.y = y;
+      this.ev = ev;
+      this.ov = ov;
+    };
+
+    Celldiff.prototype.toString = function() {
+      return '(' + this.x + ',' + this.y + ')["' + this.ev + '" != "' + this.ov + '"]';
+    };
+
+    try {
+      var obtained_json = JSON.parse(buffer);
+
+      // compare grid
+      var obtained_grid = obtained_json.grid;
+      var expected_grid = expected_json.grid;
+      var nrows = obtained_grid.length
+      if (nrows != expected_grid.length) {
+        throw new Error( "Obtained grid rows (" + nrows +
+                    ") != expected grid rows (" + expected_grid.length + ")" );
+      }
+      var celldiff = [];
+      for (var i=0; i<nrows; ++i) {
+        var ocols = obtained_grid[i];
+        var ecols = expected_grid[i];
+        var ncols = ocols.length;
+        if ( ncols != ecols.length ) {
+          throw new Error( "Obtained grid cols (" + ncols +
+                   ") != expected grid cols (" + ecols.length +
+                   ") on row " + i ); 
+        }
+        for (var j=0; j<ncols; ++j) {
+          var ocell = ocols[j];
+          var ecell = ecols[j];
+          if ( ocell !== ecell ) {
+            celldiff.push(new Celldiff(i, j, ecell, ocell));
+          }
+        }
+      }
+
+      if ( celldiff.length > tolerance ) {
+        throw new Error( celldiff.length + " cell differences: " + celldiff );
+      }
+
+      assert.deepEqual(obtained_json.keys, expected_json.keys);
+    } catch (e) { err = e; }
+
+    callback(err);
+};
+
+/**
+ * Takes an image data as an input and an image path and compare them using ImageMagick fuzz algorithm, if case the
+ * similarity is not within the tolerance limit it will callback with an error.
+ *
+ * @param buffer The image data to compare from
+ * @param {string} referenceImageRelativeFilePath The relative file to compare against
+ * @param {number} tolerance tolerated mean color distance, as a per mil (‰)
+ * @param {function} callback Will call to home with null in case there is no error, otherwise with the error itself
+ * @see FUZZY in http://www.imagemagick.org/script/command-line-options.php#metric
+ */
+assert.imageEqualsFile = function(buffer, referenceImageRelativeFilePath, tolerance, callback) {
     if (!callback) callback = function(err) { if (err) throw err; };
-    file_b = path.resolve(file_b);
-    var file_a = '/tmp/' + (Math.random() * 1e16);
-    var err = fs.writeFileSync(file_a, buffer, 'binary');
+    var referenceImageFilePath = path.resolve(referenceImageRelativeFilePath),
+        testImageFilePath = '/tmp/windshaft-test-image-' + (Math.random() * 1e16); // TODO: make predictable
+    var err = fs.writeFileSync(testImageFilePath, buffer, 'binary');
     if (err) throw err;
 
-    exec('compare -metric PSNR "' + file_a + '" "' +
-            file_b + '" /dev/null', function(err, stdout, stderr) {
+    var imageMagickCmd = util.format(
+        'compare -metric fuzz "%s" "%s" /dev/null',
+        testImageFilePath, referenceImageFilePath
+    );
+
+    exec(imageMagickCmd, function(err, stdout, stderr) {
         if (err) {
-            fs.unlinkSync(file_a);
+            fs.unlinkSync(testImageFilePath);
             callback(err);
         } else {
             stderr = stderr.trim();
-            if (stderr === 'inf') {
-                fs.unlinkSync(file_a);
-                callback(null);
-            } else {
-                var similarity = parseFloat(stderr);
-                var err = new Error('Images not equal(' + similarity + '): ' +
-                        file_a  + '    ' + file_b);
+            var metrics = stderr.match(/([0-9]*) \((.*)\)/);
+            if ( ! metrics ) {
+                callback(new Error("No match for " + stderr));
+                return;
+            }
+            var similarity = parseFloat(metrics[2]),
+                tolerancePerMil = (tolerance / 1000);
+            if (similarity > tolerancePerMil) {
+                err = new Error(util.format(
+                        'Images %s and %s are not equal (got %d similarity, expected %d)',
+                        testImageFilePath, referenceImageFilePath, similarity, tolerancePerMil)
+                );
                 err.similarity = similarity;
                 callback(err);
+            } else {
+                fs.unlinkSync(testImageFilePath);
+                callback(null);
             }
         }
     });
@@ -145,6 +226,8 @@ assert.response = function(server, req, res, msg){
             response.on('end', function(){
                 if (timer) clearTimeout(timer);
 
+                check();
+
                 // Assert response body
                 if (res.body !== undefined) {
                     var eql = res.body instanceof RegExp
@@ -190,7 +273,6 @@ assert.response = function(server, req, res, msg){
 
                 // Callback
                 callback(response);
-                check();
             });
         });
 

test/support/config.js

@@ -1,5 +1,8 @@
 var _ = require('underscore');
 
+
+require(__dirname + '/test_helper');
+
 module.exports = function(opts) {
 
     var config = {
@@ -7,7 +10,7 @@ module.exports = function(opts) {
             max: 10, 
             idleTimeoutMillis: 1, 
             reapIntervalMillis: 1,
-            port: 6333 // TODO: read from test env ?
+            port: global.environment.redis.port
         }
     }
 

test/support/prepare_db.sh

@@ -0,0 +1,112 @@
+#!/bin/sh
+
+# this script prepare database and redis instance to run accpetance test
+#
+# NOTE: assumes existance of a "template_postgis"
+# NOTE2: use PG* environment variables to control who and where
+#
+# NOTE3: a side effect of the db preparation is the persistent creation
+#        of two database roles which will be valid for the whole cluster
+#        TODO: fix that
+#
+
+PREPARE_REDIS=yes
+PREPARE_PGSQL=yes
+
+while [ -n "$1" ]; do
+  if test "$1" = "--skip-pg"; then
+    PREPARE_PGSQL=no
+    shift; continue
+  elif test "$1" = "--skip-redis"; then
+    PREPARE_REDIS=no
+    shift; continue
+  fi
+done
+
+die() {
+        msg=$1
+        echo "${msg}" >&2
+        exit 1
+}
+
+# This is where postgresql connection parameters are read from
+TESTENV=../../config/environments/test.js
+if [ \! -r ${TESTENV} ]; then
+  echo "Cannot read ${TESTENV}" >&2
+  exit 1
+fi
+
+TESTUSERID=1
+
+TESTUSER=`node -e "console.log(require('${TESTENV}').postgres_auth_user || '')"`
+if test -z "$TESTUSER"; then
+  echo "Missing postgres_auth_user from ${TESTENV}" >&2
+  exit 1
+fi
+TESTUSER=`echo ${TESTUSER} | sed "s/<%= user_id %>/${TESTUSERID}/"`
+
+TESTPASS=`node -e "console.log(require('${TESTENV}').postgres_auth_pass || 'test')"`
+# TODO: should postgres_auth_pass be optional ?
+if test -z "$TESTPASS"; then
+  echo "Missing postgres_auth_pass from ${TESTENV}" >&2
+  exit 1
+fi
+TESTPASS=`echo ${TESTPASS} | sed "s/<%= user_id %>/${TESTUSERID}/"`
+
+TEST_DB="${TESTUSER}_db"
+
+# NOTE: will be set by caller trough environment
+if test -z "$REDIS_PORT"; then REDIS_PORT=6333; fi
+
+PUBLICUSER=`node -e "console.log(require('${TESTENV}').postgres.user || 'xxx')"`
+PUBLICPASS=`node -e "console.log(require('${TESTENV}').postgres.password || 'xxx')"`
+echo "PUBLICUSER: ${PUBLICUSER}"
+echo "PUBLICPASS: ${PUBLICPASS}"
+echo "TESTUSER: ${TESTUSER}"
+echo "TESTPASS: ${TESTPASS}"
+
+if test x"$PREPARE_PGSQL" = xyes; then
+
+  echo "preparing postgres..."
+  dropdb "${TEST_DB}"
+  createdb -Ttemplate_postgis -EUTF8 "${TEST_DB}" || die "Could not create test database"
+
+  cat sql/windshaft.test.sql sql/gadm4.sql |
+    sed "s/:PUBLICUSER/${PUBLICUSER}/" | 
+    sed "s/:PUBLICPASS/${PUBLICPASS}/" | 
+    sed "s/:TESTUSER/${TESTUSER}/" | 
+    sed "s/:TESTPASS/${TESTPASS}/" | 
+    psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
+
+fi
+
+if test x"$PREPARE_REDIS" = xyes; then
+
+  echo "preparing redis..."
+
+  cat <<EOF | redis-cli -p ${REDIS_PORT} -n 5
+HMSET rails:users:localhost id ${TESTUSERID} \
+                            database_name "${TEST_DB}" \
+                            database_host localhost \
+                            map_key 1234
+SADD rails:users:localhost:map_key 1235
+EOF
+
+  # A user configured as with cartodb-2.5.0+ 
+  cat <<EOF | redis-cli -p ${REDIS_PORT} -n 5
+HMSET rails:users:cartodb250user id ${TESTUSERID} \
+                                 database_name "${TEST_DB}" \
+                                 database_host "localhost" \
+                                 database_password "${TESTPASS}" \
+                                 map_key 4321
+EOF
+
+  cat <<EOF | redis-cli -p ${REDIS_PORT} -n 0
+HSET rails:${TEST_DB}:my_table infowindow "this, that, the other"
+HSET rails:${TEST_DB}:test_table_private_1 privacy "0"
+EOF
+
+fi
+
+echo "Finished preparing data. Ready to run tests"
+

test/support/sql/windshaft.test.sql

@@ -15,11 +15,13 @@ SET search_path = public, pg_catalog;
 SET default_tablespace = '';
 SET default_with_oids = false;
 
--- publicuser role
-CREATE USER publicuser;
+-- public user role
+DROP USER IF EXISTS :PUBLICUSER;
+CREATE USER :PUBLICUSER WITH PASSWORD ':PUBLICPASS';
 
 -- db owner role
-CREATE USER test_cartodb_user_1;
+DROP USER IF EXISTS :TESTUSER;
+CREATE USER :TESTUSER WITH PASSWORD ':TESTPASS';
 
 -- first table
 CREATE TABLE test_table (
@@ -51,19 +53,20 @@ SELECT pg_catalog.setval('test_table_cartodb_id_seq', 60, true);
 
 ALTER TABLE test_table ALTER COLUMN cartodb_id SET DEFAULT nextval('test_table_cartodb_id_seq'::regclass);
 
-INSERT INTO test_table VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241');
-INSERT INTO test_table VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241');
-INSERT INTO test_table VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.324', 3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241');
-INSERT INTO test_table VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.329509', 4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241');
-INSERT INTO test_table VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.334931', 5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241');
+INSERT INTO test_table VALUES
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.324', 3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.329509', 4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.334931', 5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241');
 
 ALTER TABLE ONLY test_table ADD CONSTRAINT test_table_pkey PRIMARY KEY (cartodb_id);
 
 CREATE INDEX test_table_the_geom_idx ON test_table USING gist (the_geom);
 CREATE INDEX test_table_the_geom_webmercator_idx ON test_table USING gist (the_geom_webmercator);
 
-GRANT ALL ON TABLE test_table TO test_cartodb_user_1;
-GRANT SELECT ON TABLE test_table TO publicuser;
+GRANT ALL ON TABLE test_table TO :TESTUSER;
+GRANT SELECT ON TABLE test_table TO :PUBLICUSER;
 
 -- second table
 CREATE TABLE test_table_2 (
@@ -95,19 +98,20 @@ SELECT pg_catalog.setval('test_table_2_cartodb_id_seq', 60, true);
 
 ALTER TABLE test_table_2 ALTER COLUMN cartodb_id SET DEFAULT nextval('test_table_2_cartodb_id_seq'::regclass);
 
-INSERT INTO test_table_2 VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241');
-INSERT INTO test_table_2 VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241');
-INSERT INTO test_table_2 VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.324', 3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241');
-INSERT INTO test_table_2 VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.329509', 4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241');
-INSERT INTO test_table_2 VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.334931', 5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241');
+INSERT INTO test_table_2 VALUES
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.324', 3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.329509', 4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.334931', 5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241');
 
 ALTER TABLE ONLY test_table_2 ADD CONSTRAINT test_table_2_pkey PRIMARY KEY (cartodb_id);
 
 CREATE INDEX test_table_2_the_geom_idx ON test_table_2 USING gist (the_geom);
 CREATE INDEX test_table_2_the_geom_webmercator_idx ON test_table_2 USING gist (the_geom_webmercator);
 
-GRANT ALL ON TABLE test_table_2 TO test_cartodb_user_1;
-GRANT SELECT ON TABLE test_table_2 TO publicuser;
+GRANT ALL ON TABLE test_table_2 TO :TESTUSER;
+GRANT SELECT ON TABLE test_table_2 TO :PUBLICUSER;
 
 -- third table
 CREATE TABLE test_table_3 (
@@ -139,19 +143,20 @@ SELECT pg_catalog.setval('test_table_3_cartodb_id_seq', 60, true);
 
 ALTER TABLE test_table_3 ALTER COLUMN cartodb_id SET DEFAULT nextval('test_table_3_cartodb_id_seq'::regclass);
 
-INSERT INTO test_table_3 VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241');
-INSERT INTO test_table_3 VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241');
-INSERT INTO test_table_3 VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.324', 3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241');
-INSERT INTO test_table_3 VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.329509', 4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241');
-INSERT INTO test_table_3 VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.334931', 5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241');
+INSERT INTO test_table_3 VALUES
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.324', 3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.329509', 4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.334931', 5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241');
 
 ALTER TABLE ONLY test_table_3 ADD CONSTRAINT test_table_3_pkey PRIMARY KEY (cartodb_id);
 
 CREATE INDEX test_table_3_the_geom_idx ON test_table_3 USING gist (the_geom);
 CREATE INDEX test_table_3_the_geom_webmercator_idx ON test_table_3 USING gist (the_geom_webmercator);
 
-GRANT ALL ON TABLE test_table_3 TO test_cartodb_user_1;
-GRANT SELECT ON TABLE test_table_3 TO publicuser;
+GRANT ALL ON TABLE test_table_3 TO :TESTUSER;
+GRANT SELECT ON TABLE test_table_3 TO :PUBLICUSER;
 
 -- private table
 CREATE TABLE test_table_private_1 (
@@ -169,5 +174,6 @@ CREATE TABLE test_table_private_1 (
     CONSTRAINT enforce_srid_the_geom CHECK ((st_srid(the_geom) = 4326)),
     CONSTRAINT enforce_srid_the_geom_webmercator CHECK ((st_srid(the_geom_webmercator) = 3857))
 );
+INSERT INTO test_table_private_1 SELECT * from test_table;
 
-GRANT ALL ON TABLE test_table_private_1 TO test_cartodb_user_1;
+GRANT ALL ON TABLE test_table_private_1 TO :TESTUSER;

test/support/test_helper.js

@@ -6,11 +6,54 @@
  */
 
 var _ = require('underscore');
+var assert = require('assert');
+var LZMA  = require('lzma').LZMA;
+
+var lzmaWorker = new LZMA();
 
 // set environment specific variables
-global.settings     = require(__dirname + '/../../config/settings');
 global.environment  = require(__dirname + '/../../config/environments/test');
-_.extend(global.settings, global.environment);
+process.env.NODE_ENV = 'test';
 
 
+// Utility function to compress & encode LZMA
+function lzma_compress_to_base64(payload, mode, callback) {
+  lzmaWorker.compress(payload, mode,
+    function(ints) {
+      ints = ints.map(function(c) { return String.fromCharCode(c + 128) }).join('')
+      var base64 = new Buffer(ints, 'binary').toString('base64');
+      callback(null, base64);
+    },
+    function(percent) {
+      //console.log("Compressing: " + percent + "%");
+    }
+  );
+}
+
+// Check that the response headers do not request caching
+// Throws on failure
+function checkNoCache(res) {
+  assert.ok(!res.headers.hasOwnProperty('x-cache-channel'));
+  assert.ok(!res.headers.hasOwnProperty('cache-control')); // is this correct ?
+  assert.ok(!res.headers.hasOwnProperty('last-modified')); // is this correct ?
+}
+
+
+/**
+ * Check that the response headers do not request caching
+ * @see checkNoCache
+ * @param res
+ */
+function checkCache(res) {
+    assert.ok(res.headers.hasOwnProperty('x-cache-channel'));
+    assert.ok(res.headers.hasOwnProperty('cache-control'));
+    assert.ok(res.headers.hasOwnProperty('last-modified'));
+}
+
+
+module.exports = {
+  lzma_compress_to_base64: lzma_compress_to_base64,
+  checkNoCache: checkNoCache,
+  checkCache: checkCache
+};
 

test/unit/cartodb/redis_pool.test.js

@@ -1,61 +0,0 @@
-var assert = require('../../support/assert')
-  , _ = require('underscore')
-  , RedisPool = require('../../../lib/cartodb/redis_pool')
-  , tests = module.exports = {};
-
-suite('redis_pool', function() {
-
-    // configure redis pool instance to use in tests
-    var test_opts = require('../../support/config').redis_pool;
-    
-    var redis_pool = new RedisPool(test_opts);
-
-    test('RedisPool object exists', function(done){
-      assert.ok(RedisPool);
-      done();
-    });
-    
-    test('RedisPool can create new redis_pool objects with default settings', function(done){
-      var redis_pool = new RedisPool();
-      done();
-    });
-    
-    test('RedisPool can create new redis_pool objects with specific settings', function(done){
-      var redis_pool = new RedisPool(_.extend({host:'127.0.0.1', port: '6379'}, test_opts));
-      done();
-    });
-    
-    
-    test('pool object has an acquire function', function(done){
-      var found=false;
-      var functions = _.functions(redis_pool);
-      for (var i=0; i<functions.length; ++i) {
-          if ( functions[i] == 'acquire' ) { found=true; break; }
-      }
-      assert.ok(found);
-      done();
-    });
-    
-    test('calling aquire returns a redis client object that can get/set', function(done){
-      redis_pool.acquire(0, function(err, client){
-        client.set("key","value");
-        client.get("key", function(err,data){      
-          assert.equal(data, "value");      
-          redis_pool.release(0, client); // needed to exit tests
-          done();
-        })
-      });    
-    });
-    
-    test('calling aquire on another DB returns a redis client object that can get/set', function(done){
-      redis_pool.acquire(2, function(err, client){
-        client.set("key","value");
-        client.get("key", function(err,data){      
-          assert.equal(data, "value");      
-          redis_pool.release(2, client); // needed to exit tests
-          done();
-        })
-      });      
-    });
-
-});

test/unit/cartodb/req2params.test.js

@@ -7,60 +7,81 @@ var assert = require('assert')
 suite('req2params', function() {
 
     // configure redis pool instance to use in tests
-    var opts = require('../../../lib/cartodb/server_options');
+    var opts = require('../../../lib/cartodb/server_options')();
+
+    var test_user = _.template(global.environment.postgres_auth_user, {user_id:1});
+    var test_pubuser = global.environment.postgres.user;
+    var test_database = test_user + '_db';
+
     
     test('can be found in server_options', function(){
       assert.ok(_.isFunction(opts.req2params));
     });
 
     test('cleans up request', function(done){
-      opts.req2params({headers: { host:'h1' }, query: {dbuser:'hacker',dbname:'secret'}}, function(err, req) {
-          if ( err ) { console.log(err); throw new Error(err); }
+      opts.req2params({headers: { host:'localhost' }, query: {dbuser:'hacker',dbname:'secret'}}, function(err, req) {
+          if ( err ) { done(err); return; }
           assert.ok(_.isObject(req.query), 'request has query');
           assert.ok(!req.query.hasOwnProperty('dbuser'), 'dbuser was removed from query');
           assert.ok(req.hasOwnProperty('params'), 'request has params');
           assert.ok(req.params.hasOwnProperty('interactivity'), 'request params have interactivity');
-          assert.ok(_.isNull(req.params.dbname), 'could forge dbname');
-          assert.ok(!req.hasOwnProperty('dbuser'), 'could inject dbuser ('+req.params.dbuser+')');
+          assert.equal(req.params.dbname, test_database, 'could forge dbname: '+ req.params.dbname);
+          assert.ok(req.params.dbuser === test_pubuser, 'could inject dbuser ('+req.params.dbuser+')');
           done();
       });
     });
 
     test('sets dbname from redis metadata', function(done){
-      opts.req2params({headers: { host:'vizzuality' }, query: {} }, function(err, req) {
-          if ( err ) { console.log(err); throw new Error(err); }
+      opts.req2params({headers: { host:'localhost' }, query: {} }, function(err, req) {
+          if ( err ) { done(err); return; }
           //console.dir(req);
           assert.ok(_.isObject(req.query), 'request has query');
           assert.ok(!req.query.hasOwnProperty('dbuser'), 'dbuser was removed from query');
           assert.ok(req.hasOwnProperty('params'), 'request has params');
           assert.ok(req.params.hasOwnProperty('interactivity'), 'request params have interactivity');
-          // database_name for user "vizzuality" (see test/support/prepare_db.sh)
-          assert.equal(req.params.dbname, 'cartodb_test_user_1_db');
-          // unauthenticated request gets no dbuser
-          assert.ok(!req.params.hasOwnProperty('dbuser'), 'could inject dbuser ('+req.params.dbuser+')');
+          assert.equal(req.params.dbname, test_database);
+          assert.ok(req.params.dbuser === test_pubuser, 'could inject dbuser ('+req.params.dbuser+')');
           done();
       });
     });
 
     test('sets also dbuser for authenticated requests', function(done){
-      opts.req2params({headers: { host:'vizzuality' }, query: {map_key: '1234'} }, function(err, req) {
-          if ( err ) { console.log(err); throw new Error(err); }
+      opts.req2params({headers: { host:'localhost' }, query: {map_key: '1234'} }, function(err, req) {
+          if ( err ) { done(err); return; }
           //console.dir(req);
           assert.ok(_.isObject(req.query), 'request has query');
           assert.ok(!req.query.hasOwnProperty('dbuser'), 'dbuser was removed from query');
           assert.ok(req.hasOwnProperty('params'), 'request has params');
           assert.ok(req.params.hasOwnProperty('interactivity'), 'request params have interactivity');
-          // database_name for user "vizzuality" (see test/support/prepare_db.sh)
-          assert.equal(req.params.dbname, 'cartodb_test_user_1_db');
-          // id for user "vizzuality" (see test/support/prepare_db.sh)
-          assert.equal(req.dbuser, 'test_cartodb_user_1');
+          assert.equal(req.params.dbname, test_database);
+          assert.equal(req.params.dbuser, test_user);
  
-          opts.req2params({headers: { host:'vizzuality' }, query: {map_key: '1235'} }, function(err, req) {
+          opts.req2params({headers: { host:'localhost' }, query: {map_key: '1235'} }, function(err, req) {
               // wrong key resets params to no user
-              assert.ok(!req.hasOwnProperty('dbuser'), 'could inject dbuser ('+req.params.dbuser+')');
+              assert.ok(req.params.dbuser === test_pubuser, 'could inject dbuser ('+req.params.dbuser+')');
               done();
           });
       });
     });
+
+    test('it should extend params with decoded lzma', function(done) {
+      var qo = {
+        style: 'test',
+        style_version: '2.1.0',
+        cache_buster: 5
+      };
+      test_helper.lzma_compress_to_base64(JSON.stringify(qo), 1, function(err, data) {
+        opts.req2params({ headers: { host:'localhost' }, query: { non_included: 'toberemoved', api_key: 'test', style: 'override', lzma: data }}, function(err, req) {
+          if ( err ) { done(err); return; }
+          var query = req.params
+          assert.equal(qo.style, query.style)
+          assert.equal(qo.style_version, query.style_version)
+          assert.equal(qo.cache_buster, query.cache_buster)
+          assert.equal('test', query.api_key)
+          assert.equal(undefined, query.non_included)
+          done();
+        });
+      });
+    });
     
 });

test/unit/cartodb/signed_maps.test.js

@@ -0,0 +1,109 @@
+var assert = require('assert')
+  //, _ = require('underscore')
+  , RedisPool = require('redis-mpool')
+  , SignedMaps = require('../../../lib/cartodb/signed_maps.js')
+  , test_helper = require('../../support/test_helper')
+  , Step = require('step')
+  , tests = module.exports = {};
+
+suite('signed_maps', function() {
+
+    // configure redis pool instance to use in tests
+    var redis_pool = RedisPool(global.environment.redis);
+    
+    test('can sign map with open and token-based auth', function(done) {
+      var smap = new SignedMaps(redis_pool);
+      assert.ok(smap);
+      var sig = 'sig1';
+      var map = 'map1';
+      var tok = 'tok1';
+      var crt = {
+        version:'0.0.1',
+        layergroup_id:map,
+        auth: {}
+      };
+      var crt1_id; // by token
+      var crt2_id; // open
+      Step(
+        function() {
+          smap.isAuthorized(sig,map,tok,this);
+        },
+        function checkAuthFailure1(err, authorized) {
+          if ( err ) throw err;
+          assert.ok(!authorized, "unexpectedly authorized");
+          crt.auth.method = 'token';
+          crt.auth.valid_tokens = [tok];
+          smap.addSignature(sig, map, crt, this)
+        },
+        function getCert1(err, id) {
+          if ( err ) throw err;
+          assert.ok(id, "undefined signature id");
+          crt1_id = id; // keep note of it
+//console.log("Certificate 1 is " + crt1_id);
+          smap.isAuthorized(sig,map,'',this);
+        },
+        function checkAuthFailure2(err, authorized) {
+          if ( err ) throw err;
+          assert.ok(!authorized, "unexpectedly authorized");
+          smap.isAuthorized(sig,map,tok,this);
+        },
+        function checkAuthSuccess1(err, authorized) {
+          if ( err ) throw err;
+          assert.ok(authorized, "unauthorized :(");
+          crt.auth.method = 'open';
+          delete crt.auth.valid_tokens;
+          smap.addSignature(sig, map, crt, this)
+        },
+        function getCert2(err, id) {
+          if ( err ) throw err;
+          assert.ok(id, "undefined signature id");
+          crt2_id = id; // keep note of it
+//console.log("Certificate 2 is " + crt2_id);
+          smap.isAuthorized(sig,map,'arbitrary',this);
+        },
+        function checkAuthSuccess2_delCert2(err, authorized) {
+          if ( err ) throw err;
+          assert.ok(authorized, "unauthorized :(");
+          var next = this;
+          smap.delCertificate(sig, crt2_id, function(e) {
+            if (e) next(e);
+            else smap.isAuthorized(sig,map,'arbitrary',next);
+          });
+        },
+        function checkAuthFailure3_delCert2(err, authorized) {
+          if ( err ) throw err;
+          assert.ok(!authorized, "unexpectedly authorized");
+          smap.delCertificate(sig, crt1_id, this);
+        },
+        function finish(err) {
+          done(err);
+        }
+      );
+    });
+
+    test('can validate certificates', function(done) {
+      var smap = new SignedMaps(redis_pool);
+      assert.ok(smap);
+      Step(
+        function invalidVersion() {
+          var cert = { version: '-1' };
+          var err = smap.checkInvalidCertificate(cert);
+          assert.ok(err);
+          assert.equal(err.message, "Unsupported certificate version -1");
+          return null;
+        },
+        function invalidTokenAuth() {
+          var cert = { version: '0.0.1', auth: { method:'token', valid_token:[] } };
+          var err = smap.checkInvalidCertificate(cert);
+          assert.ok(err);
+          assert.equal(err.message, "Invalid 'token' authentication: missing valid_tokens");
+          return null;
+        },
+        function finish(err) {
+          done(err);
+        }
+      );
+    });
+
+    
+});

test/unit/cartodb/template_maps.test.js

@@ -0,0 +1,598 @@
+var assert = require('assert')
+  //, _ = require('underscore')
+  , RedisPool = require('redis-mpool')
+  , SignedMaps = require('../../../lib/cartodb/signed_maps.js')
+  , TemplateMaps = require('../../../lib/cartodb/template_maps.js')
+  , test_helper = require('../../support/test_helper')
+  , Step = require('step')
+  , tests = module.exports = {};
+
+suite('template_maps', function() {
+
+  // configure redis pool instance to use in tests
+  var redis_pool = RedisPool(global.environment.redis);
+  var signed_maps = new SignedMaps(redis_pool);
+
+    var validTemplate = {
+        version:'0.0.1',
+        name: 'first',
+        auth: {},
+        layergroup: {}
+    };
+    var owner = 'me';
+    
+  test('does not accept template with unsupported version', function(done) {
+    var tmap = new TemplateMaps(redis_pool, signed_maps);
+    assert.ok(tmap);
+    var tpl = { version:'6.6.6',
+      name:'k', auth: {}, layergroup: {} };
+    Step(
+      function() {
+        tmap.addTemplate('me', tpl, this);
+      },
+      function checkFailed(err) {
+        assert.ok(err);
+        assert.ok(err.message.match(/unsupported.*version/i), err);
+        return null;
+      },
+      function finish(err) {
+        done(err);
+      }
+    );
+  });
+
+  test('does not accept template with missing name', function(done) {
+    var tmap = new TemplateMaps(redis_pool, signed_maps);
+    assert.ok(tmap);
+    var tpl = { version:'0.0.1',
+      auth: {}, layergroup: {} };
+    Step(
+      function() {
+        tmap.addTemplate('me', tpl, this);
+      },
+      function checkFailed(err) {
+        assert.ok(err);
+        assert.ok(err.message.match(/missing.*name/i), err);
+        return null;
+      },
+      function finish(err) {
+        done(err);
+      }
+    );
+  });
+
+  test('does not accept template with invalid name', function(done) {
+    var tmap = new TemplateMaps(redis_pool, signed_maps);
+    assert.ok(tmap);
+    var tpl = { version:'0.0.1',
+      auth: {}, layergroup: {} };
+    var invalidnames = [ "ab|", "a b", "a@b", "1ab", "_x", "", " x", "x " ];
+    var testNext = function() {
+      if ( ! invalidnames.length ) { done(); return; }
+      var n = invalidnames.pop();
+      tpl.name = n;
+      tmap.addTemplate('me', tpl, function(err) {
+        if ( ! err ) {
+          done(new Error("Unexpected success with invalid name '" + n + "'"));
+        }
+        else if ( ! err.message.match(/template.*name/i) ) {
+          done(new Error("Unexpected error message with invalid name '" + n
+            + "': " + err));
+        }
+        else {
+          testNext();
+        }
+      });
+    };
+    testNext();
+  });
+
+  test('does not accept template with invalid placeholder name', function(done) {
+    var tmap = new TemplateMaps(redis_pool, signed_maps);
+    assert.ok(tmap);
+    var tpl = { version:'0.0.1',
+      name: "valid", placeholders: {},
+      auth: {}, layergroup: {} };
+    var invalidnames = [ "ab|", "a b", "a@b", "1ab", "_x", "", " x", "x " ];
+    var testNext = function() {
+      if ( ! invalidnames.length ) { done(); return; }
+      var n = invalidnames.pop();
+      tpl.placeholders = {};
+      tpl.placeholders[n] = { type:'number', default:1 };
+      tmap.addTemplate('me', tpl, function(err) {
+        if ( ! err ) {
+          done(new Error("Unexpected success with invalid name '" + n + "'"));
+        }
+        else if ( ! err.message.match(/invalid.*name/i) ) {
+          done(new Error("Unexpected error message with invalid name '" + n
+            + "': " + err));
+        }
+        else {
+          testNext();
+        }
+      });
+    };
+    testNext();
+  });
+
+  test('does not accept template with missing placeholder default', function(done) {
+    var tmap = new TemplateMaps(redis_pool, signed_maps);
+    assert.ok(tmap);
+    var tpl = { version:'0.0.1',
+      name: "valid", placeholders: { v: {} },
+      auth: {}, layergroup: {} };
+    tmap.addTemplate('me', tpl, function(err) {
+        if ( ! err ) {
+          done(new Error("Unexpected success with missing placeholder default"));
+        }
+        else if ( ! err.message.match(/missing default/i) ) {
+          done(new Error("Unexpected error message with missing placeholder default: "
+            + err));
+        }
+        else {
+          done();
+        }
+      });
+  });
+
+  test('does not accept template with missing placeholder type', function(done) {
+    var tmap = new TemplateMaps(redis_pool, signed_maps);
+    assert.ok(tmap);
+    var tpl = { version:'0.0.1',
+      name: "valid", placeholders: { v: { default:1 } },
+      auth: {}, layergroup: {} };
+    tmap.addTemplate('me', tpl, function(err) {
+        if ( ! err ) {
+          done(new Error("Unexpected success with missing placeholder type"));
+        }
+        else if ( ! err.message.match(/missing type/i) ) {
+          done(new Error("Unexpected error message with missing placeholder default: "
+            + err));
+        }
+        else {
+          done();
+        }
+      });
+  });
+
+  // See http://github.com/CartoDB/Windshaft-cartodb/issues/128
+  test('does not accept template with invalid token auth (undefined tokens)',
+  function(done) {
+    var tmap = new TemplateMaps(redis_pool, signed_maps);
+    assert.ok(tmap);
+    var tpl = { version:'0.0.1',
+      name: "invalid_auth1", placeholders: { },
+      auth: { method: 'token' }, layergroup: {} };
+    tmap.addTemplate('me', tpl, function(err) {
+        if ( ! err ) {
+          done(new Error("Unexpected success with invalid token auth (undefined tokens)"));
+        }
+        else if ( ! err.message.match(/invalid 'token' authentication/i) ) {
+          done(new Error("Unexpected error message with invalid token auth (undefined tokens): "
+            + err));
+        }
+        else {
+          done();
+        }
+      });
+  });
+
+  test('add, get and delete a valid template', function(done) {
+    var tmap = new TemplateMaps(redis_pool, signed_maps);
+    assert.ok(tmap);
+    var expected_failure = false;
+    var tpl_id;
+    var tpl = { version:'0.0.1',
+      name: 'first', auth: {}, layergroup: {} };
+    Step(
+      function() {
+        tmap.addTemplate('me', tpl, this);
+      },
+      function addOmonimousTemplate(err, id) {
+        if ( err ) throw err;
+        tpl_id = id;
+        assert.equal(tpl_id, 'first');
+        expected_failure = true;
+        // should fail, as it already exists
+        tmap.addTemplate('me', tpl, this);
+      },
+      function getTemplate(err) {
+        if ( ! expected_failure && err ) throw err;
+        assert.ok(err);
+        assert.ok(err.message.match(/already exists/i), err);
+        tmap.getTemplate('me', tpl_id, this);
+      },
+      function delTemplate(err, got_tpl) {
+        if ( err ) throw err;
+        assert.deepEqual(got_tpl, tpl);
+        tmap.delTemplate('me', tpl_id, this);
+      },
+      function finish(err) {
+        done(err);
+      }
+    );
+  });
+
+  test('add multiple templates, list them', function(done) {
+    var tmap = new TemplateMaps(redis_pool, signed_maps);
+    assert.ok(tmap);
+    var expected_failure = false;
+    var tpl1 = { version:'0.0.1', name: 'first', auth: {}, layergroup: {} };
+    var tpl1_id;
+    var tpl2 = { version:'0.0.1', name: 'second', auth: {}, layergroup: {} };
+    var tpl2_id;
+    Step(
+      function addTemplate1() {
+        tmap.addTemplate('me', tpl1, this);
+      },
+      function addTemplate2(err, id) {
+        if ( err ) throw err;
+        tpl1_id = id;
+        tmap.addTemplate('me', tpl2, this);
+      },
+      function listTemplates(err, id) {
+        if ( err ) throw err;
+        tpl2_id = id;
+        tmap.listTemplates('me', this);
+      },
+      function checkTemplates(err, ids) {
+        if ( err ) throw err;
+        assert.equal(ids.length, 2);
+        assert.ok(ids.indexOf(tpl1_id) != -1, ids.join(','));
+        assert.ok(ids.indexOf(tpl2_id) != -1, ids.join(','));
+        return null;
+      },
+      function delTemplate1(err) {
+        if ( tpl1_id ) {
+          var next = this;
+          tmap.delTemplate('me', tpl1_id, function(e) {
+            if ( err || e ) next(new Error(err + '; ' + e));
+            else next();
+          });
+        } else {
+          if ( err ) throw err;
+          return null;
+        }
+      },
+      function delTemplate2(err) {
+        if ( tpl2_id ) {
+          var next = this;
+          tmap.delTemplate('me', tpl2_id, function(e) {
+            if ( err || e ) next(new Error(err + '; ' + e));
+            else next();
+          });
+        } else {
+          if ( err ) throw err;
+          return null;
+        }
+      },
+      function finish(err) {
+        done(err);
+      }
+    );
+  });
+
+  test('update templates', function(done) {
+    var tmap = new TemplateMaps(redis_pool, signed_maps);
+    assert.ok(tmap);
+    var expected_failure = false;
+    var owner = 'me';
+    var tpl = { version:'0.0.1',
+      name: 'first',
+      auth: { method: 'open' },
+      layergroup: {}
+    };
+    var tpl_id;
+    Step(
+      function addTemplate() {
+        tmap.addTemplate(owner, tpl, this);
+      },
+      // Updating template name should fail
+      function updateTemplateName(err, id) {
+        if ( err ) throw err;
+        tpl_id = id;
+        expected_failure = true;
+        tpl.name = 'second';
+        tmap.updTemplate(owner, tpl_id, tpl, this);
+      },
+      function updateTemplateAuth(err) {
+        if ( err && ! expected_failure) throw err;
+        expected_failure = false;
+        assert.ok(err);
+        tpl.name = 'first';
+        tpl.auth.method = 'token';
+        tpl.auth.valid_tokens = [ 'tok1' ];
+        tmap.updTemplate(owner, tpl_id, tpl, this);
+      },
+      function updateTemplateWithInvalid(err) {
+        if ( err ) throw err;
+        tpl.version = '999.999.999';
+        expected_failure = true;
+        tmap.updTemplate(owner, tpl_id, tpl, this);
+      },
+      function updateUnexistentTemplate(err) {
+        if ( err && ! expected_failure) throw err;
+        expected_failure = false;
+        assert.ok(err);
+        assert.ok(err.message.match(/unsupported.*version/i), err);
+        tpl.version = '0.0.1';
+        expected_failure = true;
+        tmap.updTemplate(owner, 'unexistent', tpl, this);
+      },
+      function delTemplate(err) {
+        if ( err && ! expected_failure) throw err;
+        expected_failure = false;
+        assert.ok(err);
+        assert.ok(err.message.match(/cannot update name/i), err);
+        tmap.delTemplate(owner, tpl_id, this);
+      },
+      function finish(err) {
+        done(err);
+      }
+    );
+  });
+
+  test('instanciate templates', function() {
+    var tmap = new TemplateMaps(redis_pool, signed_maps);
+    assert.ok(tmap);
+
+    var tpl1 =  {
+        version: '0.0.1',
+        name: 'acceptance1',
+        auth: { method: 'open' },
+        placeholders: {
+          fill: { type: "css_color", default: "red" },
+          color: { type: "css_color", default: "#a0fF9A" },
+          name: { type: "sql_literal", default: "test" },
+          zoom: { type: "number", default: "0" },
+          test_number: { type: "number", default: 23 },
+        },
+        layergroup: {
+          version: '1.0.0',
+          global_cartocss_version: '2.0.2',
+          layers: [
+             { options: {
+                 sql: "select '<%=name %>' || id, g from t",
+                 cartocss: '#layer { marker-fill:<%= fill %>; marker-width: <%=test_number  %>; }'
+               } },
+             { options: {
+                 sql: "select fun('<%=     name%>') g from x",
+                 cartocss: '#layer { line-color:<%= color %>; marker-fill:<%= color %>; }'
+               } },
+             { options: {
+                 sql: "select g from x",
+                 cartocss: '#layer[zoom=<%=zoom%>] { }'
+               } }
+          ]
+        }
+    };
+
+    var inst = tmap.instance(tpl1, {});
+
+    var lyr = inst.layers[0].options;
+    assert.equal(lyr.sql, "select 'test' || id, g from t");
+    assert.equal(lyr.cartocss, '#layer { marker-fill:red; marker-width: 23; }');
+
+    lyr = inst.layers[1].options;
+    assert.equal(lyr.sql, "select fun('test') g from x");
+    assert.equal(lyr.cartocss, '#layer { line-color:#a0fF9A; marker-fill:#a0fF9A; }');
+
+    inst = tmap.instance(tpl1, {color:'yellow', name:"it's dangerous"});
+
+    lyr = inst.layers[0].options;
+    assert.equal(lyr.sql, "select 'it''s dangerous' || id, g from t");
+    assert.equal(lyr.cartocss, '#layer { marker-fill:red; marker-width: 23; }');
+
+    lyr = inst.layers[1].options;
+    assert.equal(lyr.sql, "select fun('it''s dangerous') g from x");
+    assert.equal(lyr.cartocss, '#layer { line-color:yellow; marker-fill:yellow; }');
+
+    // Invalid css_color
+    var err = null;
+    try { inst = tmap.instance(tpl1, {color:'##ff00ff'}); }
+    catch (e) { err = e; }
+    assert.ok(err);
+    assert.ok(err.message.match(/invalid css_color/i), err);
+
+    // Invalid css_color 2 (too few digits)
+    var err = null;
+    try { inst = tmap.instance(tpl1, {color:'#ff'}); }
+    catch (e) { err = e; }
+    assert.ok(err);
+    assert.ok(err.message.match(/invalid css_color/i), err);
+
+    // Invalid css_color 3 (too many digits)
+    var err = null;
+    try { inst = tmap.instance(tpl1, {color:'#1234567'}); }
+    catch (e) { err = e; }
+    assert.ok(err);
+    assert.ok(err.message.match(/invalid css_color/i), err);
+
+    // Invalid number
+    var err = null;
+    try { inst = tmap.instance(tpl1, {zoom:'#'}); }
+    catch (e) { err = e; }
+    assert.ok(err);
+    assert.ok(err.message.match(/invalid number/i), err);
+
+    // Invalid number 2
+    var err = null;
+    try { inst = tmap.instance(tpl1, {zoom:'23e'}); }
+    catch (e) { err = e; }
+    assert.ok(err);
+    assert.ok(err.message.match(/invalid number/i), err);
+
+    // Valid number
+    var err = null;
+    try { inst = tmap.instance(tpl1, {zoom:'-.23e10'}); }
+    catch (e) { err = e; }
+    assert.ok(!err);
+  });
+
+  // Can set a limit on the number of user templates
+  test('can limit number of user templates', function(done) {
+    var tmap = new TemplateMaps(redis_pool, signed_maps, {
+      max_user_templates: 2
+    });
+    assert.ok(tmap);
+    var tpl = { version:'0.0.1', auth: {}, layergroup: {} };
+    var expectErr = false;
+    var idMe = [];
+    var idYou = [];
+    Step(
+      function oneForMe() {
+        tpl.name = 'oneForMe';
+        tmap.addTemplate('me', tpl, this);
+      },
+      function twoForMe(err, id) {
+        if ( err ) throw err;
+        assert.ok(id);
+        idMe.push(id);
+        tpl.name = 'twoForMe';
+        tmap.addTemplate('me', tpl, this);
+      },
+      function threeForMe(err, id) {
+        if ( err ) throw err;
+        assert.ok(id);
+        idMe.push(id);
+        tpl.name = 'threeForMe';
+        expectErr = true;
+        tmap.addTemplate('me', tpl, this);
+      },
+      function errForMe(err, id) {
+        if ( err && ! expectErr ) throw err;
+        expectErr = false;
+        assert.ok(err);
+        assert.ok(err.message.match(/limit.*template/), err);
+        return null;
+      },
+      function delOneMe(err) {
+        if ( err ) throw err;
+        tmap.delTemplate('me', idMe.shift(), this);
+      },
+      function threeForMeRetry(err) {
+        if ( err ) throw err;
+        tpl.name = 'threeForMe';
+        tmap.addTemplate('me', tpl, this);
+      },
+      function oneForYou(err, id) {
+        if ( err ) throw err;
+        assert.ok(id);
+        idMe.push(id);
+        tpl.name = 'oneForYou';
+        tmap.addTemplate('you', tpl, this);
+      },
+      function twoForYou(err, id) {
+        if ( err ) throw err;
+        assert.ok(id);
+        idYou.push(id);
+        tpl.name = 'twoForYou';
+        tmap.addTemplate('you', tpl, this);
+      },
+      function threeForYou(err, id) {
+        if ( err ) throw err;
+        assert.ok(id);
+        idYou.push(id);
+        tpl.name = 'threeForYou';
+        expectErr = true;
+        tmap.addTemplate('you', tpl, this);
+      },
+      function errForYou(err, id) {
+        if ( err && ! expectErr ) throw err;
+        expectErr = false;
+        assert.ok(err);
+        assert.ok(err.message.match(/limit.*template/), err);
+        return null;
+      },
+      function finish(err) {
+        // TODO: delete all templates
+        done(err);
+      }
+    );
+  });
+
+    var redisCmdFunc = TemplateMaps.prototype._redisCmd;
+
+    function runWithRedisStubbed(stubbedCommands, func) {
+        TemplateMaps.prototype._redisCmd = function(redisFunc, redisArgs, callback) {
+            redisFunc = redisFunc.toLowerCase();
+            if (stubbedCommands.hasOwnProperty(redisFunc)) {
+                callback(null, stubbedCommands[redisFunc]);
+            } else {
+                throw 'Unknown command';
+            }
+        };
+
+        func();
+
+        TemplateMaps.prototype._redisCmd = redisCmdFunc;
+    }
+
+    test('_obtainTemplateLock with no previous value, happy case', function(done) {
+        runWithRedisStubbed({hget: null, hset: 1}, function() {
+            var templateMaps = new TemplateMaps(redis_pool, signed_maps);
+
+            templateMaps._obtainTemplateLock(owner, validTemplate.name, function(err, gotLock) {
+                assert.ok(!err);
+                assert.ok(gotLock);
+                done();
+            });
+        });
+    });
+
+    test('_obtainTemplateLock no lock for non expired ttl, simulates obtaining two locks at same time', function(done) {
+        runWithRedisStubbed({hget: Date.now()}, function() {
+            var templateMaps = new TemplateMaps(redis_pool, signed_maps);
+
+            templateMaps._obtainTemplateLock(owner, validTemplate.name, function(err, gotLock) {
+                assert.ok(!!err);
+                assert.equal(gotLock, false);
+                done();
+            });
+        });
+    });
+
+
+    test('_obtainTemplateLock no lock for non expired ttl, last millisecond of valid ttl', function(done) {
+        var nowValue = Date.now(),
+            nowFunc = Date.now;
+        Date.now = function() {
+            return nowValue;
+        };
+        var lockTtl = 1000;
+        runWithRedisStubbed({hget: Date.now() - lockTtl, hset: true}, function() {
+            var templateMaps = new TemplateMaps(redis_pool, signed_maps, {lock_ttl: lockTtl});
+
+            templateMaps._obtainTemplateLock(owner, validTemplate.name, function(err, gotLock) {
+                assert.ok(!!err);
+                assert.equal(gotLock, false);
+
+                Date.now = nowFunc;
+
+                done();
+            });
+        });
+    });
+
+    test('_obtainTemplateLock gets lock for expired ttl, first millisecond of invalid ttl', function(done) {
+        var nowValue = Date.now(),
+            nowFunc = Date.now;
+        Date.now = function() {
+            return nowValue;
+        };
+        var lockTtl = 1000;
+        runWithRedisStubbed({hget: Date.now() - lockTtl - 1, hset: true}, function() {
+            var templateMaps = new TemplateMaps(redis_pool, signed_maps, {lock_ttl: lockTtl});
+
+            templateMaps._obtainTemplateLock(owner, validTemplate.name, function(err, gotLock) {
+                assert.ok(!err);
+                assert.ok(gotLock);
+
+                Date.now = nowFunc;
+
+                done();
+            });
+        });
+    });
+
+});

tools/convert_database_styles

@@ -0,0 +1,81 @@
+#!/usr/bin/env node
+
+var path = require('path');
+
+var grainstore = require('../node_modules/windshaft/node_modules/grainstore');
+var mapnik = require('mapnik');
+var redis = require('redis');
+
+function usage(me, exitcode) {
+  console.log("Usage: " + me + " <database_name> <table_name> [<target_mapnik_version>]");
+  process.exit(exitcode);
+}
+
+var node_path = process.argv.shift(); 
+var script_path = process.argv.shift(); 
+var me = path.basename(script_path);
+
+var database_name = process.argv.shift()
+var table_name = process.argv.shift()
+var MAPNIK_VERSION = process.argv.shift()
+
+
+if ( ! MAPNIK_VERSION ) {
+  MAPNIK_VERSION = mapnik.versions.mapnik;
+}
+
+if ( ! database_name || ! table_name) {
+  usage(me, 1);
+}
+
+var REDIS_PORT = 6379; // TODO: make a command line parameter
+
+var dbnum = 0;
+
+var mml_store = new grainstore.MMLStore({port:REDIS_PORT}, {mapnik_version:MAPNIK_VERSION});
+
+var failures = [];
+
+var client = redis.createClient(REDIS_PORT, 'localhost');
+client.on('connect', function() {
+  client.select(dbnum);
+  client.keys('map_style|' + database_name + '|' + table_name, function(err, matches) {
+
+    processNext = function() {
+      if ( ! matches.length ) process.exit(failures.length);
+      var k = matches.shift();
+
+      if ( /map_style\|.*\|.*\|/.test(k) ) {
+        //console.warn("Key " + k + " is EXTENDED, skipping");
+        processNext();
+      } 
+
+      var out = 'map_style|' + database_name + '|' + table_name + ': ';
+
+      var mml_builder = mml_store.mml_builder({
+        dbname:database_name, 
+        table:table_name},
+        function(err, payload) {
+          if ( err ) { 
+            console.warn(out + err.message); 
+            failures.push(k); processNext(); 
+          }
+          else {
+            mml_builder.resetStyle(function(err, data) {
+              if ( err ) { 
+                console.warn(out + err.message); 
+                failures.push(k); 
+              }
+              else console.log(out + 'OK');
+              processNext();
+            }, true);
+          }
+      });
+
+    };
+
+    processNext();
+
+  });
+
+});

tools/create_multilayer

@@ -0,0 +1,40 @@
+#!/bin/sh
+
+verbose=no
+tiler_url=http://dev.localhost.lan:8181/tiles/layergroup
+
+# This is for direct windshaft connection
+#tiler_url=http://dev.localhost.lan:8083/database/cartodb_dev_user_1_db/layergroup
+
+while test -n "$1"; do
+  if test "$1" = "-v"; then
+    verbose=yes
+  elif test -z "$cfg"; then
+    cfg="$1"
+  else
+    tiler_url="$1"
+  fi
+  shift
+done
+
+if test -z "$cfg"; then
+  echo "Usage: $0 [-v] <config_file> [<tiler_url>]" >&2
+  echo "Default <tiler_url> is ${tiler_url}" >&2
+  exit 1
+fi
+
+cmd="curl -skH Content-Type:application/json --data-binary @- ${tiler_url}"
+if test x${verbose} = xyes; then
+  cmd="${cmd} -v"
+fi
+res=`cat ${cfg} | tr '\n' ' ' | ${cmd}`
+if test $? -gt 0; then
+  echo "curl command failed: ${cmd}"
+fi
+
+if test x${verbose} = xyes; then
+  echo "${res}"
+fi
+
+tok=`echo "$res" | sed 's/.*"layergroupid":"\([^"]*\)".*/\1/'`
+echo $tok

tools/create_template

@@ -0,0 +1,49 @@
+#!/bin/sh
+
+verbose=no
+tiler_url=http://dev.localhost.lan:8181/tiles/template
+apikey=${CDB_APIKEY}
+
+while test -n "$1"; do
+  if test "$1" = "-v"; then
+    verbose=yes
+  elif test "$1" = "-k"; then
+    shift
+    apikey="$1"
+  elif test "$1" = "-u"; then
+    shift
+    tiler_url="$1"
+  elif test -z "$cfg"; then
+    cfg="$1"
+  else
+    echo "Unused parameter $1" >&2
+  fi
+  shift
+done
+
+if test -z "$cfg"; then
+  echo "Usage: $0 [-v] [-k <api_key>] [-u <tiler_url>] <template_config>" >&2
+  echo "Default <tiler_url> is ${tiler_url}" >&2
+  echo "Default <api_key> is read from CDB_APIKEY env variable" >&2
+  exit 1
+fi
+
+cmd="curl -skH Content-Type:application/json --data-binary @- ${tiler_url}?api_key=${apikey}"
+if test x${verbose} = xyes; then
+  cmd="${cmd} -v"
+fi
+res=`cat ${cfg} | tr '\n' ' ' | ${cmd}`
+if test $? -gt 0; then
+  echo "curl command failed: ${cmd}"
+fi
+
+if test x${verbose} = xyes; then
+  echo "${res}"
+fi
+
+# Successful response contains no space
+echo "$res" | grep " " && { echo $res && exit 1; }
+
+tok=`echo "$res" | sed 's/.*"template_id":"\([^"]*\)".*/\1/'`
+echo $tok
+

tools/delete_template

@@ -0,0 +1,45 @@
+#!/bin/sh
+
+verbose=no
+tiler_url=http://dev.localhost.lan:8181/tiles/template
+apikey=${CDB_APIKEY}
+
+while test -n "$1"; do
+  if test "$1" = "-v"; then
+    verbose=yes
+  elif test "$1" = "-k"; then
+    shift
+    apikey="$1"
+  elif test "$1" = "-u"; then
+    shift
+    tiler_url="$1"
+  elif test -z "$tpl"; then
+    tpl="$1"
+  else
+    echo "Unused parameter $1" >&2
+  fi
+  shift
+done
+
+if test -z "$tpl"; then
+  echo "Usage: $0 [-v] [-k <api_key>] [-u <tiler_url>] <template_id>" >&2
+  echo "Default <tiler_url> is ${tiler_url}" >&2
+  echo "Default <api_key> is read from CDB_APIKEY env variable" >&2
+  exit 1
+fi
+
+cmd="curl -X DELETE -skH Content-Type:application/json ${tiler_url}/${tpl}?api_key=${apikey}"
+if test x${verbose} = xyes; then
+  cmd="${cmd} -v"
+fi
+res=`${cmd}`
+if test $? -gt 0; then
+  echo "curl command failed: ${cmd}"
+fi
+
+if test x${verbose} = xyes; then
+  echo "${res}"
+fi
+
+tok=`echo "$res" | sed 's/.*"template_id":"\([^"]*\)".*/\1/'`
+echo $tok

tools/examples/mapconfig_simple.js

@@ -0,0 +1,11 @@
+{"version":"1.0.1",
+	"layers":[{
+    "type":"cartodb",
+	  "options":{
+      "sql":"select 1 as id, ST_SetSRID(ST_MakePoint(0,0),3857) as the_geom_webmercator",
+	    "cartocss":"#style{ marker-width: 12;}",
+	    "cartocss_version":"2.1.1",
+      "Interactivity":"id"
+    }
+  }]
+}

tools/examples/mapconfig_slow.js

@@ -0,0 +1,11 @@
+{"version":"1.0.1",
+	"layers":[{
+    "type":"cartodb",
+	  "options":{
+      "sql":"select 1 as id, ST_Transform(ST_SetSRID(ST_MakePoint(x/1000,x/2000),4326),3857) as the_geom_webmercator FROM generate_series(-170000,170000) x",
+	    "cartocss":"#style{ marker-width: 12;}",
+	    "cartocss_version":"2.1.1",
+      "Interactivity":"id"
+    }
+  }]
+}

tools/examples/mapconfig_torque.js

@@ -0,0 +1,10 @@
+{"version":"1.0.1",
+	"layers":[{
+    "type":"torque",
+	  "options":{
+      "sql":"select 1 as id, ST_SetSRID(ST_MakePoint(0,0),3857) as the_geom_webmercator",
+	    "cartocss":"Map{ -torque-time-attribute:'id'; -torque-aggregation-function:'count(id)'; -torque-frame-count:2; -torque-resolution:2}",
+      "cartocss_version": "2.1.1"
+    }
+  }]
+}

tools/examples/template_simple.js

@@ -0,0 +1,17 @@
+{
+ "version":"0.0.1",
+ "name":"simple",
+ "placeholders":{},
+ "auth":{ "method":"open" },
+ "layergroup":{
+   "version":"1.0.1",
+    "layers":[{
+      "type":"cartodb",
+      "options":{
+        "sql":"select ST_SetSRID(ST_MakePoint(0,0),3857) as the_geom_webmercator",
+        "cartocss":"#s{ marker-width: 12;}",
+        "cartocss_version":"2.1.1"
+      }
+    }]
+  }
+}

tools/flush_cache

@@ -0,0 +1,59 @@
+#!/usr/bin/env node
+
+var path = require('path');
+var request    = require('request');
+
+function usage(me, exitcode) {
+  console.log("Usage: " + me + " [--env <environment>] <username> <tablename>");
+  process.exit(exitcode);
+}
+
+var node_path = process.argv.shift(); 
+var script_path = process.argv.shift(); 
+var basedir = path.dirname(script_path);
+var me = path.basename(script_path);
+
+var ENV = 'development.js';
+var username, table;
+var arg;
+while ( arg = process.argv.shift() ) {
+  if ( arg == '--env' ) {
+    ENV = process.argv.shift();
+  }
+  else if ( ! username ) {
+    username = arg;
+  }
+  else if ( ! table ) {
+    table = arg;
+  }
+  else {
+    console.warn("Unused parameter " + arg);
+  }
+}
+
+if ( ! table ) {
+  usage(me, 1);
+}
+
+global.environment = require('../config/environments/' + ENV);
+
+// _after_ setting global.environment
+var serverOptions = require('../lib/cartodb/server_options');
+
+var host = global.environment.host;
+var port = global.environment.port;
+var re = ''+serverOptions.re_userFromHost;
+var hostname = re.replace(/^\/\^/, '')
+                 .replace(/\/$$/, '')
+                 .replace(/\\/g,'')
+                 .replace(/\([^)]*\)/,username)
+              ;
+//console.log("re: " + re);
+//console.log("hostname: " + hostname);
+
+var url = 'http://' + host + ':' + port + '/tiles/' + table + '/flush_cache';
+request.del({ url: url, headers: { host: hostname } },
+function(err, res, body) {
+  if ( err ) throw err;
+  console.log(res.body);
+});

tools/instanciate_template

@@ -0,0 +1,53 @@
+#!/bin/sh
+
+verbose=no
+tiler_url=http://dev.localhost.lan:8181/tiles/template
+apikey=${CDB_APIKEY}
+
+while test -n "$1"; do
+  if test "$1" = "-v"; then
+    verbose=yes
+  elif test "$1" = "-k"; then
+    shift
+    apikey="$1"
+  elif test "$1" = "-u"; then
+    shift
+    tiler_url="$1"
+  elif test -z "$tpl"; then
+    tpl="$1"
+  elif test -z "$cfg"; then
+    cfg="$1"
+  else
+    echo "Unused parameter $1" >&2
+  fi
+  shift
+done
+
+if test -z "$tpl"; then
+  echo "Usage: $0 [-v] [-k <api_key>] [-u <tiler_url>] <template_id> [<template_params>]" >&2
+  echo "Default <tiler_url> is ${tiler_url}" >&2
+  echo "Default <api_key> is read from CDB_APIKEY env variable" >&2
+  exit 1
+fi
+
+if test -z "$cfg"; then
+  cfg="/dev/null"
+fi
+
+tiler_url="${tiler_url}/${tpl}"
+
+cmd="curl -X POST -skH Content-Type:application/json --data-binary @- ${tiler_url}?api_key=${apikey}"
+if test x${verbose} = xyes; then
+  cmd="${cmd} -v"
+fi
+res=`cat ${cfg} | tr '\n' ' ' | ${cmd}`
+if test $? -gt 0; then
+  echo "curl command failed: ${cmd}"
+fi
+
+if test x${verbose} = xyes; then
+  echo "${res}"
+fi
+
+tok=`echo "$res" | sed 's/.*"layergroupid":"\([^"]*\)".*/\1/'`
+echo $tok

tools/list_templates

@@ -0,0 +1,45 @@
+#!/bin/sh
+
+verbose=no
+tiler_url=http://dev.localhost.lan:8181/tiles/template
+apikey=${CDB_APIKEY}
+
+while test -n "$1"; do
+  if test "$1" = "-v"; then
+    verbose=yes
+  elif test "$1" = "-k"; then
+    shift
+    apikey="$1"
+  elif test "$1" = "-u"; then
+    shift
+    tiler_url="$1"
+  elif test "$1" = "-h" -o "$1" = "-?"; then
+    echo "Usage: $0 [-v] [-k <api_key>] [-u <tiler_url>]" >&2
+    echo "Default <tiler_url> is ${tiler_url}" >&2
+    echo "Default <api_key> is read from CDB_APIKEY env variable" >&2
+    exit 0
+  else
+    echo "Unused parameter $1" >&2
+  fi
+  shift
+done
+
+
+cmd="curl -X GET -sk ${tiler_url}?api_key=${apikey}"
+if test x${verbose} = xyes; then
+  cmd="${cmd} -v"
+fi
+res=`${cmd}`
+if test $? -gt 0; then
+  echo "curl command failed: ${cmd}"
+fi
+
+if test x${verbose} = xyes; then
+  echo "${res}"
+fi
+
+node <<EOF
+var parsed = JSON.parse('$res');
+console.dir(parsed);
+EOF
+

tools/munin/Makefile

@@ -0,0 +1,19 @@
+MUNIN_PLUGINS_DIR=/etc/munin/plugins
+MUNIN_PLUGINS_CONFIG_DIR=/etc/munin/plugin-conf.d
+PWD=$(shell pwd)
+
+all: windshaft.conf
+
+windshaft.conf: windshaft.conf.in
+	sed 's#@PWD@#$(PWD)#' < $< > $@
+
+install-munin-plugin-conf: windshaft.conf 
+	install -m 644 $< $(MUNIN_PLUGINS_CONFIG_DIR)/windshaft.conf
+
+install-munin-plugin: windshaft
+	install -m 755 $< $(MUNIN_PLUGINS_DIR)/windshaft
+
+install: install-munin-plugin install-munin-plugin-conf
+
+clean:
+	rm -f windshaft.conf

tools/munin/windshaft

@@ -0,0 +1,87 @@
+#!/bin/sh
+
+envnik=$(basename "${TILER_ENVIRONMENT}" .js)
+
+if test "$1" = "config"; then
+echo "graph_title fd usage (${envnik})"
+cat <<"EOM"
+graph_vlabel number of file descriptors
+graph_category windshaft
+graph_scale no
+procs.label Number of tiler processes
+pgsql.label PostgreSQL connections (max)
+redis.label Redis connections (max)
+http.label Incoming http requests (max)
+caches.label Number of renderer caches (max)
+nfd.label Total file descriptors (max)
+EOM
+exit 0
+fi
+
+if test x"$1" != x; then
+  TILER_ENVIRONMENT=$(cd $(dirname $0); pwd)/../../config/environments/${1}.js
+fi
+
+if test -z "$TILER_ENVIRONMENT"; then
+  echo "Usage: $0 [<environment>]" >&2
+  echo "   or: [TILER_ENVIRONMENT=<environment>] $0" >&2
+  exit 1
+fi
+
+http_port=$(echo "console.log(require('${TILER_ENVIRONMENT}').port)" | node) || exit 1
+pgsql_port=$(echo "console.log(require('${TILER_ENVIRONMENT}').postgres.port)" | node) || exit 1
+redis_port=$(echo "console.log(require('${TILER_ENVIRONMENT}').redis.port)" | node) || exit 1
+
+pids=$(lsof -i :${http_port} | grep LISTEN | awk '{print $2}')
+nworkers=$(echo "${pids}" | wc -l)
+pids=$(echo "${pids}" | paste -sd ' ')
+
+if test -z "${pids}"; then
+  echo "No processes found listening on tcp port '${http_port}'" >&2
+  exit 1
+fi
+
+tmpreport="/tmp/checkfd.$$.txt"
+
+lsof -Pp $(echo "${pids}" | tr ' ' ',') > "${tmpreport}"
+
+maxdb=0
+maxredis=0
+maxhttp=0
+maxtot=0
+maxcache=0
+
+for pid in ${pids}; do
+
+  cnt=$(grep "${pid}" "${tmpreport}" | grep ":${pgsql_port} " | wc -l);
+  if test $cnt -gt $maxdb; then maxdb=$cnt; fi
+  
+  cnt=$(grep "${pid}" "${tmpreport}" | grep ":${redis_port} " | wc -l);
+  if test $cnt -gt $maxredis; then maxredis=$cnt; fi
+
+  cnt=$(grep "${pid}" "${tmpreport}" | grep ":${http_port}-" | grep -v "LISTEN" | wc -l);
+  if test $cnt -gt $maxhttp; then maxhttp=$cnt; fi
+
+  cnt=$(grep "${pid}" "${tmpreport}" | wc -l);
+  if test $cnt -gt $maxtot; then maxtot=$cnt; fi
+
+  log=$(grep "${pid}" "${tmpreport}" | grep -w 1w | awk '{print $9}')
+  if test -e "${log}"; then
+    kill -USR2 "${pid}"
+    cnt=$(tac ${log} | sed -n -e '/ItemKey/p;/ RenderCache /q' | wc -l)
+    if test $cnt -gt $maxcache; then maxcache=$cnt; fi
+  else
+    # report the error...
+    maxcache=-1
+  fi
+
+done
+
+echo "procs.value ${nworkers}"
+echo "pgsql.value ${maxdb}"
+echo "redis.value ${maxredis}"
+echo "http.value ${maxhttp}"
+echo "caches.value ${maxcache}"
+echo "nfd.value ${maxtot}"
+
+rm -f "${tmpreport}"

tools/munin/windshaft.conf.in

@@ -0,0 +1,5 @@
+# Configuration file for munin plugin
+
+[windshaft]
+user root
+env.TILER_ENVIRONMENT @PWD@/../../config/environments/production.js

tools/performance/stresstest_templates.sh

@@ -0,0 +1,54 @@
+#!/bin/sh
+
+verbose=no
+tiler_url=http://dev.localhost.lan:8181/tiles/template
+apikey=${CDB_APIKEY}
+
+while test -n "$1"; do
+  if test "$1" = "-k"; then
+    shift
+    apikey="$1"
+  elif test "$1" = "-u"; then
+    shift
+    tiler_url="$1"
+  elif test -z "$tpl"; then
+    tpl="$1"
+  else
+    echo "Unused parameter $1" >&2
+  fi
+  shift
+done
+
+if test -z "$tpl"; then
+  echo "Usage: $0 [-v] [-k <api_key>] [-u <tiler_url>] <template_config>" >&2
+  echo "Default <tiler_url> is ${tiler_url}" >&2
+  echo "Default <api_key> is read from CDB_APIKEY env variable" >&2
+  exit 1
+fi
+
+basedir=$(cd $(dirname $0); cd ..; pwd)
+export CDB_APIKEY=${apikey}
+max=3000000
+i=0
+while test "$i" -le "$max"; do
+  tpln=`cat ${tpl} | sed "s/\"name\":\"\(.*\)\"/\"name\":\"\1${i}\"/"`
+  tpl_id=`echo ${tpln} | ${basedir}/create_template -u ${tiler_url} /dev/stdin`
+  if test $? -ne 0; then
+    echo $tpl_id >&2
+    break
+  fi
+  tpl_id=`echo ${tpln} | ${basedir}/update_template -u ${tiler_url} ${tpl_id} /dev/stdin`
+  if test $? -ne 0; then
+    echo $tpl_id >&2
+    break
+  fi
+  out=`${basedir}/delete_template -u ${tiler_url} ${tpl_id}`
+  if test $? -ne 0; then
+    echo $out >&2
+    break
+  fi
+  i=$((i+1))
+  if test `expr $i % 100` -eq 0; then
+    echo -n "."
+  fi
+done

tools/reset_styles

@@ -1,35 +1,105 @@
 #!/usr/bin/env node
 
-// Reset redis-stored XML styles so that they are regenerated
-// from CartoCSS on next tile request
+/*
 
-var redis = require('redis')
+This scripts drops all extended map_style keys in redis and regenerates
+the XML caches in all the base ones to target the configured mapnik_version.
 
-var REDIS_PORT = 6379; // TODO: make a parameter
+Optionally (with --convert) it also re-writes the CartoCSS if needed
+to target the configured mapnik_version.
+
+It is recommended to make a backup of the redis database before using
+this script.
+
+*/
+
+var path = require('path');
+
+// Reset all styles in the store
+var grainstore = require('../node_modules/windshaft/node_modules/grainstore/lib/grainstore');
+var mapnik = require('mapnik');
+var redis = require('redis');
+
+function usage(me, exitcode) {
+  console.log("Usage: " + me + " [--convert] <environment>");
+  process.exit(exitcode);
+}
+
+var doConvert = false;
+
+var node_path = process.argv.shift(); 
+var script_path = process.argv.shift(); 
+var me = path.basename(script_path);
+var ENV;
+var arg;
+while ( arg = process.argv.shift() ) {
+  if ( arg == '--convert' ) {
+    doConvert = true;
+  }
+  else if ( ! ENV ) {
+    ENV = arg;
+  }
+  else {
+    usage(me, 1);
+  }
+}
+
+if ( ! ENV ) usage(me, 1);
+
+global.environment = require('../config/environments/' + ENV);
+var serverOptions = require('../lib/cartodb/server_options'); // _after_ setting global.environment
+
+var MAPNIK_VERSION = global.environment.mapnik_version || mapnik.versions.mapnik;
+
+console.log( (doConvert ? "Converting" : "Resetting" ) + ' all styles to target ' + MAPNIK_VERSION);
 
 var dbnum = 0;
 
-var client = redis.createClient(REDIS_PORT, 'localhost');
+var mml_store = new grainstore.MMLStore(serverOptions.redis, serverOptions.grainstore);
+
+var failures = [];
+
+var client = redis.createClient(serverOptions.redis.port, serverOptions.redis.host);
 client.on('connect', function() {
   client.select(dbnum);
   client.keys('map_style|*', function(err, matches) {
 
     processNext = function() {
-      if ( ! matches.length ) process.exit(0);
+      if ( ! matches.length ) process.exit(failures.length);
       var k = matches.shift();
-      console.log("Resetting XML in key: " + k);
-      client.get(k, function(err, val) {
-        if ( err ) throw err;
-        val = JSON.parse(val);
-        delete val.xml;
-        client.set(k, JSON.stringify(val), function() {
-          console.log("done with style " + k);
+
+      if ( /map_style\|.*\|.*\|/.test(k) ) {
+        // See https://github.com/Vizzuality/Windshaft-cartodb/issues/58
+        //console.warn("Key " + k + " is EXTENDED, dropping");
+        client.del(k, function(err) {
+          if ( err ) console.warn("Error dropping key " + k);
           processNext();
         });
+        return;
+      } 
+
+      var params = RegExp(/map_style\|(.*)\|(.*)/).exec(k);
+      var db = params[1];
+      var tab = params[2];
+      var out = 'map_style|' + db + '|' + tab + ': ';
+
+      var mml_builder = mml_store.mml_builder({dbname:db, table:tab},
+                                              function(err, payload) {
+
+          if ( err ) { console.warn(out + err.message); failures.push(k); processNext(); }
+          else {
+            mml_builder.resetStyle(function(err, data) {
+              if ( err ) { console.warn(out + err.message); failures.push(k); }
+              else console.log(out + 'OK' + ( doConvert ? ' (converted)' : '' ));
+              processNext();
+            }, doConvert);
+          }
       });
-    }
+
+    };
 
     processNext();
 
   });
+
 });

tools/show_style

@@ -0,0 +1,148 @@
+#!/usr/bin/env node
+
+var path = require('path');
+var redis = require('redis');
+var Step = require('step');
+
+
+function usage(me, exitcode) {
+  console.log("Usage: " + me + " [--env <environment>] <username> [<tablename>|~<token>]");
+  process.exit(exitcode);
+}
+
+var node_path = process.argv.shift(); 
+var script_path = process.argv.shift(); 
+var basedir = path.dirname(script_path);
+var me = path.basename(script_path);
+
+var ENV = 'development.js';
+var username, token;
+var arg;
+while ( arg = process.argv.shift() ) {
+  if ( arg == '--env' ) {
+    ENV = process.argv.shift();
+  }
+  else if ( ! username ) {
+    username = arg;
+  }
+  else if ( ! token ) {
+    token = arg;
+  }
+  else {
+    console.warn("Unused parameter " + arg);
+  }
+}
+
+if ( ! username ) usage(me, 1);
+
+console.log("Using environment " + ENV);
+
+global.environment = require('../config/environments/' + ENV);
+// _after_ setting global.environment
+var serverOptions = require('../lib/cartodb/server_options')();
+
+var client;
+var dbname;
+Step(
+  function getClient() {
+    client = redis.createClient(serverOptions.redis.port, serverOptions.redis.host);
+    client.on('connect', this);
+  },
+  function getUserMeta(err) {
+    if ( err ) throw err;
+    client.select(5);
+    client.hgetall('rails:users:' + username, this);
+  },
+  function readDB(err, data) {
+    if ( err ) throw err;
+    if ( ! data )
+      throw new Error('Username ' + username + ' unknown by redis on port '
+        + serverOptions.redis.port + ' (try CARTODB/script/restore_redis?)');
+    //console.log("Data:"); console.dir(data);
+    dbname = data['database_name'];
+    console.log("Database name for user " + username + ": " + dbname);
+    client.select(0);
+    return null;
+  },
+  function showTokens(err) {
+    if ( err ) throw err;
+    if ( token ) return null;
+    var next = this;
+    Step(
+      function getTokens() {
+        client.keys('map_style|' + dbname + '|*', this);
+      },
+      function showTokens(err, data) {
+        if (err) throw err;
+        if ( data ) console.log(data.join('\n'));
+        return null;
+      },
+      function showTokensFinish(err) {
+        next(err);
+      }
+    );
+  },
+  function showStyle(err) {
+    if ( err ) throw err;
+    if ( ! token ) return null;
+    var next = this;
+    Step(
+      function getStyle() {
+        client.get('map_style|' + dbname + '|' + token, this);
+      },
+      function showStyle(err, data) {
+        if ( err ) throw err;
+        if ( ! data ) {
+          throw new Error(token + ': no such map style known by redis on port '
+            + serverOptions.redis.port);
+        }
+//console.log("data: " + data);
+        var x=JSON.parse(data);
+        printMapnikStyle(x, this);
+      },
+      function showStyleFinish(err) {
+        next(err);
+      }
+    );
+  },
+  function finish(err) {
+    if ( err ) {
+      console.error(err.message)
+      process.exit(1);
+    }
+    process.exit(0);
+  }
+);
+
+function printMapnikStyle(x, callback) {
+  console.log('style: ' + x.style);
+  console.log('version: ' + x.version);
+  var grainstore = require(basedir + '/../node_modules/windshaft/node_modules/grainstore/lib/grainstore');
+  var mml_store = new grainstore.MMLStore(serverOptions.redis, serverOptions.grainstore);
+  var builderconfig = {dbname:dbname};
+  if ( token.match(/^~/) ) {
+    builderconfig.token = token.substring(1);
+  } else {
+    builderconfig.table = token;
+  }
+  var mml_builder;
+  Step(
+    function getBuilder() {
+      mml_builder = mml_store.mml_builder(builderconfig, this);
+    },
+    function getXML(err, builder) {
+      if ( err ) throw err;
+      mml_builder.toXML(this);
+    },
+    function showXML(err, xml) {
+      if ( err ) throw err;
+      console.log('- XML - ');
+      console.log(xml);
+      return null;
+    },
+    function finish(err) {
+      callback(err);
+    }
+  );
+}
+

tools/update_template

@@ -0,0 +1,47 @@
+#!/bin/sh
+
+verbose=no
+tiler_url=http://dev.localhost.lan:8181/tiles/template
+apikey=${CDB_APIKEY}
+
+while test -n "$1"; do
+  if test "$1" = "-v"; then
+    verbose=yes
+  elif test "$1" = "-k"; then
+    shift
+    apikey="$1"
+  elif test "$1" = "-u"; then
+    shift
+    tiler_url="$1"
+  elif test -z "$tpl"; then
+    tpl="$1"
+  elif test -z "$cfg"; then
+    cfg="$1"
+  else
+    echo "Unused parameter $1" >&2
+  fi
+  shift
+done
+
+if test -z "$cfg"; then
+  echo "Usage: $0 [-v] [-k <api_key>] [-u <tiler_url>] <template_id> <template_config>" >&2
+  echo "Default <tiler_url> is ${tiler_url}" >&2
+  echo "Default <api_key> is read from CDB_APIKEY env variable" >&2
+  exit 1
+fi
+
+cmd="curl -X PUT -skH Content-Type:application/json --data-binary @- ${tiler_url}/${tpl}?api_key=${apikey}"
+if test x${verbose} = xyes; then
+  cmd="${cmd} -v"
+fi
+res=`cat ${cfg} | tr '\n' ' ' | ${cmd}`
+if test $? -gt 0; then
+  echo "curl command failed: ${cmd}"
+fi
+
+if test x${verbose} = xyes; then
+  echo "${res}"
+fi
+
+tok=`echo "$res" | sed 's/.*"template_id":"\([^"]*\)".*/\1/'`
+echo $tok