Release 2.39.0

Use a more aggressive cache control header for node status endpoint

.gitignore

@@ -7,3 +7,5 @@ logs/
 pids/
 redis.pid
 test.log
+npm-debug.log
+coverage/

.jshintignore

@@ -0,0 +1,4 @@
+test/results/
+test/monkey/
+test/benchmark.js
+test/support/

.jshintrc

@@ -0,0 +1,95 @@
+{
+//    // JSHint Default Configuration File (as on JSHint website)
+//    // See http://jshint.com/docs/ for more details
+//
+//    "maxerr"        : 50,       // {int} Maximum error before stopping
+//
+//    // Enforcing
+//    "bitwise"       : true,     // true: Prohibit bitwise operators (&, |, ^, etc.)
+//    "camelcase"     : false,    // true: Identifiers must be in camelCase
+    "curly"         : true,     // true: Require {} for every new block or scope
+    "eqeqeq"        : true,     // true: Require triple equals (===) for comparison
+    "forin"         : true,     // true: Require filtering for..in loops with obj.hasOwnProperty()
+    "freeze"        : true,     // true: prohibits overwriting prototypes of native objects such as Array, Date etc.
+    "immed"         : true,    // true: Require immediate invocations to be wrapped in parens e.g. `(function () { } ());`
+//    "indent"        : 4,        // {int} Number of spaces to use for indentation
+//    "latedef"       : false,    // true: Require variables/functions to be defined before being used
+    "newcap"        : true,    // true: Require capitalization of all constructor functions e.g. `new F()`
+    "noarg"         : true,     // true: Prohibit use of `arguments.caller` and `arguments.callee`
+//    "noempty"       : true,     // true: Prohibit use of empty blocks
+    "nonbsp"        : true,     // true: Prohibit "non-breaking whitespace" characters.
+    "nonew"         : true,    // true: Prohibit use of constructors for side-effects (without assignment)
+//    "plusplus"      : false,    // true: Prohibit use of `++` & `--`
+//    "quotmark"      : false,    // Quotation mark consistency:
+//    //   false    : do nothing (default)
+//    //   true     : ensure whatever is used is consistent
+//    //   "single" : require single quotes
+//    //   "double" : require double quotes
+    "undef"         : true,     // true: Require all non-global variables to be declared (prevents global leaks)
+    "unused"        : true,     // true: Require all defined variables be used
+//    "strict"        : true,     // true: Requires all functions run in ES5 Strict Mode
+//    "maxparams"     : false,    // {int} Max number of formal params allowed per function
+//    "maxdepth"      : false,    // {int} Max depth of nested blocks (within functions)
+//    "maxstatements" : false,    // {int} Max number statements per function
+    "maxcomplexity" : 6,    // {int} Max cyclomatic complexity per function
+    "maxlen"        : 120,    // {int} Max number of characters per line
+//
+//    // Relaxing
+//    "asi"           : false,     // true: Tolerate Automatic Semicolon Insertion (no semicolons)
+//    "boss"          : false,     // true: Tolerate assignments where comparisons would be expected
+    "debug"         : false,     // true: Allow debugger statements e.g. browser breakpoints.
+//    "eqnull"        : false,     // true: Tolerate use of `== null`
+//    "es5"           : false,     // true: Allow ES5 syntax (ex: getters and setters)
+//    "esnext"        : false,     // true: Allow ES.next (ES6) syntax (ex: `const`)
+//    "moz"           : false,     // true: Allow Mozilla specific syntax (extends and overrides esnext features)
+//    // (ex: `for each`, multiple try/catch, function expression…)
+//    "evil"          : false,     // true: Tolerate use of `eval` and `new Function()`
+//    "expr"          : false,     // true: Tolerate `ExpressionStatement` as Programs
+//    "funcscope"     : false,     // true: Tolerate defining variables inside control statements
+//    "globalstrict"  : false,     // true: Allow global "use strict" (also enables 'strict')
+//    "iterator"      : false,     // true: Tolerate using the `__iterator__` property
+//    "lastsemic"     : false,     // true: Tolerate omitting a semicolon for the last statement of a 1-line block
+//    "laxbreak"      : false,     // true: Tolerate possibly unsafe line breakings
+//    "laxcomma"      : false,     // true: Tolerate comma-first style coding
+//    "loopfunc"      : false,     // true: Tolerate functions being defined in loops
+//    "multistr"      : false,     // true: Tolerate multi-line strings
+//    "noyield"       : false,     // true: Tolerate generator functions with no yield statement in them.
+//    "notypeof"      : false,     // true: Tolerate invalid typeof operator values
+//    "proto"         : false,     // true: Tolerate using the `__proto__` property
+//    "scripturl"     : false,     // true: Tolerate script-targeted URLs
+//    "shadow"        : false,     // true: Allows re-define variables later in code e.g. `var x=1; x=2;`
+//    "sub"           : false,     // true: Tolerate using `[]` notation when it can still be expressed in dot notation
+//    "supernew"      : false,     // true: Tolerate `new function () { ... };` and `new Object;`
+//    "validthis"     : false,     // true: Tolerate using this in a non-constructor function
+//
+//    // Environments
+//    "browser"       : true,     // Web Browser (window, document, etc)
+//    "browserify"    : false,    // Browserify (node.js code in the browser)
+//    "couch"         : false,    // CouchDB
+//    "devel"         : true,     // Development/debugging (alert, confirm, etc)
+//    "dojo"          : false,    // Dojo Toolkit
+//    "jasmine"       : false,    // Jasmine
+//    "jquery"        : false,    // jQuery
+//    "mocha"         : true,     // Mocha
+//    "mootools"      : false,    // MooTools
+    "node"          : true,    // Node.js
+//    "nonstandard"   : false,    // Widely adopted globals (escape, unescape, etc)
+//    "prototypejs"   : false,    // Prototype and Scriptaculous
+//    "qunit"         : false,    // QUnit
+//    "rhino"         : false,    // Rhino
+//    "shelljs"       : false,    // ShellJS
+//    "worker"        : false,    // Web Workers
+//    "wsh"           : false,    // Windows Scripting Host
+//    "yui"           : false,    // Yahoo User Interface
+
+    // Custom predefined global variables
+    "predef": [
+        "-console", // disallows console, use debug
+        "beforeEach",
+        "afterEach",
+        "before",
+        "after",
+        "describe",
+        "it"
+    ]
+}

.travis.yml

@@ -1,9 +1,18 @@
+sudo: false
+
 addons:
   postgresql: "9.3"
+  apt:
+    packages:
+      - pkg-config
+      - libcairo2-dev
+      - libjpeg8-dev
+      - libgif-dev
 
 before_install:
-  - sudo apt-get install -y pkg-config libcairo2-dev libjpeg8-dev libgif-dev
+  - npm install -g npm@2
   - createdb template_postgis
+  - createuser publicuser
   - psql -c "CREATE EXTENSION postgis" template_postgis
 
 env:
@@ -11,7 +20,6 @@ env:
 
 language: node_js
 node_js:
-  - "0.8"
   - "0.10"
 
 notifications:

CONTRIBUTING.md

@@ -0,0 +1,11 @@
+Contributing
+---
+
+The issue tracker is at [github.com/CartoDB/Windshaft-cartodb](https://github.com/CartoDB/Windshaft-cartodb).
+
+We love pull requests from everyone, see [Contributing to Open Source on GitHub](https://guides.github.com/activities/contributing-to-open-source/#contributing).
+
+
+## Submitting Contributions
+
+* You will need to sign a Contributor License Agreement (CLA) before making a submission. [Learn more here](https://cartodb.com/contributing).

HOWTO_RELEASE

@@ -1,12 +1,11 @@
 1. Test (make clean all check), fix if broken before proceeding
 2. Ensure proper version in package.json 
 3. Ensure NEWS section exists for the new version, review it, add release date
-4. Drop npm-shrinkwrap.json
-5. Run npm shrinkwrap to recreate npm-shrinkwrap.json
-6. Commit package.json, npm-shrinwrap.json, NEWS
-7. git tag -a Major.Minor.Patch # use NEWS section as content
-8. Announce on cartodb@googlegroups.com
-9. Stub NEWS/package for next version
+4. Recreate npm-shrinkwrap.json with: `npm install --no-shrinkwrap && npm shrinkwrap`
+5. Commit package.json, npm-shrinwrap.json, NEWS
+6. git tag -a Major.Minor.Patch # use NEWS section as content
+7. Announce on cartodb@googlegroups.com
+8. Stub NEWS/package for next version
 
 Versions:
 

INSTALL.md

@@ -0,0 +1,53 @@
+# Installing Windshaft-CartoDB #
+
+## Requirements ##
+Make sure that you have the requirements needed. These are
+
+- Core
+  - Node.js >=0.8
+  - npm >=1.2.1 <2.0.0
+  - PostgreSQL >8.3.x, PostGIS >1.5.x
+  - Redis >2.4.0 (http://www.redis.io)
+  - Mapnik 2.0.1, 2.0.2, 2.1.0, 2.2.0, 2.3.0. See [Installing Mapnik](https://github.com/CartoDB/Windshaft#installing-mapnik).
+  - Windshaft: check [Windshaft dependencies and installation notes](https://github.com/CartoDB/Windshaft#dependencies)
+  - libcairo2-dev, libpango1.0-dev, libjpeg8-dev and libgif-dev for server side canvas support
+
+- For cache control (optional)
+  - CartoDB 0.9.5+ (for `CDB_QueryTables`)
+  - Varnish (http://www.varnish-cache.org)
+
+On Ubuntu 14.04 the dependencies can be installed with
+
+```shell
+sudo apt-get update
+sudo apt-get install -y make g++ pkg-config git-core \
+  libgif-dev libjpeg-dev libcairo2-dev \
+  libhiredis-dev redis-server \
+  nodejs nodejs-legacy npm \
+  postgresql-9.3-postgis-2.1 postgresql-plpython-9.3 postgresql-server-dev-9.3
+```
+
+On Ubuntu 12.04 the [cartodb/cairo PPA](https://launchpad.net/~cartodb/+archive/ubuntu/cairo) may be useful.
+
+## PostGIS setup ##
+
+A `template_postgis` database is expected. One can be set up with
+
+```shell
+createdb --owner postgres --template template0 template_postgis
+psql -d template_postgis -c 'CREATE EXTENSION postgis;'
+```
+
+## Build/install ##
+
+To fetch and build all node-based dependencies, run:
+
+```
+npm install
+```
+
+Note that the ```npm install``` step will populate the node_modules/
+directory with modules, some of which being compiled on demand. If you
+happen to have startup errors you may need to force rebuilding those
+modules. At any time just wipe out the node_modules/ directory and run
+```npm install``` again.

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2014, Vizzuality
+Copyright (c) 2015, CartoDB
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without 

Makefile

@@ -1,7 +1,10 @@
-srcdir=$(shell pwd)
+SHELL=/bin/bash
+
+pre-install:
+	@$(SHELL) ./scripts/check-node-canvas.sh
 
 all:
-	npm install
+	@$(SHELL) ./scripts/install.sh
 
 clean:
 	rm -rf node_modules/*
@@ -15,21 +18,36 @@ config.status--test:
 config/environments/test.js: config.status--test
 	./config.status--test 
 
-check-local: config/environments/test.js
-	./run_tests.sh ${RUNTESTFLAGS} \
-    test/unit/cartodb/*.js \
-    test/acceptance/*.js 
+TEST_SUITE := $(shell find test/{acceptance,integration,unit} -name "*.js")
+TEST_SUITE_UNIT := $(shell find test/unit -name "*.js")
+TEST_SUITE_INTEGRATION := $(shell find test/integration -name "*.js")
+TEST_SUITE_ACCEPTANCE := $(shell find test/acceptance -name "*.js")
 
-check-submodules:
-	PATH="$$PATH:$(srcdir)/node_modules/.bin/"; \
-	for sub in windshaft grainstore node-varnish mapnik; do \
-	  if test -e node_modules/$${sub}; then \
-	      echo "Testing submodule $${sub}"; \
-	      make -C node_modules/$${sub} check || exit 1; \
-	  fi; \
-	done
+test: config/environments/test.js
+	@echo "***tests***"
+	@$(SHELL) ./run_tests.sh ${RUNTESTFLAGS} $(TEST_SUITE)
 
-check-full: check-local check-submodules
+test-unit: config/environments/test.js
+	@echo "***tests***"
+	@$(SHELL) ./run_tests.sh ${RUNTESTFLAGS} $(TEST_SUITE_UNIT)
 
-check: check-local
+test-integration: config/environments/test.js
+	@echo "***tests***"
+	@$(SHELL) ./run_tests.sh ${RUNTESTFLAGS} $(TEST_SUITE_INTEGRATION)
 
+test-acceptance: config/environments/test.js
+	@echo "***tests***"
+	@$(SHELL) ./run_tests.sh ${RUNTESTFLAGS} $(TEST_SUITE_ACCEPTANCE)
+
+jshint:
+	@echo "***jshint***"
+	@./node_modules/.bin/jshint lib/ test/ app.js
+
+test-all: jshint test
+
+coverage:
+	@RUNTESTFLAGS=--with-coverage make test
+
+check: test
+
+.PHONY: pre-install test jshint coverage

README.md

@@ -1,39 +1,21 @@
 Windshaft-CartoDB
 ==================
 
-[![Build Status](http://travis-ci.org/CartoDB/Windshaft-cartodb.png)]
-(http://travis-ci.org/CartoDB/Windshaft-cartodb)
+[![Build Status](https://travis-ci.org/CartoDB/Windshaft-cartodb.svg?branch=master)](https://travis-ci.org/CartoDB/Windshaft-cartodb)
 
-This is the CartoDB map tiler. It extends Windshaft with some extra
-functionality and custom filters for authentication
+This is the [CartoDB Maps API](http://docs.cartodb.com/cartodb-platform/maps-api.html) tiler. It extends
+[Windshaft](https://github.com/CartoDB/Windshaft) with some extra functionality and custom filters for authentication.
 
 * reads dbname from subdomain and cartodb redis for pretty tile urls
-* configures windshaft to publish ``cartodb_id`` as the interactivity layer
+* configures windshaft to publish `cartodb_id` as the interactivity layer
 * gets the default geometry type from the cartodb redis store
 * allows tiles to be styled individually
 * provides a link to varnish high speed cache
-* provides a ``infowindow`` endpoint for windshaft (DEPRECATED)
-* provides a ``map_metadata`` endpoint for windshaft (DEPRECATED)
-* provides signed template maps API
-  (http://github.com/CartoDB/Windshaft-cartodb/wiki/Template-maps)
+* provides a [template maps API](https://github.com/CartoDB/Windshaft-cartodb/blob/master/docs/Template-maps.md)
 
-Requirements
-------------
-
- [core]
- - node-0.8.x+
- - PostgreSQL-8.3+
- - PostGIS-1.5.0+
- - Redis 2.4.0+ (http://www.redis.io)
- - Mapnik 2.0 or 2.1 
-
- [for cache control]
- - CartoDB-SQL-API 1.0.0+
- - CartoDB 0.9.5+ (for ``CDB_QueryTables``)
- - Varnish (http://www.varnish-cache.org)
-
- [for running the testsuite]
- - Imagemagick (http://www.imagemagick.org)
+Install
+-------
+See [INSTALL.md](INSTALL.md) for detailed installation instructions.
 
 Configure
 ---------
@@ -44,23 +26,15 @@ see ```./configure --help``` to see available options.
 
 Look at lib/cartodb/server_options.js for more on config
 
-Build/install
--------------
+Upgrading
+---------
 
-To fetch and build all node-based dependencies, run:
+Checkout your commit/branch. If you need to reinstall dependencies (you can check [NEWS](NEWS.md)) do the following:
 
 ```
-git clone
-npm install
+rm -rf node_modules; npm install
 ```
 
-Note that the ```npm install``` step will populate the node_modules/
-directory with modules, some of which being compiled on demand. If you
-happen to have startup errors you may need to force rebuilding those
-modules. At any time just wipe out the node_modules/ directory and run
-```npm install``` again.
-
-
 Run
 ---
 
@@ -75,59 +49,34 @@ there may be out-of-sync records in there.
 Take a look: http://redis.io/commands
 
 
-URLs
-----
+Documentation
+-------------
 
-**TILES**
-
-[GET] subdomain.cartodb.com/tiles/:table_name/:z/:x/:y.[png|png8|grid.json]
-
-Args:
-
-* sql - plain SQL arguments
-* interactivity - specify the column to use in UTFGrid
-* cache_buster - Specify an identifier for the internal tile cache.
-                 Requesting tiles with the same cache_buster value may
-                 result in being served a cached version of the tile
-                 (even when requesting a tile for the first time, as tiles
-                 can be prepared in advance)
-* cache_policy - Set to "persist" to have the server send an Cache-Control
-                 header requesting caching devices to keep the response
-                 cached as much as possible. This is best used with a
-                 timestamp value in cache_buster for manual control of
-                 updates.
-* geom_type - override the cartodb default
-* style - override the default map style with Carto
+The [docs directory](https://github.com/CartoDB/Windshaft-cartodb/tree/master/docs) contains different documentation
+resources, from higher level to more detailed ones:
+The [Maps API](https://github.com/CartoDB/Windshaft-cartodb/blob/master/docs/Map-API.md) defined the endpoints and their
+expected parameters and outputs.
 
 
-**STYLE**
+Examples
+--------
 
-[GET/POST] subdomain.cartodb.com/tiles/:table_name/style
+[CartoDB's Map Gallery](http://cartodb.com/gallery/) showcases several examples of visualisations built on top of this.
 
-Args:
+Contributing
+---
 
-* style - the style in CartoCSS you want to set
-* style_version - the version of the style for POST
-* style_convert - request conversion to target version (both POST and GET)
+See [CONTRIBUTING.md](CONTRIBUTING.md).
 
+### Developing with a custom windshaft version
 
-**INFOWINDOW**
+If you plan or want to use a custom / not released yet version of windshaft (or any other dependency) the best option is
+to use `npm link`. You can read more about it at [npm-link: Symlink a package folder](https://docs.npmjs.com/cli/link).
 
-[GET] subdomain.cartodb.com/tiles/:table_name/infowindow
+**Quick start**:
 
-Args:
-
-* infowindow - returns contents of infowindow from CartoDB.
-
-
-**MAP METADATA**
-
-[GET] subdomain.cartodb.com/tiles/:table_name/map_metadata
-
-Args:
-
-* infowindow - returns contents of infowindow from CartoDB.
-
-
-All GET requests are wrappable with JSONP using callback argument,
-including the UTFGrid map tile call.
+```shell
+~/windshaft-directory $ npm install
+~/windshaft-directory $ npm link
+~/windshaft-cartodb-directory $ npm link windshaft
+```

app.js

@@ -1,56 +1,72 @@
-/*
- * Windshaft-CartoDB
- * ===============
- *
- * ./app.js [environment]
- *
- * environments: [development, production]
- */
-
-var path = require('path'),
-    fs = require('fs'),
-    RedisPool = require('redis-mpool')
-;
-
-
-if ( process.argv[2] ) ENV = process.argv[2];
-else if ( process.env['NODE_ENV'] ) ENV = process.env['NODE_ENV'];
-else ENV = 'development';
-
-process.env['NODE_ENV'] = ENV;
-
-// sanity check
-if (ENV != 'development' && ENV != 'production' && ENV != 'staging' ){
-    console.error("\nnode app.js [environment]");
-    console.error("environments: development, production, staging\n");
-    process.exit(1);
-}
+var http = require('http');
+var https = require('https');
+var path = require('path');
+var fs = require('fs');
 
 var _ = require('underscore');
 
+var ENVIRONMENT;
+if ( process.argv[2] ) {
+    ENVIRONMENT = process.argv[2];
+} else if ( process.env.NODE_ENV ) {
+    ENVIRONMENT = process.env.NODE_ENV;
+} else {
+    ENVIRONMENT = 'development';
+}
+
+// jshint undef:false
+var log = console.log.bind(console);
+var logError = console.error.bind(console);
+// jshint undef:true
+
+var availableEnvironments = {
+    production: true,
+    staging: true,
+    development: true
+};
+
+// sanity check
+if (!availableEnvironments[ENVIRONMENT]){
+    logError('node app.js [environment]');
+    logError('environments: %s', Object.keys(availableEnvironments).join(', '));
+    process.exit(1);
+}
+
+process.env.NODE_ENV = ENVIRONMENT;
+
 // set environment specific variables
-global.environment  = require(__dirname + '/config/environments/' + ENV);
-global.environment.api_hostname = require('os').hostname().split('.')[0];
+global.environment = require('./config/environments/' + ENVIRONMENT);
 
 global.log4js = require('log4js');
-log4js_config = {
+var log4js_config = {
   appenders: [],
-  replaceConsole:true
+  replaceConsole: true
 };
 
 if (global.environment.uv_threadpool_size) {
     process.env.UV_THREADPOOL_SIZE = global.environment.uv_threadpool_size;
 }
 
+// set global HTTP and HTTPS agent default configurations
+// ref https://nodejs.org/api/http.html#http_new_agent_options
+var agentOptions = _.defaults(global.environment.httpAgent || {}, {
+    keepAlive: false,
+    keepAliveMsecs: 1000,
+    maxSockets: Infinity,
+    maxFreeSockets: 256
+});
+http.globalAgent = new http.Agent(agentOptions);
+https.globalAgent = new https.Agent(agentOptions);
+
 if ( global.environment.log_filename ) {
   var logdir = path.dirname(global.environment.log_filename);
   // See cwd inlog4js.configure call below
   logdir = path.resolve(__dirname, logdir);
   if ( ! fs.existsSync(logdir) ) {
-    console.error("Log filename directory does not exist: " + logdir);
+    logError("Log filename directory does not exist: " + logdir);
     process.exit(1);
   }
-  console.log("Logs will be written to " + global.environment.log_filename);
+  log("Logs will be written to " + global.environment.log_filename);
   log4js_config.appenders.push(
     { type: "file", filename: global.environment.log_filename }
   );
@@ -60,69 +76,50 @@ if ( global.environment.log_filename ) {
   );
 }
 
-if ( global.environment.rollbar ) {
-  log4js_config.appenders.push({
-    type: __dirname + "/lib/cartodb/log4js_rollbar.js",
-    options: global.environment.rollbar
-  });
-}
+global.log4js.configure(log4js_config, { cwd: __dirname });
+global.logger = global.log4js.getLogger();
 
-log4js.configure(log4js_config, { cwd: __dirname });
-global.logger = log4js.getLogger();
-
-var redisOpts = _.extend(global.environment.redis, { name: 'windshaft' }),
-    redisPool = new RedisPool(redisOpts);
+global.environment.api_hostname = require('os').hostname().split('.')[0];
 
 // Include cartodb_windshaft only _after_ the "global" variable is set
 // See https://github.com/Vizzuality/Windshaft-cartodb/issues/28
-var CartodbWindshaft = require('./lib/cartodb/cartodb_windshaft'),
-    serverOptions = require('./lib/cartodb/server_options')(redisPool);
+var cartodbWindshaft = require('./lib/cartodb/server');
+var serverOptions = require('./lib/cartodb/server_options');
 
-ws = CartodbWindshaft(serverOptions);
-
-if (global.statsClient) {
-    redisPool.on('status', function(status) {
-        var keyPrefix = 'windshaft.redis-pool.' + status.name + '.db' + status.db + '.';
-        global.statsClient.gauge(keyPrefix + 'count', status.count);
-        global.statsClient.gauge(keyPrefix + 'unused', status.unused);
-        global.statsClient.gauge(keyPrefix + 'waiting', status.waiting);
-    });
-}
+var server = cartodbWindshaft(serverOptions);
 
 // Maximum number of connections for one process
 // 128 is a good number if you have up to 1024 filedescriptors
 // 4 is good if you have max 32 filedescriptors
 // 1 is good if you have max 16 filedescriptors
-ws.maxConnections = global.environment.maxConnections || 128;
+var backlog = global.environment.maxConnections || 128;
 
-ws.listen(global.environment.port, global.environment.host);
+var listener = server.listen(serverOptions.bind.port, serverOptions.bind.host, backlog);
 
 var version = require("./package").version;
 
-ws.on('listening', function() {
-  console.log("Windshaft tileserver " + version + " started on "
-              + global.environment.host + ':' + global.environment.port
-              + " (" + ENV + ")");
+listener.on('listening', function() {
+    log(
+        "Windshaft tileserver %s started on %s:%s PID=%d (%s)",
+        version, serverOptions.bind.host, serverOptions.bind.port, process.pid, ENVIRONMENT
+    );
 });
 
-// DEPRECATED, use SIGUSR2
-process.on('SIGUSR1', function() {
-  console.log('WARNING: handling of SIGUSR1 by Windshaft-CartoDB is deprecated, please send SIGUSR2 instead');
-  ws.dumpCacheStats();
-});
-
-process.on('SIGUSR2', function() {
-  ws.dumpCacheStats();
-});
+setInterval(function() {
+    var memoryUsage = process.memoryUsage();
+    Object.keys(memoryUsage).forEach(function(k) {
+        global.statsClient.gauge('windshaft.memory.' + k, memoryUsage[k]);
+    });
+}, 5000);
 
 process.on('SIGHUP', function() {
     global.log4js.clearAndShutdownAppenders(function() {
         global.log4js.configure(log4js_config);
-        global.logger = log4js.getLogger();
-        console.log('Log files reloaded');
+        global.logger = global.log4js.getLogger();
+        log('Log files reloaded');
     });
 });
 
 process.on('uncaughtException', function(err) {
-  logger.error('Uncaught exception: ' + err.stack); 
+    global.logger.error('Uncaught exception: ' + err.stack);
 });

config/environments/development.js.example

@@ -2,6 +2,9 @@ var config = {
      environment: 'development'
     ,port: 8181
     ,host: '127.0.0.1'
+    // Size of the threadpool which can be used to run user code and get notified in the loop thread
+    // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
+    // See http://docs.libuv.org/en/latest/threadpool.html
     ,uv_threadpool_size: undefined
     // Regular expression pattern to extract username
     // from hostname. Must have a single grabbing block.
@@ -10,17 +13,15 @@ var config = {
     // Base URLs for the APIs
     //
     // See http://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API
-    // 
+    //
     // Base url for the Templated Maps API
     // "/api/v1/map/named" is the new API,
     // "/tiles/template" is for compatibility with versions up to 1.6.x
-    ,base_url_templated: '(?:/api/v1/map/named|/tiles/template)'
+    ,base_url_templated: '(?:/api/v1/map/named|/user/:user/api/v1/map/named|/tiles/template)'
     // Base url for the Detached Maps API
     // "maps" is the the new API,
     // "tiles/layergroup" is for compatibility with versions up to 1.6.x
-    ,base_url_detached: '(?:/api/v1/map|/tiles/layergroup)'
-    // Base url for the Inline Maps and Table Maps API
-    ,base_url_legacy: '/tiles/:table'
+    ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)'
 
     // Maximum number of connections for one process
     // 128 is a good value with a limit of 1024 open file descriptors
@@ -32,7 +33,7 @@ var config = {
     // to be able to navigate the map without a reload ?
     // Defaults to 7200 (2 hours)
     ,mapConfigTTL: 7200
-    // idle socket timeout, in milliseconds 
+    // idle socket timeout, in milliseconds
     ,socket_timeout: 600000
     ,enable_cors: true
     ,cache_enabled: false
@@ -86,17 +87,126 @@ var config = {
     ,renderer: {
       // Milliseconds since last access before renderer cache item expires
       cache_ttl: 60000,
-      metatile: 4,
-      bufferSize: 64,
       statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
+      mapnik: {
+          // The size of the pool of internal mapnik backend
+          // This pool size is per mapnik renderer created in Windshaft's RendererFactory
+          // See https://github.com/CartoDB/Windshaft/blob/master/lib/windshaft/renderers/renderer_factory.js
+          // Important: check the configuration of uv_threadpool_size to use suitable value
+          poolSize: 8,
+
+          // Metatile is the number of tiles-per-side that are going
+          // to be rendered at once. If all of them will be requested
+          // we'd have saved time. If only one will be used, we'd have
+          // wasted time.
+          metatile: 2,
+
+          // tilelive-mapnik uses an internal cache to store tiles/grids
+          // generated when using metatile. This options allow to tune
+          // the behaviour for that internal cache.
+          metatileCache: {
+              // Time an object must stay in the cache until is removed
+              ttl: 0,
+              // Whether an object must be removed after the first hit
+              // Usually you want to use `true` here when ttl>0.
+              deleteOnHit: false
+          },
+
+          // Override metatile behaviour depending on the format
+          formatMetatile: {
+              png: 2,
+              'grid.json': 1
+          },
+
+          // Buffer size is the tickness in pixel of a buffer
+          // around the rendered (meta?)tile.
+          //
+          // This is important for labels and other marker that overlap tile boundaries.
+          // Setting to 128 ensures no render artifacts.
+          // 64 may have artifacts but is faster.
+          // Less important if we can turn metatiling on.
+          bufferSize: 64,
+
+          // SQL queries will be wrapped with ST_SnapToGrid
+          // Snapping all points of the  geometry to a regular grid
+          snapToGrid: false,
+
+          // SQL queries will be wrapped with ST_ClipByBox2D
+          // Returning the portion of a geometry falling within a rectangle
+          // It will only work if snapToGrid is enabled
+          clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
+
+          limits: {
+              // Time in milliseconds a render request can take before it fails, some notes:
+              //  - 0 means no render limit
+              //  - it considers metatiling, naive implementation: (render timeout) * (number of tiles in metatile)
+              render: 0,
+              // As the render request will finish even if timed out, whether it should be placed in the internal
+              // cache or it should be fully discarded. When placed in the internal cache another attempt to retrieve
+              // the same tile will result in an immediate response, however that will use a lot of more application
+              // memory. If we want to enforce this behaviour we have to implement a cache eviction policy for the
+              // internal cache.
+              cacheOnTimeout: true
+          },
+
+          geojson: {
+                dbPoolParams: {
+                      // maximum number of resources to create at any given time
+                      size: 16,
+                      // max milliseconds a resource can go unused before it should be destroyed
+                      idleTimeout: 3000,
+                      // frequency to check for idle resources
+                      reapInterval: 1000
+                },
+
+              // SQL queries will be wrapped with ST_ClipByBox2D
+              // Returning the portion of a geometry falling within a rectangle
+              // It will only work if snapToGrid is enabled
+              clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
+          }
+
+      },
       http: {
           timeout: 2000, // the timeout in ms for a http tile request
           proxy: undefined, // the url for a proxy server
           whitelist: [ // the whitelist of urlTemplates that can be used
+              '.*', // will enable any URL
               'http://{s}.example.com/{z}/{x}/{y}.png'
-          ]
+          ],
+          // image to use as placeholder when urlTemplate is not in the whitelist
+          // if provided the http renderer will use it instead of throw an error
+          fallbackImage: {
+              type: 'fs', // 'fs' and 'url' supported
+              src: __dirname + '/../../assets/default-placeholder.png'
+          }
+      },
+      torque: {
+          dbPoolParams: {
+              // maximum number of resources to create at any given time
+              size: 16,
+              // max milliseconds a resource can go unused before it should be destroyed
+              idleTimeout: 3000,
+              // frequency to check for idle resources
+              reapInterval: 1000
+          }
       }
     }
+    // anything analyses related
+    ,analysis: {
+        // batch configuration
+        batch: {
+            // Inline execution avoid the use of SQL API as batch endpoint
+            // When set to true it will run all analysis queries in series, with a direct connection to the DB
+            // This might be useful for:
+            //  - testing
+            //  - running an standalone server without any dependency on external services
+            inlineExecution: false,
+            // where the SQL API is running, it will use a custom Host header to specify the username.
+            endpoint: 'http://127.0.0.1:8080/api/v2/sql/job',
+            // the template to use for adding the host header in the batch api requests
+            hostHeaderTemplate: '{{=it.username}}.localhost.lan'
+        }
+    }
     ,millstone: {
         // Needs to be writable by server user
         cache_basedir: '/tmp/cdb-tiler-dev/millstone-dev'
@@ -125,27 +235,16 @@ var config = {
         },
         emitter: {
             statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd
-        }
+        },
+        unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161
+        noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading
     }
-    ,sqlapi: {
-        protocol: 'http',
-        // If "host" is given, it will be used
-        // to connect to the SQL-API without a
-        // DNS lookup
-        host: '127.0.0.1',
-        port: 8080,
-        // The "domain" part will be appended to 
-        // the cartodb username and passed to
-        // SQL-API requests in the Host HTTP header
-        domain: 'localhost.lan',
-        version: 'v1',
-        // Maximum lenght of SQL query for GET
-        // requests. Longer queries will be sent
-        // using POST. Defaults to 2048
-        max_get_sql_length: 2048,
-        // Maximum time to wait for a response,
-        // in milliseconds. Defaults to 100.
-        timeout: 100
+    // For more details about this options check https://nodejs.org/api/http.html#http_new_agent_options
+    ,httpAgent: {
+        keepAlive: true,
+        keepAliveMsecs: 1000,
+        maxSockets: 25,
+        maxFreeSockets: 256
     }
     ,varnish: {
         host: 'localhost',
@@ -156,8 +255,17 @@ var config = {
         ttl: 86400,
         layergroupTtl: 86400 // the max-age for cache-control header in layergroup responses
     }
+    // this [OPTIONAL] configuration enables invalidating by surrogate key in fastly
+    ,fastly: {
+        // whether the invalidation is enabled or not
+        enabled: false,
+        // the fastly api key
+        apiKey: 'wadus_api_key',
+        // the service that will get surrogate key invalidation
+        serviceId: 'wadus_service_id'
+    }
     // If useProfiler is true every response will be served with an
-    // X-Tiler-Profile header containing elapsed timing for various 
+    // X-Tiler-Profile header containing elapsed timing for various
     // steps taken for producing the response.
     ,useProfiler:true
     // Settings for the health check available at /health
@@ -168,6 +276,18 @@ var config = {
       x: 0,
       y: 0
     }
+    ,disabled_file: 'pids/disabled'
+
+    // Use this as a feature flags enabling/disabling mechanism
+    ,enabledFeatures: {
+        // whether it should intercept tile render errors an act based on them, enabled by default.
+        onTileErrorStrategy: true,
+        // whether the affected tables for a given SQL must query directly postgresql or use the SQL API
+        cdbQueryTablesFromPostgres: true,
+        // whether in mapconfig is available stats & metadata for each layer
+        layerMetadata: true
+
+    }
 };
 
 module.exports = config;

config/environments/production.js.example

@@ -2,6 +2,9 @@ var config = {
     environment: 'production'
     ,port: 8181
     ,host: '127.0.0.1'
+    // Size of the threadpool which can be used to run user code and get notified in the loop thread
+    // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
+    // See http://docs.libuv.org/en/latest/threadpool.html
     ,uv_threadpool_size: undefined
     // Regular expression pattern to extract username
     // from hostname. Must have a single grabbing block.
@@ -10,17 +13,15 @@ var config = {
     // Base URLs for the APIs
     //
     // See http://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API
-    // 
+    //
     // Base url for the Templated Maps API
     // "/api/v1/map/named" is the new API,
     // "/tiles/template" is for compatibility with versions up to 1.6.x
-    ,base_url_templated: '(?:/api/v1/map/named|/tiles/template)'
+    ,base_url_templated: '(?:/api/v1/map/named|/user/:user/api/v1/map/named|/tiles/template)'
     // Base url for the Detached Maps API
     // "maps" is the the new API,
     // "tiles/layergroup" is for compatibility with versions up to 1.6.x
-    ,base_url_detached: '(?:/api/v1/map|/tiles/layergroup)'
-    // Base url for the Inline Maps and Table Maps API
-    ,base_url_legacy: '/tiles/:table'
+    ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)'
 
     // Maximum number of connections for one process
     // 128 is a good value with a limit of 1024 open file descriptors
@@ -33,7 +34,7 @@ var config = {
     // Defaults to 7200 (2 hours)
     ,mapConfigTTL: 7200
     // idle socket timeout, in milliseconds
-    ,socket_timeout: 600000 
+    ,socket_timeout: 600000
     ,enable_cors: true
     ,cache_enabled: true
     ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
@@ -80,17 +81,126 @@ var config = {
     ,renderer: {
       // Milliseconds since last access before renderer cache item expires
       cache_ttl: 60000,
-      metatile: 4,
-      bufferSize: 64,
       statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
+      mapnik: {
+          // The size of the pool of internal mapnik backend
+          // This pool size is per mapnik renderer created in Windshaft's RendererFactory
+          // See https://github.com/CartoDB/Windshaft/blob/master/lib/windshaft/renderers/renderer_factory.js
+          // Important: check the configuration of uv_threadpool_size to use suitable value
+          poolSize: 8,
+
+          // Metatile is the number of tiles-per-side that are going
+          // to be rendered at once. If all of them will be requested
+          // we'd have saved time. If only one will be used, we'd have
+          // wasted time.
+          metatile: 2,
+
+          // tilelive-mapnik uses an internal cache to store tiles/grids
+          // generated when using metatile. This options allow to tune
+          // the behaviour for that internal cache.
+          metatileCache: {
+              // Time an object must stay in the cache until is removed
+              ttl: 0,
+              // Whether an object must be removed after the first hit
+              // Usually you want to use `true` here when ttl>0.
+              deleteOnHit: false
+          },
+
+          // Override metatile behaviour depending on the format
+          formatMetatile: {
+              png: 2,
+              'grid.json': 1
+          },
+
+          // Buffer size is the tickness in pixel of a buffer
+          // around the rendered (meta?)tile.
+          //
+          // This is important for labels and other marker that overlap tile boundaries.
+          // Setting to 128 ensures no render artifacts.
+          // 64 may have artifacts but is faster.
+          // Less important if we can turn metatiling on.
+          bufferSize: 64,
+
+          // SQL queries will be wrapped with ST_SnapToGrid
+          // Snapping all points of the  geometry to a regular grid
+          snapToGrid: false,
+
+          // SQL queries will be wrapped with ST_ClipByBox2D
+          // Returning the portion of a geometry falling within a rectangle
+          // It will only work if snapToGrid is enabled
+          clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
+
+          limits: {
+              // Time in milliseconds a render request can take before it fails, some notes:
+              //  - 0 means no render limit
+              //  - it considers metatiling, naive implementation: (render timeout) * (number of tiles in metatile)
+              render: 0,
+              // As the render request will finish even if timed out, whether it should be placed in the internal
+              // cache or it should be fully discarded. When placed in the internal cache another attempt to retrieve
+              // the same tile will result in an immediate response, however that will use a lot of more application
+              // memory. If we want to enforce this behaviour we have to implement a cache eviction policy for the
+              // internal cache.
+              cacheOnTimeout: true
+          },
+
+          geojson: {
+                dbPoolParams: {
+                      // maximum number of resources to create at any given time
+                      size: 16,
+                      // max milliseconds a resource can go unused before it should be destroyed
+                      idleTimeout: 3000,
+                      // frequency to check for idle resources
+                      reapInterval: 1000
+                },
+
+              // SQL queries will be wrapped with ST_ClipByBox2D
+              // Returning the portion of a geometry falling within a rectangle
+              // It will only work if snapToGrid is enabled
+              clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
+          }
+
+      },
       http: {
           timeout: 2000, // the timeout in ms for a http tile request
           proxy: undefined, // the url for a proxy server
           whitelist: [ // the whitelist of urlTemplates that can be used
+              '.*', // will enable any URL
               'http://{s}.example.com/{z}/{x}/{y}.png'
-          ]
+          ],
+          // image to use as placeholder when urlTemplate is not in the whitelist
+          // if provided the http renderer will use it instead of throw an error
+          fallbackImage: {
+              type: 'fs', // 'fs' and 'url' supported
+              src: __dirname + '/../../assets/default-placeholder.png'
+          }
+      },
+      torque: {
+          dbPoolParams: {
+              // maximum number of resources to create at any given time
+              size: 16,
+              // max milliseconds a resource can go unused before it should be destroyed
+              idleTimeout: 3000,
+              // frequency to check for idle resources
+              reapInterval: 1000
+          }
       }
     }
+    // anything analyses related
+    ,analysis: {
+        // batch configuration
+        batch: {
+            // Inline execution avoid the use of SQL API as batch endpoint
+            // When set to true it will run all analysis queries in series, with a direct connection to the DB
+            // This might be useful for:
+            //  - testing
+            //  - running an standalone server without any dependency on external services
+            inlineExecution: false,
+            // where the SQL API is running, it will use a custom Host header to specify the username.
+            endpoint: 'http://127.0.0.1:8080/api/v2/sql/job',
+            // the template to use for adding the host header in the batch api requests
+            hostHeaderTemplate: '{{=it.username}}.localhost.lan'
+        }
+    }
     ,millstone: {
         // Needs to be writable by server user
         cache_basedir: '/home/ubuntu/tile_assets/'
@@ -119,27 +229,16 @@ var config = {
         },
         emitter: {
             statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd
-        }
+        },
+        unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161
+        noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading
     }
-    ,sqlapi: {
-        protocol: 'https',
-        // If "host" is given, it will be used
-        // to connect to the SQL-API without a
-        // DNS lookup
-        //host: '127.0.0.1',
-        port: 8080,
-        // The "domain" part will be appended to 
-        // the cartodb username and passed to
-        // SQL-API requests in the Host HTTP header
-        domain: 'cartodb.com',
-        version: 'v2',
-        // Maximum lenght of SQL query for GET
-        // requests. Longer queries will be sent
-        // using POST. Defaults to 2048
-        max_get_sql_length: 2048,
-        // Maximum time to wait for a response,
-        // in milliseconds. Defaults to 100.
-        timeout: 100
+    // For more details about this options check https://nodejs.org/api/http.html#http_new_agent_options
+    ,httpAgent: {
+        keepAlive: true,
+        keepAliveMsecs: 1000,
+        maxSockets: 25,
+        maxFreeSockets: 256
     }
     ,varnish: {
         host: 'localhost',
@@ -150,8 +249,17 @@ var config = {
         ttl: 86400,
         layergroupTtl: 86400 // the max-age for cache-control header in layergroup responses
     }
+    // this [OPTIONAL] configuration enables invalidating by surrogate key in fastly
+    ,fastly: {
+        // whether the invalidation is enabled or not
+        enabled: false,
+        // the fastly api key
+        apiKey: 'wadus_api_key',
+        // the service that will get surrogate key invalidation
+        serviceId: 'wadus_service_id'
+    }
     // If useProfiler is true every response will be served with an
-    // X-Tiler-Profile header containing elapsed timing for various 
+    // X-Tiler-Profile header containing elapsed timing for various
     // steps taken for producing the response.
     ,useProfiler:false
     ,serverMetadata: {
@@ -160,15 +268,6 @@ var config = {
         https: 'cartocdn.global.ssl.fastly.net'
       }
     }
-    // Optional rollbar support
-    ,rollbar: {
-      token: 'secret',
-      // See http://github.com/rollbar/node_rollbar#configuration-reference
-      options: {
-        endpoint: 'https://api.rollbar.com/api/1/',
-        handler: 'inline'
-      }
-    }
     // Settings for the health check available at /health
     ,health: {
       enabled: true,
@@ -177,6 +276,17 @@ var config = {
       x: 0,
       y: 0
     }
+    ,disabled_file: 'pids/disabled'
+
+    // Use this as a feature flags enabling/disabling mechanism
+    ,enabledFeatures: {
+        // whether it should intercept tile render errors an act based on them, enabled by default.
+        onTileErrorStrategy: true,
+        // whether the affected tables for a given SQL must query directly postgresql or use the SQL API
+        cdbQueryTablesFromPostgres: true,
+        // whether in mapconfig is available stats & metadata for each layer
+        layerMetadata: false
+    }
 };
 
 module.exports = config;

config/environments/staging.js.example

@@ -2,6 +2,9 @@ var config = {
     environment: 'production'
     ,port: 8181
     ,host: '127.0.0.1'
+    // Size of the threadpool which can be used to run user code and get notified in the loop thread
+    // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
+    // See http://docs.libuv.org/en/latest/threadpool.html
     ,uv_threadpool_size: undefined
     // Regular expression pattern to extract username
     // from hostname. Must have a single grabbing block.
@@ -10,17 +13,15 @@ var config = {
     // Base URLs for the APIs
     //
     // See http://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API
-    // 
+    //
     // Base url for the Templated Maps API
     // "/api/v1/maps/named" is the new API,
     // "/tiles/template" is for compatibility with versions up to 1.6.x
-    ,base_url_templated: '(?:/api/v1/maps/named|/tiles/template)'
+    ,base_url_templated: '(?:/api/v1/map/named|/user/:user/api/v1/map/named|/tiles/template)'
     // Base url for the Detached Maps API
     // "/api/v1/maps" is the the new API,
     // "/tiles/layergroup" is for compatibility with versions up to 1.6.x
-    ,base_url_detached: '(?:/api/v1/maps|/tiles/layergroup)'
-    // Base url for the Inline Maps and Table Maps API
-    ,base_url_legacy: '/tiles/:table'
+    ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)'
 
     // Maximum number of connections for one process
     // 128 is a good value with a limit of 1024 open file descriptors
@@ -33,7 +34,7 @@ var config = {
     // Defaults to 7200 (2 hours)
     ,mapConfigTTL: 7200
     // idle socket timeout, in milliseconds
-    ,socket_timeout: 600000 
+    ,socket_timeout: 600000
     ,enable_cors: true
     ,cache_enabled: true
     ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms (:res[X-Tiler-Profiler]) -> :res[Content-Type]'
@@ -80,17 +81,126 @@ var config = {
     ,renderer: {
       // Milliseconds since last access before renderer cache item expires
       cache_ttl: 60000,
-      metatile: 4,
-      bufferSize: 64,
       statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
+      mapnik: {
+          // The size of the pool of internal mapnik backend
+          // This pool size is per mapnik renderer created in Windshaft's RendererFactory
+          // See https://github.com/CartoDB/Windshaft/blob/master/lib/windshaft/renderers/renderer_factory.js
+          // Important: check the configuration of uv_threadpool_size to use suitable value
+          poolSize: 8,
+
+          // Metatile is the number of tiles-per-side that are going
+          // to be rendered at once. If all of them will be requested
+          // we'd have saved time. If only one will be used, we'd have
+          // wasted time.
+          metatile: 2,
+
+          // tilelive-mapnik uses an internal cache to store tiles/grids
+          // generated when using metatile. This options allow to tune
+          // the behaviour for that internal cache.
+          metatileCache: {
+              // Time an object must stay in the cache until is removed
+              ttl: 0,
+              // Whether an object must be removed after the first hit
+              // Usually you want to use `true` here when ttl>0.
+              deleteOnHit: false
+          },
+
+          // Override metatile behaviour depending on the format
+          formatMetatile: {
+              png: 2,
+              'grid.json': 1
+          },
+
+          // Buffer size is the tickness in pixel of a buffer
+          // around the rendered (meta?)tile.
+          //
+          // This is important for labels and other marker that overlap tile boundaries.
+          // Setting to 128 ensures no render artifacts.
+          // 64 may have artifacts but is faster.
+          // Less important if we can turn metatiling on.
+          bufferSize: 64,
+
+          // SQL queries will be wrapped with ST_SnapToGrid
+          // Snapping all points of the  geometry to a regular grid
+          snapToGrid: false,
+
+          // SQL queries will be wrapped with ST_ClipByBox2D
+          // Returning the portion of a geometry falling within a rectangle
+          // It will only work if snapToGrid is enabled
+          clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
+
+          limits: {
+              // Time in milliseconds a render request can take before it fails, some notes:
+              //  - 0 means no render limit
+              //  - it considers metatiling, naive implementation: (render timeout) * (number of tiles in metatile)
+              render: 0,
+              // As the render request will finish even if timed out, whether it should be placed in the internal
+              // cache or it should be fully discarded. When placed in the internal cache another attempt to retrieve
+              // the same tile will result in an immediate response, however that will use a lot of more application
+              // memory. If we want to enforce this behaviour we have to implement a cache eviction policy for the
+              // internal cache.
+              cacheOnTimeout: true
+          },
+
+          geojson: {
+                dbPoolParams: {
+                      // maximum number of resources to create at any given time
+                      size: 16,
+                      // max milliseconds a resource can go unused before it should be destroyed
+                      idleTimeout: 3000,
+                      // frequency to check for idle resources
+                      reapInterval: 1000
+                },
+
+              // SQL queries will be wrapped with ST_ClipByBox2D
+              // Returning the portion of a geometry falling within a rectangle
+              // It will only work if snapToGrid is enabled
+              clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
+          }
+
+      },
       http: {
           timeout: 2000, // the timeout in ms for a http tile request
           proxy: undefined, // the url for a proxy server
           whitelist: [ // the whitelist of urlTemplates that can be used
+              '.*', // will enable any URL
               'http://{s}.example.com/{z}/{x}/{y}.png'
-          ]
+          ],
+          // image to use as placeholder when urlTemplate is not in the whitelist
+          // if provided the http renderer will use it instead of throw an error
+          fallbackImage: {
+              type: 'fs', // 'fs' and 'url' supported
+              src: __dirname + '/../../assets/default-placeholder.png'
+          }
+      },
+      torque: {
+          dbPoolParams: {
+              // maximum number of resources to create at any given time
+              size: 16,
+              // max milliseconds a resource can go unused before it should be destroyed
+              idleTimeout: 3000,
+              // frequency to check for idle resources
+              reapInterval: 1000
+          }
       }
     }
+    // anything analyses related
+    ,analysis: {
+        // batch configuration
+        batch: {
+            // Inline execution avoid the use of SQL API as batch endpoint
+            // When set to true it will run all analysis queries in series, with a direct connection to the DB
+            // This might be useful for:
+            //  - testing
+            //  - running an standalone server without any dependency on external services
+            inlineExecution: false,
+            // where the SQL API is running, it will use a custom Host header to specify the username.
+            endpoint: 'http://127.0.0.1:8080/api/v2/sql/job',
+            // the template to use for adding the host header in the batch api requests
+            hostHeaderTemplate: '{{=it.username}}.localhost.lan'
+        }
+    }
     ,millstone: {
         // Needs to be writable by server user
         cache_basedir: '/home/ubuntu/tile_assets/'
@@ -119,27 +229,16 @@ var config = {
         },
         emitter: {
             statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd
-        }
+        },
+        unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161
+        noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading
     }
-    ,sqlapi: {
-        protocol: 'https',
-        // If "host" is given, it will be used
-        // to connect to the SQL-API without a
-        // DNS lookup
-        //host: '127.0.0.1',
-        port: 8080,
-        // The "domain" part will be appended to 
-        // the cartodb username and passed to
-        // SQL-API requests in the Host HTTP header
-        domain: 'cartodb.com',
-        version: 'v2',
-        // Maximum lenght of SQL query for GET
-        // requests. Longer queries will be sent
-        // using POST. Defaults to 2048
-        max_get_sql_length: 2048,
-        // Maximum time to wait for a response,
-        // in milliseconds. Defaults to 100.
-        timeout: 100
+    // For more details about this options check https://nodejs.org/api/http.html#http_new_agent_options
+    ,httpAgent: {
+        keepAlive: true,
+        keepAliveMsecs: 1000,
+        maxSockets: 25,
+        maxFreeSockets: 256
     }
     ,varnish: {
         host: 'localhost',
@@ -150,8 +249,17 @@ var config = {
         ttl: 86400,
         layergroupTtl: 86400 // the max-age for cache-control header in layergroup responses
     }
+    // this [OPTIONAL] configuration enables invalidating by surrogate key in fastly
+    ,fastly: {
+        // whether the invalidation is enabled or not
+        enabled: false,
+        // the fastly api key
+        apiKey: 'wadus_api_key',
+        // the service that will get surrogate key invalidation
+        serviceId: 'wadus_service_id'
+    }
     // If useProfiler is true every response will be served with an
-    // X-Tiler-Profile header containing elapsed timing for various 
+    // X-Tiler-Profile header containing elapsed timing for various
     // steps taken for producing the response.
     ,useProfiler:true
     ,serverMetadata: {
@@ -160,15 +268,6 @@ var config = {
         https: 'cartocdn.global.ssl.fastly.net'
       }
     }
-    // Optional rollbar support
-    ,rollbar: {
-      token: 'secret',
-      // See http://github.com/rollbar/node_rollbar#configuration-reference
-      options: {
-        endpoint: 'https://api.rollbar.com/api/1/',
-        handler: 'inline'
-      }
-    }
     // Settings for the health check available at /health
     ,health: {
       enabled: false,
@@ -177,6 +276,17 @@ var config = {
       x: 0,
       y: 0
     }
+    ,disabled_file: 'pids/disabled'
+
+    // Use this as a feature flags enabling/disabling mechanism
+    ,enabledFeatures: {
+        // whether it should intercept tile render errors an act based on them, enabled by default.
+        onTileErrorStrategy: true,
+        // whether the affected tables for a given SQL must query directly postgresql or use the SQL API
+        cdbQueryTablesFromPostgres: true,
+        // whether in mapconfig is available stats & metadata for each layer
+        layerMetadata: true
+    }
 };
 
 module.exports = config;

config/environments/test.js.example

@@ -2,6 +2,9 @@ var config = {
      environment: 'test'
     ,port: 8888
     ,host: '127.0.0.1'
+    // Size of the threadpool which can be used to run user code and get notified in the loop thread
+    // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
+    // See http://docs.libuv.org/en/latest/threadpool.html
     ,uv_threadpool_size: undefined
     // Regular expression pattern to extract username
     // from hostname. Must have a single grabbing block.
@@ -10,17 +13,15 @@ var config = {
     // Base URLs for the APIs
     //
     // See https://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API
-    // 
+    //
     // Base url for the Templated Maps API
     // "/api/v1/map/named" is the new API,
     // "/tiles/template" is for compatibility with versions up to 1.6.x
-    ,base_url_templated: '(?:/api/v1/map/named|/tiles/template)'
+    ,base_url_templated: '(?:/api/v1/map/named|/user/:user/api/v1/map/named|/tiles/template)'
     // Base url for the Detached Maps API
     // "maps" is the the new API,
     // "tiles/layergroup" is for compatibility with versions up to 1.6.x
-    ,base_url_detached: '(?:/api/v1/map|/tiles/layergroup)'
-    // Base url for the Inline Maps and Table Maps API
-    ,base_url_legacy: '/tiles/:table'
+    ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)'
 
     // Maximum number of connections for one process
     // 128 is a good value with a limit of 1024 open file descriptors
@@ -33,7 +34,7 @@ var config = {
     // Defaults to 7200 (2 hours)
     ,mapConfigTTL: 7200
     // idle socket timeout, in milliseconds
-    ,socket_timeout: 600000 
+    ,socket_timeout: 600000
     ,enable_cors: true
     ,cache_enabled: false
     ,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
@@ -50,7 +51,7 @@ var config = {
     ,postgres: {
         // Parameters to pass to datasource plugin of mapnik
         // See http://github.com/mapnik/mapnik/wiki/PostGIS
-        user: "testpublicuser",
+        user: "test_windshaft_publicuser",
         password: "public",
         host: '127.0.0.1',
         port: 5432,
@@ -80,17 +81,127 @@ var config = {
     ,renderer: {
       // Milliseconds since last access before renderer cache item expires
       cache_ttl: 60000,
-      metatile: 4,
-      bufferSize: 64,
       statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
+      mapnik: {
+          // The size of the pool of internal mapnik backend
+          // This pool size is per mapnik renderer created in Windshaft's RendererFactory
+          // See https://github.com/CartoDB/Windshaft/blob/master/lib/windshaft/renderers/renderer_factory.js
+          // Important: check the configuration of uv_threadpool_size to use suitable value
+          poolSize: 8,
+
+          // Metatile is the number of tiles-per-side that are going
+          // to be rendered at once. If all of them will be requested
+          // we'd have saved time. If only one will be used, we'd have
+          // wasted time.
+          metatile: 2,
+
+          // tilelive-mapnik uses an internal cache to store tiles/grids
+          // generated when using metatile. This options allow to tune
+          // the behaviour for that internal cache.
+          metatileCache: {
+              // Time an object must stay in the cache until is removed
+              ttl: 0,
+              // Whether an object must be removed after the first hit
+              // Usually you want to use `true` here when ttl>0.
+              deleteOnHit: false
+          },
+
+          // Override metatile behaviour depending on the format
+          formatMetatile: {
+              png: 2,
+              'grid.json': 1
+          },
+
+          // Buffer size is the tickness in pixel of a buffer
+          // around the rendered (meta?)tile.
+          //
+          // This is important for labels and other marker that overlap tile boundaries.
+          // Setting to 128 ensures no render artifacts.
+          // 64 may have artifacts but is faster.
+          // Less important if we can turn metatiling on.
+          bufferSize: 64,
+
+          // SQL queries will be wrapped with ST_SnapToGrid
+          // Snapping all points of the  geometry to a regular grid
+          snapToGrid: false,
+
+          // SQL queries will be wrapped with ST_ClipByBox2D
+          // Returning the portion of a geometry falling within a rectangle
+          // It will only work if snapToGrid is enabled
+          clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
+
+          limits: {
+              // Time in milliseconds a render request can take before it fails, some notes:
+              //  - 0 means no render limit
+              //  - it considers metatiling, naive implementation: (render timeout) * (number of tiles in metatile)
+              render: 0,
+              // As the render request will finish even if timed out, whether it should be placed in the internal
+              // cache or it should be fully discarded. When placed in the internal cache another attempt to retrieve
+              // the same tile will result in an immediate response, however that will use a lot of more application
+              // memory. If we want to enforce this behaviour we have to implement a cache eviction policy for the
+              // internal cache.
+              cacheOnTimeout: true
+          },
+
+          geojson: {
+                dbPoolParams: {
+                      // maximum number of resources to create at any given time
+                      size: 16,
+                      // max milliseconds a resource can go unused before it should be destroyed
+                      idleTimeout: 3000,
+                      // frequency to check for idle resources
+                      reapInterval: 1000
+                },
+
+              // SQL queries will be wrapped with ST_ClipByBox2D
+              // Returning the portion of a geometry falling within a rectangle
+              // It will only work if snapToGrid is enabled
+              clipByBox2d: false // this requires postgis >=2.2 and geos >=3.5
+          }
+      },
       http: {
           timeout: 2000, // the timeout in ms for a http tile request
           proxy: undefined, // the url for a proxy server
           whitelist: [ // the whitelist of urlTemplates that can be used
-              'http://{s}.example.com/{z}/{x}/{y}.png'
-          ]
+              '.*', // will enable any URL
+              'http://{s}.example.com/{z}/{x}/{y}.png',
+              // for testing purposes
+              'http://{s}.basemaps.cartocdn.com/dark_nolabels/{z}/{x}/{y}.png'
+          ],
+          // image to use as placeholder when urlTemplate is not in the whitelist
+          // if provided the http renderer will use it instead of throw an error
+          fallbackImage: {
+              type: 'fs', // 'fs' and 'url' supported
+              src: __dirname + '/../../assets/default-placeholder.png'
+          }
+      },
+      torque: {
+          dbPoolParams: {
+              // maximum number of resources to create at any given time
+              size: 16,
+              // max milliseconds a resource can go unused before it should be destroyed
+              idleTimeout: 3000,
+              // frequency to check for idle resources
+              reapInterval: 1000
+          }
       }
     }
+    // anything analyses related
+    ,analysis: {
+        // batch configuration
+        batch: {
+            // Inline execution avoid the use of SQL API as batch endpoint
+            // When set to true it will run all analysis queries in series, with a direct connection to the DB
+            // This might be useful for:
+            //  - testing
+            //  - running an standalone server without any dependency on external services
+            inlineExecution: true,
+            // where the SQL API is running, it will use a custom Host header to specify the username.
+            endpoint: 'http://127.0.0.1:8080/api/v2/sql/job',
+            // the template to use for adding the host header in the batch api requests
+            hostHeaderTemplate: '{{=it.username}}.localhost.lan'
+        }
+    }
     ,millstone: {
         // Needs to be writable by server user
         cache_basedir: '/tmp/cdb-tiler-test/millstone'
@@ -119,29 +230,16 @@ var config = {
         },
         emitter: {
             statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd
-        }
+        },
+        unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161
+        noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading
     }
-    ,sqlapi: {
-        protocol: 'http',
-        // If "host" is given, it will be used
-        // to connect to the SQL-API without a
-        // DNS lookup
-        host: '127.0.0.1',
-        port: 1080,
-        // The "domain" part will be appended to 
-        // the cartodb username and passed to
-        // SQL-API requests in the Host HTTP header
-        domain: 'donot_look_this_up',
-        // This port will be used by "make check" for testing purposes
-        // It must be available
-        version: 'v1',
-        // Maximum lenght of SQL query for GET
-        // requests. Longer queries will be sent
-        // using POST. Defaults to 2048
-        max_get_sql_length: 2048,
-        // Maximum time to wait for a response,
-        // in milliseconds. Defaults to 100.
-        timeout: 100
+    // For more details about this options check https://nodejs.org/api/http.html#http_new_agent_options
+    ,httpAgent: {
+        keepAlive: true,
+        keepAliveMsecs: 1000,
+        maxSockets: 25,
+        maxFreeSockets: 256
     }
     ,varnish: {
         host: '',
@@ -152,8 +250,17 @@ var config = {
         ttl: 86400,
         layergroupTtl: 86400 // the max-age for cache-control header in layergroup responses
     }
+    // this [OPTIONAL] configuration enables invalidating by surrogate key in fastly
+    ,fastly: {
+        // whether the invalidation is enabled or not
+        enabled: false,
+        // the fastly api key
+        apiKey: 'wadus_api_key',
+        // the service that will get surrogate key invalidation
+        serviceId: 'wadus_service_id'
+    }
     // If useProfiler is true every response will be served with an
-    // X-Tiler-Profile header containing elapsed timing for various 
+    // X-Tiler-Profile header containing elapsed timing for various
     // steps taken for producing the response.
     ,useProfiler:true
     // Settings for the health check available at /health
@@ -164,6 +271,17 @@ var config = {
       x: 0,
       y: 0
     }
+    ,disabled_file: 'pids/disabled'
+
+    // Use this as a feature flags enabling/disabling mechanism
+    ,enabledFeatures: {
+        // whether it should intercept tile render errors an act based on them, enabled by default.
+        onTileErrorStrategy: true,
+        // whether the affected tables for a given SQL must query directly postgresql or use the SQL API
+        cdbQueryTablesFromPostgres: true,
+        // whether in mapconfig is available stats & metadata for each layer
+        layerMetadata: true
+    }
 };
 
 module.exports = config;

configure

@@ -20,7 +20,6 @@
 ENVDIR=config/environments
 
 PGPORT=
-SQLAPI_PORT=
 MAPNIK_VERSION=
 ENVIRONMENT=development
 
@@ -32,7 +31,6 @@ usage() {
   echo "Configuration:"
   echo "  --help                        display this help and exit"
   echo "  --with-pgport=NUM             access PostgreSQL server on TCP port NUM [$PGPORT]"
-  echo "  --with-sqlapi-port=NUM        access SQL-API server on TCP port NUM [$SQLAPI_PORT]"
   echo "  --with-mapnik-version=STRING  set mapnik version string [$MAPNIK_VERSION]"
   echo "  --environment=STRING          set output environment name [$ENVIRONMENT]"
 }
@@ -46,9 +44,6 @@ while test -n "$1"; do
     --with-pgport=*)
       PGPORT=`echo "$1" | cut -d= -f2`
       ;;
-    --with-sqlapi-port=*)
-      SQLAPI_PORT=`echo "$1" | cut -d= -f2`
-      ;;
     --with-mapnik-version=*)
       MAPNIK_VERSION=`echo "$1" | cut -d= -f2`
       ;;
@@ -56,9 +51,8 @@ while test -n "$1"; do
       ENVIRONMENT=`echo "$1" | cut -d= -f2`
       ;;
     *)
-      echo "Unknown option '$1'" >&2
-      usage >&2
-      exit 1
+      echo "Unused option '$1'" >&2
+      ;;
   esac
   shift
 done
@@ -68,12 +62,8 @@ ENVEX=./${ENVDIR}/${ENVIRONMENT}.js.example
 if [ -z "$PGPORT" ]; then
   PGPORT=`node -e "console.log(require('${ENVEX}').postgres.port)"`
 fi
-if [ -z "$SQLAPI_PORT" ]; then
-  SQLAPI_PORT=`node -e "console.log(require('${ENVEX}').sqlapi.port)"`
-fi
 
 echo "PGPORT: $PGPORT"
-echo "SQLAPI_PORT: $SQLAPI_PORT"
 echo "MAPNIK_VERSION: $MAPNIK_VERSION"
 echo "ENVIRONMENT: $ENVIRONMENT"
 
@@ -83,7 +73,6 @@ echo "Writing $o"
 # See http://austinmatzko.com/2008/04/26/sed-multi-line-search-and-replace/
 sed -n "1h;1!H;\${;g;s/\(,postgres: {[^}]*port: *'\?\)[^',]*\('\?,\)/\1$PGPORT\2/;p;}" < "${ENVEX}" \
   | sed "s/mapnik_version:.*/mapnik_version: '$MAPNIK_VERSION'/" \
-  | sed -n "1h;1!H;\${;g;s/\(,sqlapi: {[^}]*port: *'\?\)[^',]*\('\?,\)/\1$SQLAPI_PORT\2/;p;}" \
   > "$o"
 
 STATUSFILE=config.status--${ENVIRONMENT}

docs/Map-API-internal.md

@@ -1,111 +0,0 @@
-# Kind of maps
-
-Windshaft-CartoDB supports the following types of maps:
-
- - [Temporary maps](#temporary-maps) (created by anyone)
-   - [Detached maps](#detached-maps)
-   - [Inline maps](#inline-maps) (legacy)
- - [Persistent maps](#peristent-maps) (created by CartDB user)
-   - [Template maps](#template-maps)
-   - [Table maps](#table-maps) (legacy, deprecated) 
-
-## Temporary maps
-
-Temporary maps have no owners and are anonymous in nature.
-There are two kinds of temporary maps:
-
- - Detached maps (aka MultiLayer-API)
- - Inline maps 
-
-### Detached maps
-
-Detached maps are maps that are configured with a request
-obtaining a temporary token and then used by referencing
-the obtained token. The token expires automatically when unused.
-
-Anyone can create detached maps, but users will need read access
-to the data source of the map layers.
-
-The configuration format is a [MapConfig]
-(http://github.com/CartoDB/Windshaft/wiki/MapConfig-specification) document.
-
-The HTTP endpoints for creating the map and using it are described [here]
-(http://github.com/CartoDB/Windshaft-cartodb/wiki/MultiLayer-API)
-
-*TODO* cleanup the referenced document
-
-### Inline maps
-
-Inline maps are maps that only exist for a single request,
-being the request for a specific map resource (tile).
-
-Inline maps are always bound to a table, and can only be
-obtained by those having read access to the that table.
-Additionally, users need to have access to any datasource
-specified as part of the configuration.
-
-Inline maps only support PNG and UTF8GRID tiles.
-
-The configuration consist in a set of parameters, to be
-specified in the query string of the tile request:
-
- * sql - the query to run as datasource, can be an array
- * style - the CartoCSS style for the datasource, can be an array
- * style_version - version of the CartoCSS style, can be an array
- * interactivity - only for fetching UTF8GRID, 
-
-If the style is not provided, style of the associated table is
-used; if the sql is not provided, all records of the associated
-table are used as the datasource; the two possibilities result
-in a mix between _inline_ maps and [Table maps][].
-
-*TODO* specify (or link) api endpoints
-
-## Persistent maps
-
-Persistent maps can only be created by a CartoDB user who has full
-responsibility over editing and deleting them. There are two
-kind of persistent maps:
-
- - Template maps 
- - Table maps (legacy, deprecated) 
-
-### Templated maps
-
-Templated maps are templated [MapConfig]
-(http://github.com/CartoDB/Windshaft/wiki/MapConfig-specification) documents
-associated with an authorization certificate.
-
-The authorization certificate determines who can instanciate the
-template and use the resulting map. Authorized users of the instanciated
-maps will have the same database access privilege of the template owner.
-
-The HTTP endpoints for creating and using templated maps are described [here]
-(http://github.com/CartoDB/Windshaft-cartodb/wiki/Template-maps).
-
-*TODO* cleanup the referenced document
-
-### Table maps
-
-Table maps are maps associated with a table.
-Configuration of such maps is limited to the CartoCSS style.
-
- * style - the CartoCSS style for the datasource, can be an array
- * style_version - version of the CartoCSS style, can be an array
-
-You can only fetch PNG or UTF8GRID tiles from these maps.
-
-Access method is the same as the one for [Inline maps](#inline-maps)
-
-# Endpoints description 
-
-- **/api/maps/** (same interface than https://github.com/CartoDB/Windshaft/wiki/Multilayer-API)
-- **/api/maps/named** (same interface than https://github.com/CartoDB/Windshaft-cartodb/wiki/Template-maps)
-
-
-NOTE: in case Multilayer-API does not contain this info yet, the
-      endpoint for fetching attributes is this:
-
-- **/api/maps/:map_id/:layer_index/attributes/:feature_id** 
-   - would return { c: 1, d: 2 }
-

docs/Map-API.md

@@ -1,674 +1,19 @@
-## Maps API
+# Maps API
 
-The CartoDB Maps API allows you to generate maps based on data hosted in your CartoDB account, and you can apply custom SQL and CartoCSS to the data. The API generates a XYZ-based URL to fetch Web Mercator projected tiles using web clients such as [Leaflet](http://leafletjs.com), [Google Maps](https://developers.google.com/maps/), or [OpenLayers](http://openlayers.org/).
+The CartoDB Maps API allows you to generate maps based on data hosted in your CartoDB account and apply custom SQL and CartoCSS to the data. The API generates a XYZ-based URL to fetch Web Mercator projected tiles, using web clients such as [Leaflet](http://leafletjs.com), [Google Maps](https://developers.google.com/maps/), or [OpenLayers](http://openlayers.org/).
 
 You can create two types of maps with the Maps API:
 
-- **Anonymous maps**  
-  Maps that can be created using your CartoDB public data. Any client can change the read-only SQL and CartoCSS parameters that generate the map tiles. These maps can be created from a JavaScript application alone and no authenticated calls are needed. See [this CartoDB.js example]({{ '/cartodb-platform/cartodb-js.html' | prepend: site.baseurl }}).
+- **Anonymous Maps**  
+  You can create maps using your CartoDB public data. Any client can change the read-only SQL and CartoCSS parameters that generate the map tiles. These maps can be created from a JavaScript application alone and no authenticated calls are needed. See [this CartoDB.js example](/cartodb-platform/cartodb-js/getting-started/).
 
-- **Named maps**  
-  Maps that have access to your private data. These maps require an owner to setup and modify any SQL and CartoCSS parameters and are not modifiable without new setup calls. 
+- **Named Maps**  
+  There are also maps that have access to your private data. These maps require an owner to setup and modify any SQL and CartoCSS parameters and are not modifiable without new setup calls.
 
-## Quickstart
+## Documentation
 
-### Anonymous maps
-
-Here is an example of how to create an anonymous map with JavaScript:
-
-```javascript
-var mapconfig = {
-  "version": "1.0.1",
-  "layers": [{
-    "type": "cartodb",
-    "options": {
-      "cartocss_version": "2.1.1",
-      "cartocss": "#layer { polygon-fill: #FFF; }",
-      "sql": "select * from european_countries_e"
-    }
-  }]
-}
-
-$.ajax({
-  crossOrigin: true,
-  type: 'POST',
-  dataType: 'json',
-  contentType: 'application/json',
-  url: 'http://documentation.cartodb.com/api/v1/map',
-  data: JSON.stringify(mapconfig),
-  success: function(data) {
-    var templateUrl = 'http://documentation.cartodb.com/api/v1/map/' + data.layergroupid + '/{z}/{x}/{y}.png'
-    console.log(templateUrl);
-  }
-})
-```
-
-### Named maps
-
-Let's create a named map using some private tables in a CartoDB account.
-The following map config sets up a map of European countries that have a white fill color:
-
-```javascript
-{
-  "version": "0.0.1",
-  "name": "test",
-  "auth": {
-    "method": "open"
-  },
-  "layergroup": {
-    "layers": [{
-      "type": "cartodb",
-      "options": {
-        "cartocss_version": "2.1.1",
-        "cartocss": "#layer { polygon-fill: #FFF; }",
-        "sql": "select * from european_countries_e"
-      }
-    }]
-  }
-}
-```
-
-The map config needs to be sent to CartoDB's Map API using an authenticated call. Here we will use a command line tool called `curl`. For more info about this tool, see [this blog post](http://quickleft.com/blog/command-line-tutorials-curl), or type ``man curl`` in bash. Using `curl`, and storing the config from above in a file `mapconfig.json`, the call would look like:
-
-<div class="code-title notitle code-request"></div>
-```bash
-curl 'https://{account}.cartodb.com/api/v1/map/named?api_key=APIKEY' -H 'Content-Type: application/json' -d @mapconfig.json
-```
-
-To get the `URL` to fetch the tiles you need to instantiate the map, where `template_id` is the template name from the previous response.
-
-<div class="code-title notitle code-request"></div>
-```bash
-curl -X POST 'http://{account}.cartodb.com/api/v1/map/named/:template_id' -H 'Content-Type: application/json'
-```
-
-The response will return JSON with properties for the `layergroupid`, `cdn_url`, and the timestamp (`last_updated`) of the last data modification. 
-
-Here is an example response:
-
-```javascript
-{
-  "cdn_url": {
-    "http": "ashbu.cartocdn.com",
-    "https": "cartocdn-ashbu.global.ssl.fastly.net"
-  },
-  "layergroupid": "c01a54877c62831bb51720263f91fb33:0",
-  "last_updated": "1970-01-01T00:00:00.000Z"
-}
-```
-
-You can use the `layergroupid` to instantiate a URL template for accessing tiles on the client. Here we use the `layergroupid` from the example response above in this URL template:
-
-```bash
-http://documentation.cartodb.com/api/v1/map/c01a54877c62831bb51720263f91fb33:0/{z}/{x}/{y}.png
-```
-
-## General Concepts
-
-The following concepts are the same for every endpoint in the API except when it's noted explicitly.
-
-### Auth
-
-By default, users do not have access to private tables in CartoDB. In order to instantiate a map from private table data, an API Key is required. Additionally, to include some endpoints, an API Key must be included (e.g. creating a named map).
-
-To execute an authorized request, api_key=YOURAPIKEY should be added to the request URL. The param can be also passed as POST param. We **strongly advise** using HTTPS when you are performing requests that include your `api_key`.
-
-### Errors
-
-Errors are reported using standard HTTP codes and extended information encoded in JSON with this format:
-
-```javascript
-{
-  "errors": [
-    "access forbidden to table TABLE"
-  ]
-}
-```
-
-If you use JSONP, the 200 HTTP code is always returned so the JavaScript client can receive errors from the JSON object.
-
-### CORS support
-
-All the endpoints which might be accessed using a web browser add CORS headers and allow OPTIONS method.
-
-## Anonymous Maps
-
-Anonymous maps allows you to instantiate a map given SQL and CartoCSS. It also allows you to add interaction capabilities using [UTF Grid.](https://github.com/mapbox/utfgrid-spec)
-
-### Instantiate
-
-#### Definition
-
-<div class="code-title notitle code-request"></div>
-```html
-POST /api/v1/map
-```
-
-#### Params
-
-```javascript
-{
-  "version": "1.0.1",
-  "layers": [{
-    "type": "cartodb",
-    "options": {
-      "cartocss_version": "2.1.1", 
-      "cartocss": "#layer { polygon-fill: #FFF; }",
-      "sql": "select * from european_countries_e",
-      "interactivity": ["cartodb_id", "iso3"]
-    }
-  }]
-}
-```
-
-Should be a [Mapconfig](https://github.com/CartoDB/Windshaft/blob/0.19.1/doc/MapConfig-1.1.0.md).
-
-#### Response
-
-The response includes:
-
-- **layergroupid**  
-  The ID for that map, used to compose the URL for the tiles. The final URL is:
-
-  ```html
-  http://{account}.cartodb.com/api/v1/map/:layergroupid/{z}/{x}/{y}.png
-  ```
-
-- **updated_at**  
-  The ISO date of the last time the data involved in the query was updated.
-
-- **metadata** *(optional)*  
-  Includes information about the layers. Some layers may not have metadata.
-
-- **cdn_url**  
-  URLs to fetch the data using the best CDN for your zone.
-
-#### Example
-
-<div class="code-title code-request with-result">REQUEST</div>
-```bash
-curl 'http://documentation.cartodb.com/api/v1/map' -H 'Content-Type: application/json' -d @mapconfig.json
-```
-
-<div class="code-title">RESPONSE</div>
-```javascript
-{
-  "layergroupid":"c01a54877c62831bb51720263f91fb33:0",
-  "last_updated":"1970-01-01T00:00:00.000Z"
-  "cdn_url": {
-    "http": "http://cdb.com",
-    "https": "https://cdb.com"
-  }
-}
-```
-
-The tiles can be accessed using:
-
-```bash
-http://documentation.cartodb.com/api/v1/map/c01a54877c62831bb51720263f91fb33:0/{z}/{x}/{y}.png
-```
-
-For UTF grid tiles:
-
-```bash
-http://documentation.cartodb.com/api/v1/map/c01a54877c62831bb51720263f91fb33:0/:layer/{z}/{x}/{y}.grid.json
-```
-
-For attributes defined in `attributes` section:
-
-```bash
-http://documentation.cartodb.com/api/v1/map/c01a54877c62831bb51720263f91fb33:0/:layer/attributes/:feature_id
-```
-
-Which returns JSON with the attributes defined, like:
-
-```javascript
-{ c: 1, d: 2 }
-```
-
-Notice UTF Grid and attributes endpoints need an integer parameter, ``layer``. That number is the 0-based index of the layer inside the mapconfig. So in this case 0 returns the UTF grid tiles/attributes for layer 0, the only layer in the example mapconfig. If a second layer was available it could be returned with 1, a third layer with 2, etc.
-
-### Create JSONP
-
-The JSONP endpoint is provided in order to allow web browsers access which don't support CORS.
-
-#### Definition
-
-<div class="code-title notitle code-request"></div>
-```bash
-GET /api/v1/map?callback=method
-```
-
-#### Params
-
-- **config**
-  Encoded JSON with the params for creating named maps (the variables defined in the template).
-
-- **lmza**  
-  This attribute contains the same as config but LZMA compressed. It cannot be used at the same time as `config`.
-
-- **callback**  
-  JSON callback name.
-
-#### Example
-
-<div class="code-title code-request with-result">REQUEST</div>
-```bash
-curl "https://documentation.cartodb.com/api/v1/map?callback=callback&config=%7B%22version%22%3A%221.0.1%22%2C%22layers%22%3A%5B%7B%22type%22%3A%22cartodb%22%2C%22options%22%3A%7B%22sql%22%3A%22select+%2A+from+european_countries_e%22%2C%22cartocss%22%3A%22%23european_countries_e%7B+polygon-fill%3A+%23FF6600%3B+%7D%22%2C%22cartocss_version%22%3A%222.3.0%22%2C%22interactivity%22%3A%5B%22cartodb_id%22%5D%7D%7D%5D%7D"
-```
-
-<div class="code-title">RESPONSE</div>
-```javascript
-callback({
-    layergroupid: "d9034c133262dfb90285cea26c5c7ad7:0",
-    cdn_url: {
-        "http": "http://cdb.com",
-        "https": "https://cdb.com"
-    },
-    last_updated: "1970-01-01T00:00:00.000Z"
-})
-```
-
-### Remove
-
-Anonymous maps cannot be removed by an API call. They will expire after about five minutes but sometimes longer. If an anonymous map expires and tiles are requested from it, an error will be raised. This could happen if a user leaves a map open and after time returns to the map an attempts to interact with it in a way that requires new tiles (e.g. zoom). The client will need to go through the steps of creating the map again to fix the problem.
-
-
-## Named Maps
-
-Named maps are essentially the same as anonymous maps except the mapconfig is stored on the server and the map is given a unique name. Two other big differences are that you can create named maps from private data, and that users without an API Key can see them even though they are from that private data.
-
-The main two differences compared to anonymous maps are:
-
-- **auth layer**  
-  This allows you to control who is able to see the map based on a token auth
-
-- **templates**  
-  Since the mapconfig is static it can contain some variables so the client can modify the map's appearance using those variables
-
-Template maps are persistent with no preset expiration. They can only be created or deleted by a CartoDB user with a valid API_KEY (see [auth section](http://docs.cartodb.com/cartodb-platform/maps-api.html#auth)).
-
-### Create
-
-#### Definition
-
-<div class="code-title notitle code-request"></div>
-```html
-POST /api/v1/map/named
-```
-
-#### Params
-
-- **api_key** is required
-
-<div class="code-title">template.json</div>
-```javascript
-{
-  "version": "0.0.1",
-  "name": "template_name",
-  "auth": {
-    "method": "token",
-    "valid_tokens": [
-      "auth_token1",
-      "auth_token2"
-    ]
-  },
-  "placeholders": {
-    "color": {
-      "type": "css_color",
-      "default": "red"
-    },
-    "cartodb_id": {
-      "type": "number",
-      "default": 1
-    }
-  },
-  "layergroup": {
-    "version": "1.0.1",
-    "layers": [
-      {
-        "type": "cartodb",
-        "options": {
-          "cartocss_version": "2.1.1",
-          "cartocss": "#layer { polygon-fill: <%= color %>; }",
-          "sql": "select * from european_countries_e WHERE cartodb_id = <%= cartodb_id %>"
-        }
-      }
-    ]
-  }
-}
-```
-
-##### Arguments
-
-- **name**: There can be at most _one_ template with the same name for any user. Valid names start with a letter, and only contain letters, numbers, or underscores (_).
-- **auth**:
-  - **method** `"token"` or `"open"` (the default if no `"method"` is given).
-  - **valid_tokens** when `"method"` is set to `"token"`, the values listed here allow you to instantiate the named map.
-- **placeholders**: Variables not listed here are not substituted. Variable not provided at instantiation time trigger an error. A default is required for optional variables. Type specification is used for quoting, to avoid injections see template format section below.
-- **layergroup**: the layer list definition. This is the MapConfig explained in anonymous maps. See [MapConfig documentation](https://github.com/CartoDB/Windshaft/blob/master/doc/MapConfig-1.1.0.md) for more info.
-
-#### Template Format
-
-A templated `layergroup` allows the use of placeholders in the "cartocss" and "sql" elements of the "option" object in any "layer" of a `layergroup` configuration
-
-Valid placeholder names start with a letter and can only contain letters, numbers, or underscores. They have to be written between the `<%=` and `%>` strings in order to be replaced.
-
-##### Example
-
-```javascript
-<%= my_color %>
-```
-
-The set of supported placeholders for a template will need to be explicitly defined with a specific type and default value for each.
-
-#### Placeholder Types
-
-The placeholder type will determine the kind of escaping for the associated value. Supported types are:
-
-- **sql_literal** internal single-quotes will be sql-escaped
-- **sql_ident** internal double-quotes will be sql-escaped
-- **number** can only contain numerical representation
-- **css_color** can only contain color names or hex-values
-
-Placeholder default values will be used whenever new values are not provided as options at the time of creation on the client. They can also be used to test the template by creating a default version with now options provided.
-
-When using templates, be very careful about your selections as they can give broad access to your data if they are defined losely.
-
-<div class="code-title code-request with-result">REQUEST</div>
-```html
-curl -X POST \
-   -H 'Content-Type: application/json' \
-   -d @template.json \
-   'https://documentation.cartodb.com/api/v1/map/named?api_key=APIKEY'
-```
-
-<div class="code-title">RESPONSE</div>
-```javascript
-{
-  "template_id":"name",
-}
-```
-
-### Instantiate
-
-Instantiating a map allows you to get the information needed to fetch tiles. That temporal map is an anonymous map.
-
-#### Definition
-
-<div class="code-title notitle code-request"></div>
-```html
-POST /api/v1/map/named/:template_name
-```
-
-#### Param
-
-- **auth_token** optional, but required when `"method"` is set to `"token"`
-
-```javascript
-// params.json
-{
- "color": "#ff0000",
- "cartodb_id": 3
-}
-```
-
-The fields you pass as `params.json` depend on the variables allowed by the named map. If there are variables missing it will raise an error (HTTP 400).
-
-- **auth_token** *optional* if the named map needs auth
-
-#### Example
-
-You can initialize a template map by passing all of the required parameters in a POST to `/api/v1/map/named/:template_name`.
-
-Valid credentials will be needed if required by the template.
-
-<div class="code-title code-request with-result">REQUEST</div>
-```bash
-curl -X POST \
-  -H 'Content-Type: application/json' \
-  -d @params.json \
-  'https://documentation.cartodb.com/api/v1/map/named/@template_name?auth_token=AUTH_TOKEN'
-```
-
-<div class="code-title">Response</div>
-```javascript
-{
-  "layergroupid": "docs@fd2861af@c01a54877c62831bb51720263f91fb33:123456788",
-  "last_updated": "2013-11-14T11:20:15.000Z"
-}
-```
-
-<div class="code-title">Error</div>
-```javascript
-{
-  "error": "Some error string here"
-}
-```
-
-You can then use the `layergroupid` for fetching tiles and grids as you would normally (see anonymous map section).  However, you'll need to show the `auth_token`, if required by the template.
-
-### Using JSONP
-
-There is also a special endpoint to be able to initialize a map using JSONP (for old browsers).
-
-#### Definition
-
-<div class="code-title notitle code-request"></div>
-```bash
-GET /api/v1/map/named/:template_name/jsonp
-```
-
-#### Params
-
-- **auth_token** optional, but required when `"method"` is set to `"token"`
-- **config** Encoded JSON with the params for creating named maps (the variables defined in the template)
-- **lmza** This attribute contains the same as config but LZMA compressed. It cannot be used at the same time than `config`.
-- **callback:** JSON callback name
-
-<div class="code-title code-request with-result">REQUEST</div>
-```bash
-curl 'https://documentation.cartodb.com/api/v1/map/named/:template_name/jsonp?auth_token=AUTH_TOKEN&callback=callback&config=template_params_json'
-```
-
-<div class="code-title">RESPONSE</div>
-```javascript
-callback({
-  "layergroupid":"c01a54877c62831bb51720263f91fb33:0",
-  "last_updated":"1970-01-01T00:00:00.000Z"
-  "cdn_url": {
-    "http": "http://cdb.com",
-    "https": "https://cdb.com"
-  }
-})
-```
-
-This takes the `callback` function (required), `auth_token` if the template needs auth, and `config` which is the variable for the template (in cases where it has variables). 
-
-```javascript
-url += "config=" + encodeURIComponent(
-JSON.stringify({ color: 'red' });
-```
-
-The response is in this format:
-
-```javascript
-callback({
-  layergroupid: "dev@744bd0ed9b047f953fae673d56a47b4d:1390844463021.1401",
-  last_updated: "2014-01-27T17:41:03.021Z"
-})
-```
-
-### Update
-
-#### Definition
-
-<div class="code-title notitle code-request"></div>
-```bash
-PUT /api/v1/map/named/:template_name
-```
-
-#### Params
-
-- **api_key** is required
-
-#### Response
-
-Same as updating a map.
-
-#### Other Info
-
-Updating a named map removes all the named map instances so they need to be initialized again.
-
-#### Example
-
-<div class="code-title code-request with-result">REQUEST</div>
-```bash
-curl -X PUT \
-  -H 'Content-Type: application/json' \
-  -d @template.json \
-  'https://documentation.cartodb.com/api/v1/map/named/:template_name?api_key=APIKEY'
-```
-
-<div class="code-title">RESPONSE</div>
-```javascript
-{
-  "template_id": "@template_name"
-}
-```
-
-If any template has the same name, it will be updated.
-
-If a template with the same name does NOT exist, a 400 HTTP response is generated with an error in this format:
-
-```javascript
-{
-  "error": "error string here"
-}
-```
-
-### Delete 
-
-Delete the specified template map from the server and disables any previously initialized versions of the map.
-
-#### Definition
-
-<div class="code-title notitle code-request"></div>
-```bash
-DELETE /api/v1/map/named/:template_name
-```
-
-#### Params
-
-- **api_key** is required
-
-#### Example
-
-<div class="code-title code-request">REQUEST</div>
-```bash
-curl -X DELETE 'https://documentation.cartodb.com/api/v1/map/named/:template_name?api_key=APIKEY'
-```
-
-<div class="code-title">RESPONSE</div>
-```javascript
-{
-  "error": "Some error string here"
-}
-```
-
-On success, a 204 (No Content) response would be issued. Otherwise a 4xx response with with an error will be returned.
-
-### Listing Available Templates
-
-This allows you to get a list of all available templates. 
-
-#### Definition
-
-<div class="code-title notitle code-request"></div>
-```bash
-GET /api/v1/map/named/
-```
-
-#### Params
-
-- **api_key** is required
-
-#### Example
-
-<div class="code-title code-request with-result">REQUEST</div>
-```bash
-curl -X GET 'https://documentation.cartodb.com/api/v1/map/named?api_key=APIKEY'
-```
-
-<div class="code-title with-result">RESPONSE</div>
-```javascript
-{
-   "template_ids": ["@template_name1","@template_name2"]
-}
-```
-
-<div class="code-title">ERROR</div>
-```javascript
-{
-   "error": "Some error string here"
-}
-```
-
-### Getting a Specific Template
-
-This gets the definition of a template.
-
-#### Definition
-
-<div class="code-title notitle code-request"></div>
-```bash
-GET /api/v1/map/named/:template_name
-```
-
-#### Params
-
-- **api_key** is required
-
-#### Example
-
-<div class="code-title code-request with-result">REQUEST</div>
-```bash
-curl -X GET 'https://documentation.cartodb.com/api/v1/map/named/:template_name?api_key=APIKEY'
-```
-
-<div class="code-title with-result">RESPONSE</div>
-```javascript
-{
-  "template": {...} // see template.json above
-}
-```
-
-<div class="code-title">ERROR</div>
-```javascript
-{
-  "error": "Some error string here"
-}
-```
-
-### Use with CartoDB.js
-Named maps can be used with CartoDB.js by specifying a named map in a layer source as follows. Named maps are treated almost the same as other layer source types in most other ways.
-
-```js
-var layerSource = {
-  user_name: '{your_user_name}', 
-  type: 'namedmap', 
-  named_map: { 
-    name: '{template_name}', 
-	layers: [{ 
-	  layer_name: "layer1", 
-      interactivity: "column1, column2, ..." 
-	}] 
-  } 
-}
-
-cartodb.createLayer('map_dom_id',layerSource)
-  .addTo(map_object);
-
-```
-
-See [CartoDB.js](http://docs.cartodb.com/cartodb-platform/cartodb-js.html) methods [layer.setParams()](http://docs.cartodb.com/cartodb-platform/cartodb-js.html#layersetparamskey-value) and [layer.setAuthToken()](http://docs.cartodb.com/cartodb-platform/cartodb-js.html#layersetauthtokenauthtoken).
+* [Quickstart](quickstart.md)
+* [General Concepts](general_concepts.md)
+* [Anonymous Maps](anonymous_maps.md)
+* [Named Maps](named_maps.md)
+* [Static Maps API](static_maps_api.md)

docs/MapConfig-NamedMaps-extension.md

@@ -0,0 +1,56 @@
+# 1. Purpose
+
+This specification describes an extension for
+[MapConfig 1.3.0](https://github.com/CartoDB/Windshaft/blob/master/doc/MapConfig-1.3.0.md) version.
+
+
+# 2. Changes over specification
+
+This extension introduces a new layer type so it's possible to use a Named Map by its name as a layer.
+
+## 2.1 Named layers definition
+
+```javascript
+{
+    // REQUIRED
+    // string, `named` is the only supported value
+    type: "named",
+
+    // REQUIRED
+    // object, set `named` map layers configuration
+    options: {
+
+        // REQUIRED
+        // string, the name for the Named Map to use
+        name: "world_borders",
+
+        // OPTIONAL
+        // object, the replacement values for the Named Map's template placeholders
+        // See https://github.com/CartoDB/Windshaft-cartodb/blob/master/docs/Map-API.md#instantiate-1 for more details
+        config: {
+            "color": "#000"
+        },
+
+        // OPTIONAL
+        // string array, the authorized tokens in case the Named Map has auth method set to `token`
+        // See https://github.com/CartoDB/Windshaft-cartodb/blob/master/docs/Map-API.md#named-maps-1 for more details
+        auth_tokens: [
+            "token1",
+            "token2"
+        ]
+    }
+}
+```
+
+## 2.2 Limitations
+
+1. A Named Map will not allow to have `named` type layers inside their templates layergroup's layers definition.
+2. A `named` layer does not allow Named Maps form other accounts, it's only possible to use Named Maps from the very
+same user account.
+
+
+# History
+
+## 1.0.0
+
+ - Initial version

docs/MultiLayer-API.md

@@ -1,4 +1,4 @@
-The Windshaft-CartoDB MultiLayer API extends the [Windshaft MultiLayer API](https://github.com/Vizzuality/Windshaft/wiki/Multilayer-API) in a few ways.
+The Windshaft-CartoDB MultiLayer API extends the [Windshaft MultiLayer API](https://github.com/CartoDB/Windshaft/blob/master/doc/Multilayer-API.md) in a few ways.
 
 ## Last modification timestamp embedded in the token
 
@@ -17,8 +17,8 @@ Windshaft-CartoDB adds the following attributes in the response object
 
    ```json
    {
-     http: 'http://cdn_url.com/'
-     https: 'https://secure.cdn_url.com/'
+     "http": "http://cdn_url.com/",
+     "https": "https://secure.cdn_url.com/"
    }
    ```
 

docs/Routes.md

@@ -0,0 +1,114 @@
+This document list all routes available in Windshaft-cartodb Maps API server.
+
+## Routes list
+
+1. `GET (?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)/:token/:z/:x/:y@:scale_factor?x.:format {:user(f),:token(f),:z(f),:x(f),:y(f),:scale_factor(t),:format(f)} (1)`
+<br/>Notes: Mapnik retina tiles [0]
+
+1. `GET (?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)/:token/:z/:x/:y.:format {:user(f),:token(f),:z(f),:x(f),:y(f),:format(f)} (1)`
+<br/>Notes: Mapnik tiles [0]
+
+1. `GET (?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)/:token/:layer/:z/:x/:y.(:format) {:user(f),:token(f),:layer(f),:z(f),:x(f),:y(f),:format(f)} (1)`
+<br/>Notes: Per :layer rendering based on :format [0]
+
+1. `GET (?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)/:token/:layer/attributes/:fid {:user(f),:token(f),:layer(f),:fid(f)} (1)`
+<br/>Notes: Endpoint for info windows data, alternative for sql api when tables are private [0]
+
+1. `GET (?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)/static/center/:token/:z/:lat/:lng/:width/:height.:format {:user(f),:token(f),:z(f),:lat(f),:lng(f),:width(f),:height(f),:format(f)} (1)`
+<br/>Notes: Static Maps API [0]
+
+1. `GET (?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format {:user(f),:token(f),:west(f),:south(f),:east(f),:north(f),:width(f),:height(f),:format(f)} (1)`
+<br/>Notes: Static Maps API [0]
+
+1. `GET (?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)/:token/:layer/widget/:widgetName {:user(f),:token(f),:layer(f),:widgetName(f)} (1)`
+<br/>Notes: By :widgetName per :layer widget [0]
+
+1. `GET (?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)/:token/:layer/widget/:widgetName/search {:user(f),:token(f),:layer(f),:widgetName(f)} (1)`
+<br/>Notes: By :widgetName per :layer widget search [0]
+
+1. `GET (?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup) {:user(f)} (1)`
+<br/>Notes: Map instantiation [0]
+
+1. `POST (?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup) {:user(f)} (1)`
+<br/>Notes: Map instantiation [0]
+
+1. `GET (?:/api/v1/map/named|/user/:user/api/v1/map/named|/tiles/template)/:template_id/jsonp {:user(f),:template_id(f)} (1)`
+<br/>Notes: Named maps JSONP instantiation [1]
+
+1. `POST (?:/api/v1/map/named|/user/:user/api/v1/map/named|/tiles/template)/:template_id {:user(f),:template_id(f)} (1)`
+<br/>Notes: Instantiate named map [1]
+
+1. `OPTIONS (?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup) {:user(f)} (1)`
+<br/>Notes: CORS [0]
+
+1. `GET (?:/api/v1/map/named|/user/:user/api/v1/map/named|/tiles/template)/:template_id/:layer/:z/:x/:y.(:format) {:user(f),:template_id(f),:layer(f),:z(f),:x(f),:y(f),:0(f),:format(f)} (1)`
+<br/>Notes: Per :layer fixed URL named map tiles [1]
+
+1. `GET (?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)/static/named/:template_id/:width/:height.:format {:user(f),:template_id(f),:width(f),:height(f),:format(f)} (1)`
+<br/>Notes: Static map for named maps [1]
+
+1. `POST (?:/api/v1/map/named|/user/:user/api/v1/map/named|/tiles/template) {:user(f)} (1)`
+<br/>Notes: Create named map (w/ API KEY) [1]
+
+1. `PUT (?:/api/v1/map/named|/user/:user/api/v1/map/named|/tiles/template)/:template_id {:user(f),:template_id(f)} (1)`
+<br/>Notes: Update a named map (w/ API KEY) [1]
+
+1. `GET (?:/api/v1/map/named|/user/:user/api/v1/map/named|/tiles/template)/:template_id {:user(f),:template_id(f)} (1)`
+<br/>Notes: Named map retrieval (w/ API KEY) [1]
+
+1. `DELETE (?:/api/v1/map/named|/user/:user/api/v1/map/named|/tiles/template)/:template_id {:user(f),:template_id(f)} (1)`
+<br/>Notes: Delete named map (w/ API KEY) [1]
+
+1. `GET (?:/api/v1/map/named|/user/:user/api/v1/map/named|/tiles/template) {:user(f)} (1)`
+<br/>Notes: List named maps (w/ API KEY) [1]
+
+1. `OPTIONS (?:/api/v1/map/named|/user/:user/api/v1/map/named|/tiles/template)/:template_id {:user(f),:template_id(f)} (1)`
+<br/>Notes: CORS [1]
+
+1. `GET /health {} (1)`
+<br/>Notes: Health check
+
+1. `GET / {} (1)`
+<br/>Notes: Welcome message
+
+1. `GET /version {} (1)`
+<br/>Notes: Return relevant module versions: mapnik, grainstore, etc
+
+
+## Optional deprecated routes
+
+- [0] `/tiles/layergroup` is deprecated and `/api/v1/map` should be used but we keep it for now.
+- [1] `/tiles/template` is deprecated and `/api/v1/map/named` should be used but we keep it for now.
+
+## How to generate the list of routes
+
+Something like the following patch should do the trick
+
+```javascript
+diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js
+index 5f62850..bca377d 100644
+--- a/lib/cartodb/server.js
++++ b/lib/cartodb/server.js
+@@ -215,6 +215,20 @@ module.exports = function(serverOptions) {
+      * END Routing
+      ******************************************************************************************************************/
+
++    var format = require('util').format;
++    var routesNotes = app._router.stack
++        .filter(function(handler) { return !!handler.route; })
++        .map(function(handler) {
++            return format("\n1. `%s %s {%s} (1)`\n<br/>Notes: [DEPRECATED]? ",
++                Object.keys(handler.route.methods)[0].toUpperCase(),
++                handler.route.path,
++                handler.keys.map(function(k) {
++                    return format(':%s(%s)', k.name, k.optional ? 't' : 'f');
++                }).join(',')
++            );
++        });
++    console.log(routesNotes.join('\n'));
++
+     return app;
+ };
+
+
+```

docs/Template-maps.md

@@ -1,272 +0,0 @@
-Template maps are layergroup configurations that rather than being
-fully defined contain variables that can be set to produce a different
-layergroup configurations (instantiation).
-
-Template maps are persistent, can only be created and deleted by the
-CartoDB user showing a valid API_KEY.
-
-
-# Template format
-
-A templated layergroup would allow using placeholders
-in the "cartocss" and "sql" elements in the "option"
-field of any "layer" of a layergroup configuration
-(see https://github.com/CartoDB/Windshaft/wiki/MapConfig-specification).
-
-Valid placeholder names start with a letter and can only
-contain letters, numbers or underscores. They have to be
-written between ``<%= `` and `` %>`` strings in order to be 
-replaced. Example: ``<%= my_color %>``.
-
-The set of supported placeholders for a template will need to be
-explicitly defined specifying type and default value for each.
-
-**placeholder types**
-
-Placeholder type will determine the kind of escaping for the
-associated value. Supported types are:
-
- * sql_literal (internal single-quotes will be sql-escaped)
- * sql_ident (internal double-quotes will be sql-escaped)
- * number (can only contain numerical representation)
- * css_color (can only contain color names or hex-values)
- * ... (add more as need arises)
-
-Placeholder default value will be used when not provided at
-instantiation time and could be used to test validity of the
-template by creating a default instance.
-
-```js
-// template.json 
-{
-  version: "0.0.1",
-  // there can be at most 1 template with the same name for any user 
-  // valid names start with a letter and only contains letter, numbers
-  // or underscores
-  name: "template_name", 
-  // embedded authorization certificate
-  auth: {
-   method: "token", // or "open" (the default if no "method" is given)
-   // only (required and non empty) for "token" method
-   valid_tokens: ["auth_token1","auth_token2"]
-  },
-  // Variables not listed here are not substituted
-  // Variable not provided at instantiation time trigger an error
-  // A default is required for optional variables
-  // Type specification is used for quoting, to avoid injections
-  placeholders: {
-      color: {
-        type:"css_color",
-        default:"red"
-      },
-      cartodb_id: {
-        type:"number",
-        default: 1
-      }
-  },
-  layergroup: {
-   // see https://github.com/CartoDB/Windshaft/wiki/MapConfig-specification
-   "version": "1.0.1",
-   "layers": [{
-    "type": "cartodb",
-    "options": {
-      "cartocss_version": "2.1.1", 
-      "cartocss": "#layer { polygon-fill: <%= color %>; }",
-      "sql": "select * from european_countries_e WHERE cartodb_id = <%= cartodb_id %>"
-    }
-   }]
-  } 
-}
-```
-
-# Creating a templated map
-
-You can create a template map with a single call (for simplicity).
-
-You'd use a POST sending JSON data:
-
-```sh
-curl -X POST \
-   -H 'Content-Type: application/json' \
-   -d @template.json \
-   'https://docs.cartodb.com/tiles/template?api_key=APIKEY'
-```
-
-The response would be like this:
-```js
-{
-   "template_id":"@template_name"
-}
-```
-
-If a template with the same name exists in the user storage,
-a 400 response is generated.
-
-Errors are in this form:
-```js
-{
-   "error":"Some error string here"
-}
-```
-
-# Updating an existing template
-
-You can update a template map with a PUT:
-
-```sh
-curl -X PUT \
-   -H 'Content-Type: application/json' \
-   -d @template.json \
-   'https://docs.cartodb.com/tiles/template/:template_name?api_key=APIKEY'
-```
-A template with the same name will be updated, if any.
-
-The response would be like this:
-```js
-{
-   "template_id":"@template_name"
-}
-```
-
-If a template with the same name does NOT exist,
-a 400 HTTP response is generated with an error in this format:
-
-```js
-{
-   "error":"Some error string here"
-}
-```
-
-
-# Listing available templates
-
-You can get a list of available templates with a GET to ``/template``.
-A valid api_key is required.
-
-```sh
-curl -X GET 'https://docs.cartodb.com/tiles/template?api_key=APIKEY'
-```
-
-The response would be like this:
-```js
-{
-   "template_ids": ["@template_name1","@template_name2"]
-}
-```
-
-Or, on error:
-
-```js
-{
-   "error":"Some error string here"
-}
-```
-
-# Getting a specific template
-
-You can get the definition of a template with a
-GET to ``/template/:template_name``.
-A valid api_key is required.
-
-Example:
-
-```sh
-curl -X GET 'https://docs.cartodb.com/tiles/template/@template_name?auth_token=AUTH_TOKEN'
-```
-
-The response would be like this:
-```js
-{
-   "template": {...}  // see template.json above
-}
-```
-
-Or, on error:
-
-```js
-{
-   "error":"Some error string here"
-}
-```
-
-# Instantiating a template map
-
-You can instantiate a template map passing all required parameters with
-a POST to ``/template/:template_name``.
-
-Valid credentials will be needed, if required by the template.
-
-```js
-// params.js
-{
- color: '#ff0000',
- cartodb_id: 3
-}
-```
-
-```sh
-curl -X POST \
-  -H 'Content-Type: application/json' \
-  -d @params.js \
-  'https://docs.cartodb.com/tiles/template/@template_name?auth_token=AUTH_TOKEN'
-
-```
-
-The response would be like this:
-```js
-{
-   "layergroupid":"docs@fd2861af@c01a54877c62831bb51720263f91fb33:123456788",
-   "last_updated":"2013-11-14T11:20:15.000Z"
-}
-```
-
-or, on error:
-
-```js
-{
-   "error":"Some error string here"
-}
-```
-
-You can then use the ``layergroupid`` for fetching tiles and grids as you do
-normally ( see https://github.com/CartoDB/Windshaft/wiki/Multilayer-API).
-But you'll still have to show the ``auth_token``, if required by the template.
-
-### using JSONP
-There is also a special endpoint to be able to instanciate using JSONP (for old browsers)
-
-```
-curl 'https://docs.cartodb.com/tiles/template/@template_name/jsonp?auth_token=AUTH_TOKEN&callback=function_name&config=template_params_json'
-```
-
-it takes the ``callback`` function (required), ``auth_token`` in case the template needs auth and ``config`` which is the variabñes for the template (in case it has variables). For example config may be created (using javascript)
-```
-url += "config=" + encodeURIComponent(
-JSON.stringify({ color: 'red' });
-```
-
-the response it's in this format:
-```
-jQuery17205720721024554223_1390996319118(
-{
-layergroupid: "dev@744bd0ed9b047f953fae673d56a47b4d:1390844463021.1401",
-last_updated: "2014-01-27T17:41:03.021Z"
-}
-)
-```
-# Deleting a template map
-
-You can delete a templated map with a DELETE to ``/template/:template_name``:
-
-```sh
-curl -X DELETE 'https://docs.cartodb.com/tiles/template/@template_name?auth_token=AUTH_TOKEN'
-```
-
-On success, a 204 (No Content) response would be issued.
-Otherwise a 4xx response with this format:
-
-```js
-{
-   "error":"Some error string here"
-}
-```

docs/anonymous_maps.md

@@ -0,0 +1,194 @@
+# Anonymous Maps
+
+Anonymous Maps allows you to instantiate a map given SQL and CartoCSS. It also allows you to add interaction capabilities using [UTF Grid.](https://github.com/mapbox/utfgrid-spec)
+
+
+## Instantiate
+
+#### Definition
+
+```html
+POST /api/v1/map
+```
+
+#### Params
+
+```javascript
+{
+  "version": "1.3.0",
+  "layers": [{
+    "type": "mapnik",
+    "options": {
+      "cartocss_version": "2.1.1",
+      "cartocss": "#layer { polygon-fill: #FFF; }",
+      "sql": "select * from european_countries_e",
+      "interactivity": ["cartodb_id", "iso3"]
+    }
+  }]
+}
+```
+
+See [MapConfig File Formats](http://docs.cartodb.com/cartodb-platform/maps-api/mapconfig/) for details.
+
+#### Response
+
+The response includes:
+
+Attributes | Description
+--- | ---
+layergroupid | The ID for that map, used to compose the URL for the tiles. The final URL is: `https://{username}.cartodb.com/api/v1/map/{layergroupid}/{z}/{x}/{y}.png`
+updated_at | The ISO date of the last time the data involved in the query was updated.
+metadata | Includes information about the layers.
+cdn_url | URLs to fetch the data using the best CDN for your zone.
+
+### Example
+
+#### Call
+
+```bash
+curl 'https://{username}.cartodb.com/api/v1/map' -H 'Content-Type: application/json' -d @mapconfig.json
+```
+
+#### Response
+
+```javascript
+{
+  "layergroupid": "c01a54877c62831bb51720263f91fb33:0",
+  "last_updated": "1970-01-01T00:00:00.000Z",
+  "metadata": {
+    "layers": [
+      {
+        "type": "mapnik",
+        "meta": {}
+      }
+    ]
+  },
+  "cdn_url": {
+    "http": "http://cdb.com",
+    "https": "https://cdb.com"
+  }
+}
+```
+
+### Retrieve resources from the layergroup
+
+When you have a layergroup, there are several resources for retrieving layergoup details such as, accessing Mapnik tiles, getting individual layers, accessing defined Attributes, and blending and layer selection.
+
+#### Mapnik tiles
+
+These tiles will get just the Mapnik layers. To get individual layers, see the following section.
+
+```bash
+https://{username}.cartodb.com/api/v1/map/{layergroupid}/{z}/{x}/{y}.png
+```
+
+#### Individual layers
+
+The MapConfig specification holds the layers definition in a 0-based index. Layers can be requested individually in different formats depending on the layer type.
+
+Individual layers can be accessed using that 0-based index. For UTF grid tiles:
+
+```bash
+https://{username}.cartodb.com/api/v1/map/{layergroupid}/{layer}/{z}/{x}/{y}.grid.json
+```
+
+In this case, `layer` as 0 returns the UTF grid tiles/attributes for layer 0, the only layer in the example MapConfig.
+
+If the MapConfig had a Torque layer at index 1 it could be possible to request it with:
+
+```bash
+https://{username}.cartodb.com/api/v1/map/{layergroupid}/1/{z}/{x}/{y}.torque.json
+```
+
+#### Attributes defined in `attributes` section
+
+```bash
+https://{username}.cartodb.com/api/v1/map/{layergroupid}/{layer}/attributes/{feature_id}
+```
+
+Which returns JSON with the attributes defined, like:
+
+```javascript
+{ "c": 1, "d": 2 }
+```
+
+#### Blending and layer selection
+
+```bash
+https://{username}.cartodb.com/api/v1/map/{layergroupid}/{layer_filter}/{z}/{x}/{y}.png
+```
+
+Note: currently format is limited to `png`.
+
+`layer_filter` can be used to select some layers to be rendered together. `layer_filter` supports two formats:
+
+- `all` alias
+
+Using `all` as `layer_filter` will blend all layers in the layergroup
+
+```bash
+https://{username}.cartodb.com/api/v1/map/{layergroupid}/all/{z}/{x}/{y}.png
+```
+
+- Filter by layer index
+
+A list of comma separated layer indexes can be used to just render a subset of layers. For example `0,3,4` will filter and blend layers with indexes 0, 3, and 4.
+
+```bash
+https://{username}.cartodb.com/api/v1/map/{layergroupid}/0,3,4/{z}/{x}/{y}.png
+```
+
+Some notes about filtering:
+
+  - Invalid index values or out of bounds indexes will end in `Invalid layer filtering` errors.
+  - Once a Mapnik layer is selected, all Mapnik layers will get blended. As this may change in the future **it is
+  recommended** to always select all Mapnik layers if you want to select at least one so you will get a consistent
+  behavior in the future.
+  - Ordering is not considered. So right now filtering layers 0,3,4 is the very same thing as filtering 3,4,0. As this
+  may change in the future **it is recommended** to always select the layers in ascending order so you will get a
+  consistent behavior in the future.
+
+
+## Create JSONP
+
+The JSONP endpoint is provided in order to allow web browsers access which don't support CORS.
+
+#### Definition
+
+```bash
+GET /api/v1/map?callback=method
+```
+
+#### Params
+
+Param | Description
+--- | ---
+config | Encoded JSON with the params for creating Named Maps (the variables defined in the template).
+lmza | This attribute contains the same as config but LZMA compressed. It cannot be used at the same time as `config`.
+callback | JSON callback name.
+
+### Example
+
+#### Call
+
+```bash
+curl "https://{username}.cartodb.com/api/v1/map?callback=callback&config=%7B%22version%22%3A%221.0.1%22%2C%22layers%22%3A%5B%7B%22type%22%3A%22cartodb%22%2C%22options%22%3A%7B%22sql%22%3A%22select+%2A+from+european_countries_e%22%2C%22cartocss%22%3A%22%23european_countries_e%7B+polygon-fill%3A+%23FF6600%3B+%7D%22%2C%22cartocss_version%22%3A%222.3.0%22%2C%22interactivity%22%3A%5B%22cartodb_id%22%5D%7D%7D%5D%7D"
+```
+
+#### Response
+
+```javascript
+callback({
+  layergroupid: "d9034c133262dfb90285cea26c5c7ad7:0",
+  cdn_url: {
+    "http": "http://cdb.com",
+    "https": "https://cdb.com"
+  },
+  last_updated: "1970-01-01T00:00:00.000Z"
+})
+```
+
+
+## Remove
+
+Anonymous Maps cannot be removed by an API call. They will expire after about five minutes, or sometimes longer. If an Anonymous Map expires and tiles are requested from it, an error will be raised. This could happen if a user leaves a map open and after time, returns to the map and attempts to interact with it in a way that requires new tiles (e.g. zoom). The client will need to go through the steps of creating the map again to fix the problem.

docs/general_concepts.md

@@ -0,0 +1,27 @@
+# General Concepts
+
+The following concepts are the same for every endpoint in the API except when it's noted explicitly.
+
+## Auth
+
+By default, users do not have access to private tables in CartoDB. In order to instantiate a map from private table data an API Key is required. Additionally, to include some endpoints, an API Key must be included (e.g. creating a Named Map).
+
+To execute an authorized request, `api_key=YOURAPIKEY` should be added to the request URL. The param can be also passed as POST param. Using HTTPS is mandatory when you are performing requests that include your `api_key`.
+
+## Errors
+
+Errors are reported using standard HTTP codes and extended information encoded in JSON with this format:
+
+```javascript
+{
+  "errors": [
+    "access forbidden to table TABLE"
+  ]
+}
+```
+
+If you use JSONP, the 200 HTTP code is always returned so the JavaScript client can receive errors from the JSON object.
+
+## CORS support
+
+All the endpoints, which might be accessed using a web browser, add CORS headers and allow OPTIONS method.

docs/metrics.md

@@ -25,9 +25,7 @@ Again, each inner timer may have several inner timers.
 - **TemplateMaps_instance**: time to retrieve a map template instance, see *getTemplate* and *authorizedByCert*
 - **affectedTables**: time to check what are the affected tables for adding the cache channel, see *addCacheChannel*
 - **authorize**: time to authorize a request, see *authorizedByAPIKey*, *authorizedByCert*, *authorizedBySigner*
-- **authorizedByAPIKey**: time to authorize using an API KEY
 - **authorizedByCert**: time to authorize a template instantiation
-- **authorizedBySigner**: time to authorize a request with auth_token
 - **findLastUpdated**: time to retrieve the last update time for a list of tables, see *affectedTables*
 - **generateCacheChannel**: time to generate the headers for the cache channel based on the request, see *addCacheChannel*
 - **getSignerMapKey**: time to retrieve from redis the authorized user for a template map

docs/named_maps.md

@@ -0,0 +1,531 @@
+# Named Maps
+
+Named Maps are essentially the same as Anonymous Maps except the MapConfig is stored on the server, and the map is given a unique name. You can create Named Maps from private data, and users without an API Key can view your Named Map (while keeping your data private). 
+
+The Named Map workflow consists of uploading a MapConfig file to CartoDB servers, to select data from your CartoDB user database by using SQL, and specifying the CartoCSS for your map. 
+
+The response back from the API provides the template_id of your Named Map as the `name` (the identifier of your Named Map), which is the name that you specified in the MapConfig. You can  which you can then use to create your Named Map details, or [fetch XYZ tiles](#fetching-xyz-tiles-for-named-maps) directly for Named Maps. 
+
+**Tip:** You can also use a Named Map that you created (which is defined by its `name`), to create a map using CartoDB.js. This is achieved by adding the [`namedmap` type](http://docs.cartodb.com/cartodb-platform/cartodb-js/layer-source-object/#named-maps-layer-source-object-type-namedmap) layer source object to draw the Named Map.
+
+The main differences, compared to Anonymous Maps, is that Named Maps include:
+
+- **auth token**  
+  This allows you to control who is able to see the map based on an auth token, and create a secure Named Map with password-protection.
+
+- **template map**  
+  The template map is static and may contain placeholders, enabling you to modify your maps appearance by using variables. Templates maps are persistent with no preset expiration. They can only be created, or deleted, by a CartoDB user with a valid API KEY (See [auth argument](#arguments)).
+
+  Uploading a MapConfig creates a Named Map. MapConfigs are uploaded to the server by sending the server a "template".json file, which contain the [MapConfig specifications](http://docs.cartodb.com/cartodb-platform/maps-api/mapconfig/).
+
+**Note:** There is a limit of 4,096 Named Maps allowed per account. If you need to create more Named Maps, it is recommended to use a single Named Map and change the variables using [placeholders](#placeholder-format), instead of uploading multiple [Named Map MapConfigs](http://docs.cartodb.com/cartodb-platform/maps-api/mapconfig/#named-map-layer-options).
+
+## Create
+
+#### Definition
+
+```html
+POST /api/v1/map/named
+```
+
+#### Params
+
+Params | Description
+--- | ---
+api_key | is required
+MapConfig | a [Named Map MapConfig](http://docs.cartodb.com/cartodb-platform/maps-api/mapconfig/#named-map-layer-options) is required to create a Named Map
+
+#### template.json
+
+The `name` argument defines how to name this "template_name".json. Note that there are some requirements for how to name a Named Map template. See the [`name`](#arguments) argument description for details.
+
+```javascript
+{
+  "version": "0.0.1",
+  "name": "template_name",
+  "auth": {
+    "method": "token",
+    "valid_tokens": [
+      "auth_token1",
+      "auth_token2"
+    ]
+  },
+  "placeholders": {
+    "color": {
+      "type": "css_color",
+      "default": "red"
+    },
+    "cartodb_id": {
+      "type": "number",
+      "default": 1
+    }
+  },
+  "layergroup": {
+    "version": "1.0.1",
+    "layers": [
+      {
+        "type": "cartodb",
+        "options": {
+          "cartocss_version": "2.1.1",
+          "cartocss": "#layer { polygon-fill: <%= color %>; }",
+          "sql": "select * from european_countries_e WHERE cartodb_id = <%= cartodb_id %>"
+        }
+      }
+    ]
+  },
+  "view": {
+    "zoom": 4,
+    "center": {
+      "lng": 0,
+      "lat": 0
+    },
+    "bounds": {
+      "west": -45,
+      "south": -45,
+      "east": 45,
+      "north": 45
+    }
+  }
+}
+```
+
+#### Arguments
+
+Params | Description
+--- | ---
+name | There can only be _one_ template with the same name for any user. Valid names start with a letter or a number, and only contain letters, numbers, dashes (-), or underscores (_). _This is specific to the name of your Named Map that is specified in the `name` property of the template file_.
+
+auth | 
+--- | ---
+&#124;_ method | `"token"` or `"open"` (`"open"` is the default if no method is specified. Use `"token"` to password-protect your map)
+&#124;_ valid_tokens | when `"method"` is set to `"token"`, the values listed here allow you to instantiate the Named Map. See this [example](http://docs.cartodb.com/faqs/manipulating-your-data/#how-to-create-a-password-protected-named-map) for how to create a password-protected map.
+placeholders | Placeholders are variables that can be placed in your template.json file's SQL or CartoCSS.
+layergroup | the layergroup configurations, as specified in the template. See [MapConfig File Format](http://docs.cartodb.com/cartodb-platform/maps-api/mapconfig/) for more information.
+view (optional) | extra keys to specify the view area for the map. It can be used to have a static preview of a Named Map without having to instantiate it. It is possible to specify it with `center` + `zoom` or with a bounding box `bbox`. Center+zoom takes precedence over bounding box.
+--- | ---
+&#124;_ zoom | The zoom level to use
+
+&#124;_ center | 
+--- | ---
+&#124;_ &#124;_ lng | The longitude to use for the center
+&#124;_ &#124;_ lat | The latitude to use for the center
+
+&#124;_ bounds | 
+--- | ---
+&#124;_ &#124;_ west | LowerCorner longitude for the bounding box, in decimal degrees (aka most western)
+&#124;_ &#124;_ south | LowerCorner latitude for the bounding box, in decimal degrees (aka most southern)
+&#124;_ &#124;_ east | UpperCorner longitude for the bounding box, in decimal degrees (aka most eastern)
+&#124;_ &#124;_ north | UpperCorner latitude for the bounding box, in decimal degrees (aka most northern)
+
+
+### Placeholder Format
+
+Placeholders are variables that can be placed in your template.json file. Placeholders need to be defined with a `type` and a default value for MapConfigs. See details about defining a MapConfig `type` for [Layergoup configurations](http://docs.cartodb.com/cartodb-platform/maps-api/mapconfig/#layergroup-configurations).
+
+Valid placeholder names start with a letter and can only contain letters, numbers, or underscores. They have to be written between the `<%=` and `%>` strings in order to be replaced inside the Named Maps API.
+
+#### Example
+
+```javascript
+<%= my_color %>
+```
+
+The set of supported placeholders for a template need to be explicitly defined with a specific type, and default value, for each placeholder.
+
+### Placeholder Types
+
+The placeholder type will determine the kind of escaping for the associated value. Supported types are:
+
+Types | Description
+--- | ---
+sql_literal | internal single-quotes will be sql-escaped
+sql_ident | internal double-quotes will be sql-escaped
+number | can only contain numerical representation
+css_color | can only contain color names or hex-values
+
+Placeholder default values will be used whenever new values are not provided as options, at the time of creation on the client. They can also be used to test the template by creating a default version with new options provided.
+
+When using templates, be very careful about your selections as they can give broad access to your data if they are defined loosely.
+
+#### Call
+
+This is the call for creating the Named Map. It is sending the template.json file to the service, and the server responds with the template id.
+
+```bash
+curl -X POST \
+   -H 'Content-Type: application/json' \
+   -d @template.json \
+   'https://{username}.cartodb.com/api/v1/map/named?api_key={api_key}'
+```
+
+#### Response
+
+The response back from the API provides the name of your MapConfig as a template, enabling you to edit the Named Map details by inserting your variables into the template where placeholders are defined, and create custom queries using SQL. 
+
+```javascript
+{
+  "template_id":"name"
+}
+```
+
+## Instantiate
+
+Instantiating a Named Map allows you to fetch the map tiles. You can use the Maps API to instantiate, or use the CartoDB.js `createLayer()` function. The result is an Anonymous Map.
+
+#### Definition
+
+```html
+POST /api/v1/map/named/{template_name}
+```
+
+#### Param
+
+Param | Description
+--- | ---
+auth_token | `"token"` or `"open"` (`"open"` is the default if not specified. Use `"token"` to password-protect your map)
+
+```javascript
+// params.json, this is required if the Named Map allows variables (if placeholders were defined in the template.json by the user)
+{
+ "color": "#ff0000",
+ "cartodb_id": 3
+}
+```
+
+The fields you pass as `params.json` depend on the variables allowed by the Named Map. If there are variables missing, it will raise an error (HTTP 400).
+
+**Note:** It is required that you include a `params.json` file to instantiate a Named Map that contains variables, even if you have no fields to pass and the JSON is empty. (This is specific to when a Named Map allows variables (if placeholders were defined in the template.json by the user).
+
+#### Example
+
+You can initialize a template map by passing all of the required parameters in a POST to `/api/v1/map/named/{template_name}`.
+
+Valid auth token will be needed, if required by the template.
+
+
+#### Call
+
+```bash
+curl -X POST \
+  -H 'Content-Type: application/json' \
+  -d @params.json \
+  'https://{username}.cartodb.com/api/v1/map/named/{template_name}?auth_token={auth_token}'
+```
+
+#### Response
+
+```javascript
+{
+  "layergroupid": "docs@fd2861af@c01a54877c62831bb51720263f91fb33:123456788",
+  "last_updated": "2013-11-14T11:20:15.000Z"
+}
+```
+
+#### Error
+
+```javascript
+{
+  "errors" : ["Some error string here"]
+}
+```
+
+You can then use the `layergroupid` for fetching tiles and grids as you would normally (see [Anonymous Maps](http://docs.cartodb.com/cartodb-platform/maps-api/anonymous-maps/)).
+
+## Update
+
+#### Definition
+
+```bash
+PUT /api/v1/map/named/{template_name}
+```
+
+#### Params
+
+Param | Description
+--- | ---
+api_key | is required
+
+#### Response
+
+Same as updating a map.
+
+### Other Information
+
+Updating a Named Map removes all the Named Map instances, so they need to be initialized again.
+
+### Example
+
+#### Call
+
+```bash
+curl -X PUT \
+  -H 'Content-Type: application/json' \
+  -d @template.json \
+  'https://{username}.cartodb.com/api/v1/map/named/{template_name}?api_key={api_key}'
+```
+
+#### Response
+
+```javascript
+{
+  "template_id": "@template_name"
+}
+```
+
+If any template has the same name, it will be updated.
+
+If a template with the same name does NOT exist, a 400 HTTP response is generated with an error in this format:
+
+```javascript
+{
+  "errors" : ["error string here"]
+}
+```
+
+## Delete
+
+Deletes the specified template map from the server, and disables any previously initialized versions of the map.
+
+#### Definition
+
+```bash
+DELETE /api/v1/map/named/{template_name}
+```
+
+#### Params
+
+Param | Description
+--- | ---
+api_key | is required
+
+### Example
+
+#### Call
+
+```bash
+curl -X DELETE 'https://{username}.cartodb.com/api/v1/map/named/{template_name}?api_key={api_key}'
+```
+
+#### Response
+
+```javascript
+{
+  "errors" : ["Some error string here"]
+}
+```
+
+On success, a 204 (No Content) response will be issued. Otherwise a 4xx response with an error will be returned.
+
+## Listing Available Templates
+
+This allows you to get a list of all available templates.
+
+#### Definition
+
+```bash
+GET /api/v1/map/named/
+```
+
+#### Params
+
+Param | Description
+--- | ---
+api_key | is required
+
+### Example
+
+#### Call
+
+```bash
+curl -X GET 'https://{username}.cartodb.com/api/v1/map/named?api_key={api_key}'
+```
+
+#### Response
+
+```javascript
+{
+   "template_ids": ["@template_name1","@template_name2"]
+}
+```
+
+#### Error
+
+```javascript
+{
+  "errors" : ["Some error string here"]
+}
+```
+
+## Get Template Definition
+
+This gets the definition of a requested template.
+
+#### Definition
+
+```bash
+GET /api/v1/map/named/{template_name}
+```
+
+#### Params
+
+Param | Description
+--- | ---
+api_key | is required
+
+### Example
+
+#### Call
+
+```bash
+curl -X GET 'https://{username}.cartodb.com/api/v1/map/named/{template_name}?api_key={api_key}'
+```
+
+#### Response
+
+```javascript
+{
+  "template": {...} // see [template.json](#templatejson)
+}
+```
+
+#### Error
+
+```javascript
+{
+  "errors" : ["Some error string here"]
+}
+```
+
+## JSONP for Named Maps
+
+If using a [JSONP](https://en.wikipedia.org/wiki/JSONP) (for old browsers) request, there is a special endpoint used to initialize and create a Named Map.
+
+#### Definition
+
+```bash
+GET /api/v1/map/named/{template_name}/jsonp
+```
+
+#### Params
+
+Params | Description
+--- | ---
+auth_token | `"token"` or `"open"` (`"open"` is the default if no method is specified. Use `"token"` to password-protect your map)
+params | Encoded JSON with the params (variables) needed for the Named Map
+lmza | You can use an LZMA compressed file instead of a params JSON file
+callback | JSON callback name
+
+#### Call
+
+```bash
+curl 'https://{username}.cartodb.com/api/v1/map/named/{template_name}/jsonp?auth_token={auth_token}&callback=callback&config=template_params_json'
+```
+
+#### Response
+
+```javascript
+callback({
+  "layergroupid":"c01a54877c62831bb51720263f91fb33:0",
+  "last_updated":"1970-01-01T00:00:00.000Z"
+  "cdn_url": {
+    "http": "http://cdb.com",
+    "https": "https://cdb.com"
+  }
+})
+```
+
+This takes the `callback` function (required), `auth_token` if the template needs auth, and `config` which is the variable for the template (in cases where it has variables).
+
+```javascript
+url += "config=" + encodeURIComponent(
+JSON.stringify({ color: 'red' });
+```
+
+The response is:
+
+```javascript
+callback({
+  layergroupid: "dev@744bd0ed9b047f953fae673d56a47b4d:1390844463021.1401",
+  last_updated: "2014-01-27T17:41:03.021Z"
+})
+```
+
+## CartoDB.js for Named Maps
+You can use a Named Map that you created (which is defined by its `name`), to create a map using CartoDB.js. This is achieved by adding the [`namedmap` type](http://docs.cartodb.com/cartodb-platform/cartodb-js/layer-source-object/#named-maps-layer-source-object-type-namedmap) layer source object to draw the Named Map.
+
+```javascript
+{
+  user_name: '{username}', // Required
+  type: 'namedmap', // Required
+  named_map: {
+    name: '{name_of_map}', // Required, the 'name' of the Named Map that you have created
+    // Optional
+    layers: [{
+      layer_name: "sublayer0", // Optional
+      interactivity: "column1, column2, ..." // Optional
+    },
+    {
+      layer_name: "sublayer1",
+      interactivity: "column1, column2, ..."
+    },
+      ...
+    ],
+    // Optional
+    params: {
+      color: "hex_value",
+      num: 2
+    }
+  }
+}
+```
+**Note:** Instantiating a Named Map over a `createLayer` does not require an API Key and by default, does not include auth tokens. _If_ you defined auth tokens for the Named Map configuration, then you will have to include them.
+
+[CartoDB.js](http://docs.cartodb.com/cartodb-platform/cartodb-js/) has methods for accessing your Named Maps.
+
+1. [layer.setParams()](http://docs.cartodb.com/cartodb-platform/cartodb-js/api-methods/#layersetparamskey-value) allows you to change the template variables (in the placeholders object) via JavaScript
+
+    **Note:** The CartoDB.js `layer.setParams()` function is not supported when using Named Maps for Torque.
+
+2. [layer.setAuthToken()](http://docs.cartodb.com/cartodb-platform/cartodb-js/api-methods/#layersetauthtokenauthtoken) allows you to set the auth tokens to create the layer
+
+#### Examples of Named Maps created with CartoDB.js
+
+- [Named Map selectors with interaction](http://bl.ocks.org/ohasselblad/515a8af1f99d5e690484)
+
+- [Named Map with interactivity](http://bl.ocks.org/ohasselblad/d1a45b8ff5e7bd90cd68)
+
+- [Toggling sublayers in a Named Map](http://bl.ocks.org/ohasselblad/c1a0f4913610eec53cd3)
+
+## Fetching XYZ Tiles for Named Maps
+
+Optionally, authenticated users can fetch projected tiles (XYZ tiles or Mapnik Retina tiles) for your Named Map.
+
+### Fetch XYZ Tiles Directly with a URL
+
+Authenticated users, with an auth token, can use XYZ-based URLs to fetch tiles directly, and instantiate the Named Map as part of the request to your application. You do not have to do any other steps to initialize your map. 
+
+To call a template_id in a URL:
+
+`/{template_id}/{layer}/{z}/{x}/{y}.{format}`
+
+For example, a complete URL might appear as:
+
+"https://{username}.cartodb.com/api/v1/map/named/{template_id}/{layer}/{z}/{x}/{y}.png"
+
+The placeholders indicate the following:
+
+- [`template_id`](http://docs.cartodb.com/cartodb-platform/maps-api/named-maps/#response) is the response of your Named Map.
+- layers can be a number (referring to the # layer of your map), all layers of your map, or a list of layers.
+  - To show just the basemap layer, enter the number value `0` in the layer placeholder "https://{username}.cartodb.com/api/v1/map/named/{template_id}/0/{z}/{x}/{y}.png"
+  - To show the first layer, enter the number value `1` in the layer placeholder "https://{username}.cartodb.com/api/v1/map/named/{template_id}/1/{z}/{x}/{y}.png"
+  - To show all layers, enter the value `all` for the layer placeholder "https://{username}.cartodb.com/api/v1/map/named/{template_id}/all/{z}/{x}/{y}.png"
+  - To show a [list of layers](http://docs.cartodb.com/cartodb-platform/maps-api/anonymous-maps/#blending-and-layer-selection), enter the comma separated layer value as 0,1,2 in the layer placeholder. For example, to show the basemap and the first layer, "https://{username}.cartodb.com/api/v1/map/named/{template_id}/0,1/{z}/{x}/{y}.png"
+
+
+### Get Mapnik Retina Tiles
+
+Mapnik Retina tiles are not directly supported for Named Maps, so you cannot use the Named Map template_id. To fetch Mapnik Retina tiles, get the [layergroupid](http://docs.cartodb.com/cartodb-platform/maps-api/named-maps/#response-1) to initialize the map.
+
+Instantiate the map by using your `layergroupid` in the token placeholder:
+
+ `{token}/{z}/{x}/{y}@{scale_factor}?{x}.{format}`

docs/quickstart.md

@@ -0,0 +1,100 @@
+# Quickstart
+
+## Anonymous Maps
+
+Here is an example of how to create an Anonymous Map with JavaScript:
+
+```javascript
+var mapconfig = {
+  "version": "1.3.1",
+  "layers": [{
+    "type": "cartodb",
+    "options": {
+      "cartocss_version": "2.1.1",
+      "cartocss": "#layer { polygon-fill: #FFF; }",
+      "sql": "select * from european_countries_e"
+    }
+  }]
+}
+
+$.ajax({
+  crossOrigin: true,
+  type: 'POST',
+  dataType: 'json',
+  contentType: 'application/json',
+  url: 'https://{username}.cartodb.com/api/v1/map',
+  data: JSON.stringify(mapconfig),
+  success: function(data) {
+    var templateUrl = 'https://{username}.cartodb.com/api/v1/map/' + data.layergroupid + '/{z}/{x}/{y}.png'
+    console.log(templateUrl);
+  }
+})
+```
+
+## Named Maps
+
+Let's create a Named Map using some private tables in a CartoDB account.
+The following map config sets up a map of European countries that have a white fill color:
+
+```javascript
+{
+  "version": "0.0.1",
+  "name": "test",
+  "auth": {
+    "method": "open"
+  },
+  "layergroup": {
+    "layers": [{
+      "type": "mapnik",
+      "options": {
+        "cartocss_version": "2.1.1",
+        "cartocss": "#layer { polygon-fill: #FFF; }",
+        "sql": "select * from european_countries_e"
+      }
+    }]
+  }
+}
+```
+
+The MapConfig needs to be sent to CartoDB's Map API using an authenticated call. Here we will use a command line tool called `curl`. For more info about this tool, see [this blog post](http://quickleft.com/blog/command-line-tutorials-curl), or type `man curl` in bash. Using `curl`, and storing the config from above in a file `MapConfig.json`, the call would look like:
+
+#### Call
+
+```bash
+curl 'https://{username}.cartodb.com/api/v1/map/named?api_key={api_key}' -H 'Content-Type: application/json' -d @mapconfig.json
+```
+
+To get the `URL` to fetch the tiles you need to instantiate the map, where `template_id` is the template name from the previous response.
+
+#### Call
+
+```bash
+curl -X POST 'https://{username}.cartodb.com/api/v1/map/named/{template_id}' -H 'Content-Type: application/json'
+```
+
+The response will return JSON with properties for the `layergroupid`, the timestamp (`last_updated`) of the last data modification and some key/value pairs with `metadata` for the `layers`.
+
+Note: all `layers` in `metadata` will always have a `type` string and a `meta` dictionary with the key/value pairs.
+
+#### Response
+
+```javascript
+{
+  "layergroupid": "c01a54877c62831bb51720263f91fb33:0",
+  "last_updated": "1970-01-01T00:00:00.000Z",
+  "metadata": {
+    "layers": [
+      {
+        "type": "mapnik",
+        "meta": {}
+      }
+    ]
+  }
+}
+```
+
+You can use the `layergroupid` to instantiate a URL template for accessing tiles on the client. Here we use the `layergroupid` from the example response above in this URL template:
+
+```bash
+https://{username}.cartodb.com/api/v1/map/{layergroupid}/{z}/{x}/{y}.png
+```

docs/static_maps_api.md

@@ -0,0 +1,221 @@
+# Static Maps API
+
+The Static Maps API can be initiated using both Named and Anonymous Maps using the 'layergroupid' token. The API can be used to create static images of parts of maps and thumbnails for use in web design, graphic design, print, field work, and many other applications that require standard image formats.
+
+## Maps API endpoints
+
+Begin by instantiating either a Named or Anonymous Map using the `layergroupid token` as demonstrated in the Maps API documentation above. The `layergroupid` token calls to the map and allows for parameters in the definition to generate static images.
+
+### Zoom + center
+
+#### Definition
+
+```bash
+GET /api/v1/map/static/center/{token}/{z}/{lat}/{lng}/{width}/{height}.{format}
+```
+
+#### Params
+
+Param | Description
+--- | ---
+token | the layergroupid token from the map instantiation
+z | the zoom level of the map
+lat | the latitude for the center of the map
+
+format | the format for the image, supported types: `png`, `jpg`
+--- | ---
+|_ jpg | will have a default quality of 85.
+
+### Bounding Box
+
+#### Definition
+
+```bash
+GET /api/v1/map/static/bbox/{token}/{bbox}/{width}/{height}.{format}`
+```
+
+#### Params
+
+Param | Description
+--- | ---
+token | the layergroupid token from the map instantiation
+
+bbox | the bounding box in WGS 84 (EPSG:4326), comma separated values for:
+--- | ---
+ | LowerCorner longitude, in decimal degrees (aka most western)
+ | LowerCorner latitude, in decimal degrees (aka most southern)
+ | UpperCorner longitude, in decimal degrees (aka most eastern)
+ | UpperCorner latitude, in decimal degrees (aka most northern)
+width | the width in pixels for the output image
+height | the height in pixels for the output image
+format | the format for the image, supported types: `png`, `jpg`
+--- | ---
+|_ jpg | will have a default quality of 85.
+
+Note: you can see this endpoint as
+
+```bash
+GET /api/v1/map/static/bbox/{token}/{west},{south},{east},{north}/{width}/{height}.{format}`
+```
+
+### Named Map
+
+#### Definition
+
+```bash
+GET /api/v1/map/static/named/{name}/{width}/{height}.{format}
+```
+
+#### Params
+
+Param | Description
+--- | ---
+name | the name of the Named Map
+width | the width in pixels for the output image
+height | the height in pixels for the output image
+
+format | the format for the image, supported types: `png`, `jpg`
+--- | ---
+|_ jpg | will have a default quality of 85.
+
+A Named Maps static image will get its constraints from the [`view` argument of the Create Named Map function](http://docs.cartodb.com/cartodb-platform/maps-api/named-maps/#arguments). If `view` is not defined, it will estimate the extent based on the involved tables, otherwise it fallbacks to `"zoom": 1`, `"lng": 0` and `"lat": 0`.
+
+#### Layers
+
+The Static Maps API allows for multiple layers of incorporation into the `MapConfig` to allow for maximum versatility in creating a static map. The examples below were used to generate the static image example in the next section, and appear in the specific order designated.
+
+**Basemaps**
+
+```javascript
+{
+  "type": "http",
+  "options": {
+    "urlTemplate": "http://{s}.basemaps.cartocdn.com/dark_nolabels/{z}/{x}/{y}.png",
+    "subdomains": [
+      "a",
+      "b",
+      "c"
+    ]
+  }
+}
+```
+
+By manipulating the `"urlTemplate"` custom basemaps can be used in generating static images. Supported map types for the Static Maps API are:
+
+```javascript
+'http://{s}.basemaps.cartocdn.com/dark_all/{z}/{x}/{y}.png',
+'http://{s}.basemaps.cartocdn.com/dark_nolabels/{z}/{x}/{y}.png',
+'http://{s}.basemaps.cartocdn.com/light_all/{z}/{x}/{y}.png',
+'http://{s}.basemaps.cartocdn.com/light_nolabels/{z}/{x}/{y}.png',
+```
+
+**Mapnik**
+
+```javascript
+{
+  "type": "mapnik",
+  "options": {
+    "sql": "select null::geometry the_geom_webmercator",
+    "cartocss": "#layer {\n\tpolygon-fill: #FF3300;\n\tpolygon-opacity: 0;\n\tline-color: #333;\n\tline-width: 0;\n\tline-opacity: 0;\n}",
+    "cartocss_version": "2.2.0"
+  }
+},
+```
+
+**CartoDB**
+
+As described in the [MapConfig File Format](http://docs.cartodb.com/cartodb-platform/maps-api/mapconfig/), a "cartodb" type layer is now just an alias to a "mapnik" type layer as above, intended for backwards compatibility.
+
+```javascript
+{
+  "type": "cartodb",
+  "options": {
+    "sql": "select * from park",
+    "cartocss": "/** simple visualization */\n\n#park{\n  polygon-fill: #229A00;\n  polygon-opacity: 0.7;\n  line-color: #FFF;\n  line-width: 0;\n  line-opacity: 1;\n}",
+    "cartocss_version": "2.1.1"
+  }
+}
+```
+
+Additionally, static images from Torque maps and other map layers can be used together to generate highly customizable and versatile static maps.
+
+
+### Caching
+
+It is important to note that generated images are cached from the live data referenced with the `layergroupid token` on the specified CartoDB account. This means that if the data changes, the cached image will also change. When linking dynamically, it is important to take into consideration the state of the data and longevity of the static image to avoid broken images or changes in how the image is displayed. To obtain a static snapshot of the map as it is today and preserve the image long-term regardless of changes in data, the image must be saved and stored locally.
+
+### Limits
+
+* While images can encompass an entirety of a map, the default limit for pixel range is 8192 x 8192.
+* Image resolution by default is set to 72 DPI
+* JPEG quality by default is 85%
+* Timeout limits for generating static maps are the same across the CartoDB Editor and Platform. It is important to ensure timely processing of queries.
+
+## Examples
+
+After instantiating a map from a CartoDB account:
+
+#### Call
+
+```bash
+ GET /api/v1/map/static/center/{layergroupid}/{z}/{x}/{y}/{width}/{height}.png
+```
+
+#### Response
+
+<p class="wrap-border"><img src="https://raw.githubusercontent.com/namessanti/Pictures/master/static_api.png" alt="static-api"/></p>
+
+### MapConfig
+
+For this map, the multiple layers, order, and stylings are defined by the MapConfig.
+
+```javascript
+{
+  "version": "1.3.0",
+  "layers": [
+    {
+      "type": "http",
+      "options": {
+        "urlTemplate": "http://{s}.basemaps.cartocdn.com/dark_nolabels/{z}/{x}/{y}.png",
+        "subdomains": [
+          "a",
+          "b",
+          "c"
+        ]
+      }
+    },
+    {
+      "type": "mapnik",
+      "options": {
+        "sql": "select null::geometry the_geom_webmercator",
+        "cartocss": "#layer {\n\tpolygon-fill: #FF3300;\n\tpolygon-opacity: 0;\n\tline-color: #333;\n\tline-width: 0;\n\tline-opacity: 0;\n}",
+        "cartocss_version": "2.2.0"
+      }
+    },
+    {
+      "type": "cartodb",
+      "options": {
+        "sql": "select * from park",
+        "cartocss": "/** simple visualization */\n\n#park{\n  polygon-fill: #229A00;\n  polygon-opacity: 0.7;\n  line-color: #FFF;\n  line-width: 0;\n  line-opacity: 1;\n}",
+        "cartocss_version": "2.1.1"
+      }
+    },
+    {
+      "type": "cartodb",
+      "options": {
+        "sql": "select * from residential_zoning_2009",
+        "cartocss": "/** simple visualization */\n\n#residential_zoning_2009{\n  polygon-fill: #c7eae5;\n  polygon-opacity: 1;\n  line-color: #FFF;\n  line-width: 0.2;\n  line-opacity: 0.5;\n}",
+        "cartocss_version": "2.1.1"
+      }
+    },
+    {
+      "type": "cartodb",
+      "options": {
+        "sql": "select * from nycha_developments_july2011",
+        "cartocss": "/** simple visualization */\n\n#nycha_developments_july2011{\n  polygon-fill: #ef3b2c;\n  polygon-opacity: 0.7;\n  line-color: #FFF;\n  line-width: 0;\n  line-opacity: 1;\n}",
+        "cartocss_version": "2.1.1"
+      }
+    }
+  ]
+}
+```

lib/cartodb/api/auth_api.js

@@ -0,0 +1,141 @@
+var assert = require('assert');
+var step = require('step');
+
+/**
+ *
+ * @param {PgConnection} pgConnection
+ * @param metadataBackend
+ * @param {MapStore} mapStore
+ * @param {TemplateMaps} templateMaps
+ * @constructor
+ * @type {AuthApi}
+ */
+function AuthApi(pgConnection, metadataBackend, mapStore, templateMaps) {
+    this.pgConnection = pgConnection;
+    this.metadataBackend = metadataBackend;
+    this.mapStore = mapStore;
+    this.templateMaps = templateMaps;
+}
+
+module.exports = AuthApi;
+
+// Check if a request is authorized by a signer
+//
+// @param req express request object
+// @param callback function(err, signed_by) signed_by will be
+//                 null if the request is not signed by anyone
+//                 or will be a string cartodb username otherwise.
+//
+AuthApi.prototype.authorizedBySigner = function(req, callback) {
+    if ( ! req.params.token || ! req.params.signer ) {
+        return callback(null, false); // no signer requested
+    }
+
+    var self = this;
+
+    var layergroup_id = req.params.token;
+    var auth_token = req.params.auth_token;
+
+    this.mapStore.load(layergroup_id, function(err, mapConfig) {
+        if (err) {
+            return callback(err);
+        }
+
+        var authorized = self.templateMaps.isAuthorized(mapConfig.obj().template, auth_token);
+
+        return callback(null, authorized);
+    });
+};
+
+// Check if a request is authorized by api_key
+//
+// @param user
+// @param req express request object
+// @param callback function(err, authorized)
+//        NOTE: authorized is expected to be 0 or 1 (integer)
+//
+AuthApi.prototype.authorizedByAPIKey = function(user, req, callback) {
+    var givenKey = req.query.api_key || req.query.map_key;
+    if ( ! givenKey && req.body ) {
+        // check also in request body
+        givenKey = req.body.api_key || req.body.map_key;
+    }
+    if ( ! givenKey ) {
+        return callback(null, 0); // no api key, no authorization...
+    }
+
+    var self = this;
+
+    step(
+        function () {
+            self.metadataBackend.getUserMapKey(user, this);
+        },
+        function checkApiKey(err, val){
+            assert.ifError(err);
+            return val && givenKey === val;
+        },
+        function finish(err, authorized) {
+            callback(err, authorized);
+        }
+    );
+};
+
+/**
+ * Check access authorization
+ *
+ * @param req - standard req object. Importantly contains table and host information
+ * @param callback function(err, allowed) is access allowed not?
+ */
+AuthApi.prototype.authorize = function(req, callback) {
+    var self = this;
+    var user = req.context.user;
+
+    step(
+        function () {
+            self.authorizedByAPIKey(user, req, this);
+        },
+        function checkApiKey(err, authorized){
+            if (req.profiler) {
+                req.profiler.done('authorizedByAPIKey');
+            }
+            assert.ifError(err);
+
+            // if not authorized by api_key, continue
+            if (!authorized)  {
+                // not authorized by api_key, check if authorized by signer
+                return self.authorizedBySigner(req, this);
+            }
+
+            // authorized by api key, login as the given username and stop
+            self.pgConnection.setDBAuth(user, req.params, function(err) {
+                callback(err, true); // authorized (or error)
+            });
+        },
+        function checkSignAuthorized(err, authorized) {
+            if (err) {
+                return callback(err);
+            }
+
+            if ( ! authorized ) {
+                // request not authorized by signer.
+
+                // if no signer name was given, let dbparams and
+                // PostgreSQL do the rest.
+                //
+                if ( ! req.params.signer ) {
+                    return callback(null, true); // authorized so far
+                }
+
+                // if signer name was given, return no authorization
+                return callback(null, false);
+            }
+
+            self.pgConnection.setDBAuth(user, req.params, function(err) {
+                if (req.profiler) {
+                    req.profiler.done('setDBAuth');
+                }
+                callback(err, true); // authorized (or error)
+            });
+        }
+    );
+};

lib/cartodb/api/overviews_metadata_api.js

@@ -0,0 +1,46 @@
+function OverviewsMetadataApi(pgQueryRunner) {
+    this.pgQueryRunner = pgQueryRunner;
+}
+
+module.exports = OverviewsMetadataApi;
+
+// TODO: share this with QueryTablesApi? ... or maintain independence?
+var affectedTableRegexCache = {
+    bbox: /!bbox!/g,
+    scale_denominator: /!scale_denominator!/g,
+    pixel_width: /!pixel_width!/g,
+    pixel_height: /!pixel_height!/g
+};
+
+function prepareSql(sql) {
+    return sql
+        .replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)')
+        .replace(affectedTableRegexCache.scale_denominator, '0')
+        .replace(affectedTableRegexCache.pixel_width, '1')
+        .replace(affectedTableRegexCache.pixel_height, '1')
+    ;
+}
+
+OverviewsMetadataApi.prototype.getOverviewsMetadata = function (username, sql, callback) {
+    // FIXME: Currently using internal function _cdb_schema_name
+    //        CDB_Overviews should provide the schema information directly.
+    var query = 'SELECT *, _cdb_schema_name(base_table)' +
+                '  FROM CDB_Overviews(CDB_QueryTablesText($windshaft$' + prepareSql(sql) + '$windshaft$))';
+    this.pgQueryRunner.run(username, query, function handleOverviewsRows(err, rows) {
+        if (err){
+            callback(err);
+            return;
+        }
+        var metadata = rows.reduce(function(metadata, row){
+            var table = row.base_table;
+            var schema = row._cdb_schema_name;
+            if ( !metadata[table] ) {
+                metadata[table] = {};
+            }
+            metadata[table][row.z] = { table: row.overview_table };
+            metadata[table].schema = schema;
+            return metadata;
+        }, {});
+        return callback(null, metadata);
+    });
+};

lib/cartodb/api/query_tables_api.js

@@ -1,122 +0,0 @@
-var sqlApi = require('../sql/sql_api'),
-    PSQL   = require('cartodb-psql');
-
-function QueryTablesApi() {
-}
-
-var affectedTableRegexCache = {
-    bbox: /!bbox!/g,
-    scale_denominator: /!scale_denominator!/g,
-    pixel_width: /!pixel_width!/g,
-    pixel_height: /!pixel_height!/g
-};
-
-module.exports = QueryTablesApi;
-
-QueryTablesApi.prototype.getLastUpdatedTime = function (username, api_key, tableNames, callback) {
-    var sql = 'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max FROM CDB_TableMetadata m WHERE m.tabname = any (ARRAY['+
-        tableNames.map(function(t) { return "'" + t + "'::regclass"; }).join(',') +
-        '])';
-
-    // call sql api
-    sqlApi.query(username, api_key, sql, function(err, rows){
-        if (err){
-            var msg = err.message ? err.message : err;
-            callback(new Error('could not find last updated timestamp: ' + msg));
-            return;
-        }
-        // when the table has not updated_at means it hasn't been changed so a default last_updated is set
-        var last_updated = 0;
-        if(rows.length !== 0) {
-            last_updated = rows[0].max || 0;
-        }
-
-        callback(null, last_updated*1000);
-    });
-};
-
-QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, options, sql, callback) {
-
-    var query = 'SELECT CDB_QueryTables($windshaft$' + prepareSql(sql) + '$windshaft$)';
-
-    runQuery(username, options, query, handleAffectedTablesInQueryRows, callback);
-};
-
-function handleAffectedTablesInQueryRows(err, rows, callback) {
-    if (err){
-        var msg = err.message ? err.message : err;
-        callback(new Error('could not fetch source tables: ' + msg));
-        return;
-    }
-    var qtables = rows[0].cdb_querytables;
-    var tableNames = qtables.split(/^\{(.*)\}$/)[1];
-    tableNames = tableNames ? tableNames.split(',') : [];
-    callback(null, tableNames);
-}
-
-QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (username, options, sql, callback) {
-
-    var query = [
-        'WITH querytables AS (',
-            'SELECT * FROM CDB_QueryTables($windshaft$' + prepareSql(sql) + '$windshaft$) as tablenames',
-        ')',
-        'SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max',
-        'FROM CDB_TableMetadata m',
-        'WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])'
-    ].join(' ');
-
-    runQuery(username, options, query, handleAffectedTablesAndLastUpdatedTimeRows, callback);
-};
-
-function handleAffectedTablesAndLastUpdatedTimeRows(err, rows, callback) {
-    if (err || rows.length === 0) {
-        var msg = err.message ? err.message : err;
-        callback(new Error('could not fetch affected tables and last updated time: ' + msg));
-        return;
-    }
-
-    var result = rows[0];
-
-    var tableNames = result.tablenames.split(/^\{(.*)\}$/)[1];
-    tableNames = tableNames ? tableNames.split(',') : [];
-
-    var lastUpdatedTime = result.max || 0;
-
-    callback(null, {
-        affectedTables: tableNames,
-        lastUpdatedTime: lastUpdatedTime * 1000
-    });
-}
-
-
-function runQuery(username, options, query, queryHandler, callback) {
-    if (shouldQueryPostgresDirectly()) {
-        var psql = new PSQL(options);
-        psql.query(query, function(err, resultSet) {
-            resultSet = resultSet || {};
-            var rows = resultSet.rows || [];
-            queryHandler(err, rows, callback);
-        });
-    } else {
-        sqlApi.query(username, options.api_key, query, function(err, rows) {
-            queryHandler(err, rows, callback);
-        });
-    }
-}
-
-
-function prepareSql(sql) {
-    return sql
-        .replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)')
-        .replace(affectedTableRegexCache.scale_denominator, '0')
-        .replace(affectedTableRegexCache.pixel_width, '1')
-        .replace(affectedTableRegexCache.pixel_height, '1')
-    ;
-}
-
-
-function shouldQueryPostgresDirectly() {
-    return global.environment
-        && global.environment.enabledFeatures
-        && global.environment.enabledFeatures.cdbQueryTablesFromPostgres;
-}

lib/cartodb/api/tables_extent_api.js

@@ -0,0 +1,47 @@
+function TablesExtentApi(pgQueryRunner) {
+    this.pgQueryRunner = pgQueryRunner;
+}
+
+module.exports = TablesExtentApi;
+
+/**
+ * Given a username and a list of tables it will return the estimated extent in SRID 4326 for all the tables based on
+ * the_geom_webmercator (SRID 3857) column.
+ *
+ * @param {String} username
+ * @param {Array} tableNames The named can be schema qualified, so this accepts both `schema_name.table_name` and
+ *  `table_name` format as valid input
+ * @param {Function} callback function(err, result) {Object} result with `west`, `south`, `east`, `north`
+ */
+TablesExtentApi.prototype.getBounds = function (username, tables, callback) {
+    var estimatedExtentSQLs = tables.map(function(table) {
+        return "ST_EstimatedExtent('" + table.schema_name + "', '" + table.table_name + "', 'the_geom_webmercator')";
+    });
+
+    var query = [
+        "WITH ext as (" +
+            "SELECT ST_Transform(ST_SetSRID(ST_Extent(ST_Union(ARRAY[",
+                estimatedExtentSQLs.join(','),
+            "])), 3857), 4326) geom)",
+        "SELECT",
+            "ST_XMin(geom) west,",
+            "ST_YMin(geom) south,",
+            "ST_XMax(geom) east,",
+            "ST_YMax(geom) north",
+        "FROM ext"
+    ].join(' ');
+
+    this.pgQueryRunner.run(username, query, function handleBoundsResult (err, rows) {
+        if (err) {
+            var msg = err.message ? err.message : err;
+            return callback(new Error('could not fetch source tables: ' + msg));
+        }
+        var result = null;
+        if (rows.length > 0) {
+            result = {
+                bounds: rows[0]
+            };
+        }
+        callback(null, result);
+    });
+};

lib/cartodb/api/user_limits_api.js

@@ -0,0 +1,28 @@
+/**
+ *
+ * @param metadataBackend
+ * @param options
+ * @constructor
+ * @type {UserLimitsApi}
+ */
+function UserLimitsApi(metadataBackend, options) {
+    this.metadataBackend = metadataBackend;
+    this.options = options || {};
+    this.options.limits = this.options.limits || {};
+}
+
+module.exports = UserLimitsApi;
+
+UserLimitsApi.prototype.getRenderLimits = function (username, callback) {
+    var self = this;
+    this.metadataBackend.getTilerRenderLimit(username, function handleTilerLimits(err, renderLimit) {
+        if (err) {
+            return callback(err);
+        }
+
+        return callback(null, {
+            cacheOnTimeout: self.options.limits.cacheOnTimeout || false,
+            render: renderLimit || self.options.limits.render || 0
+        });
+    });
+};

lib/cartodb/backends/analysis-status.js

@@ -0,0 +1,49 @@
+var PSQL = require('cartodb-psql');
+
+function AnalysisStatusBackend() {
+}
+
+module.exports = AnalysisStatusBackend;
+
+
+AnalysisStatusBackend.prototype.getNodeStatus = function (params, callback) {
+    var nodeId = params.nodeId;
+
+    var statusQuery = 'SELECT node_id, status, updated_at FROM cdb_analysis_catalog where node_id = \'' + nodeId + '\'';
+
+    var pg = new PSQL(dbParamsFromReqParams(params));
+    pg.query(statusQuery, function(err, result) {
+        if (err) {
+            return callback(err, result);
+        }
+
+        result = result || {};
+
+        var rows = result.rows || [];
+
+        return callback(null, rows[0] || {
+            node_id: nodeId,
+            status: 'unknown'
+        });
+    }, true); // use read-only transaction
+};
+
+function dbParamsFromReqParams(params) {
+    var dbParams = {};
+    if ( params.dbuser ) {
+        dbParams.user = params.dbuser;
+    }
+    if ( params.dbpassword ) {
+        dbParams.pass = params.dbpassword;
+    }
+    if ( params.dbhost ) {
+        dbParams.host = params.dbhost;
+    }
+    if ( params.dbport ) {
+        dbParams.port = params.dbport;
+    }
+    if ( params.dbname ) {
+        dbParams.dbname = params.dbname;
+    }
+    return dbParams;
+}

lib/cartodb/backends/analysis.js

@@ -0,0 +1,19 @@
+var camshaft = require('camshaft');
+
+function AnalysisBackend(options) {
+    var batchConfig = options.batch || {};
+    batchConfig.endpoint = batchConfig.endpoint || 'http://127.0.0.1:8080/api/v1/sql/job';
+    batchConfig.inlineExecution = batchConfig.inlineExecution || false;
+    batchConfig.hostHeaderTemplate = batchConfig.hostHeaderTemplate || '{{=it.username}}.localhost.lan';
+    this.batchConfig = batchConfig;
+}
+
+module.exports = AnalysisBackend;
+
+AnalysisBackend.prototype.create = function(analysisConfiguration, analysisDefinition, callback) {
+    analysisConfiguration.batch.endpoint = this.batchConfig.endpoint;
+    analysisConfiguration.batch.inlineExecution = this.batchConfig.inlineExecution;
+    analysisConfiguration.batch.hostHeaderTemplate = this.batchConfig.hostHeaderTemplate;
+
+    camshaft.create(analysisConfiguration, analysisDefinition, callback);
+};

lib/cartodb/backends/dataview.js

@@ -0,0 +1,280 @@
+var assert = require('assert');
+
+var _ = require('underscore');
+var PSQL = require('cartodb-psql');
+var camshaft = require('camshaft');
+var step = require('step');
+
+var Timer = require('../stats/timer');
+
+var BBoxFilter = require('../models/filter/bbox');
+var DataviewFactory = require('../models/dataview/factory');
+
+function DataviewBackend(analysisBackend) {
+    this.analysisBackend = analysisBackend;
+}
+
+module.exports = DataviewBackend;
+
+
+DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, params, callback) {
+    var self = this;
+
+    var timer = new Timer();
+
+    var dataviewName = params.dataviewName;
+
+    var mapConfig;
+    var dataviewDefinition;
+    step(
+        function getMapConfig() {
+            mapConfigProvider.getMapConfig(this);
+        },
+        function _getDataviewDefinition(err, _mapConfig) {
+            assert.ifError(err);
+
+            mapConfig = _mapConfig;
+
+            var _dataviewDefinition = getDataviewDefinition(mapConfig.obj(), dataviewName);
+            if (!_dataviewDefinition) {
+                throw new Error("Dataview '" + dataviewName + "' does not exists");
+            }
+
+            dataviewDefinition = _dataviewDefinition;
+
+            return dataviewDefinition;
+        },
+        function loadAnalysis(err) {
+            assert.ifError(err);
+
+            var analysisConfiguration = {
+                db: {
+                    host: params.dbhost,
+                    port: params.dbport,
+                    dbname: params.dbname,
+                    user: params.dbuser,
+                    pass: params.dbpassword
+                },
+                batch: {
+                    username: user,
+                    apiKey: params.api_key
+                }
+            };
+
+            var sourceId = dataviewDefinition.source.id;
+            var analysisDefinition = getAnalysisDefinition(mapConfig.obj().analyses, sourceId);
+
+            var next = this;
+
+            self.analysisBackend.create(analysisConfiguration, analysisDefinition, function(err, analysis) {
+                if (err) {
+                    return next(err);
+                }
+
+                var sourceId2Node = {};
+                var rootNode = analysis.getRoot();
+                if (rootNode.params && rootNode.params.id) {
+                    sourceId2Node[rootNode.params.id] = rootNode;
+                }
+
+                analysis.getSortedNodes().forEach(function(node) {
+                    if (node.params && node.params.id) {
+                        sourceId2Node[node.params.id] = node;
+                    }
+                });
+
+                var node = sourceId2Node[sourceId];
+
+                if (!node) {
+                    return next(new Error('Analysis node not found for dataview'));
+                }
+
+                return next(null, node);
+            });
+        },
+        function runDataviewQuery(err, node) {
+            assert.ifError(err);
+
+            var pg = new PSQL(dbParamsFromReqParams(params));
+
+            var ownFilter = +params.own_filter;
+            ownFilter = !!ownFilter;
+
+            var query;
+            if (ownFilter) {
+                query = node.getQuery();
+            } else {
+                var applyFilters = {};
+                applyFilters[dataviewName] = false;
+                query = node.getQuery(applyFilters);
+            }
+
+            if (params.bbox) {
+                var bboxFilter = new BBoxFilter({column: 'the_geom', srid: 4326}, {bbox: params.bbox});
+                query = bboxFilter.sql(query);
+            }
+
+
+            var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins'),
+                function castNumbers(overrides, val, k) {
+                    overrides[k] = Number.isFinite(+val) ? +val : val;
+                    return overrides;
+                },
+                {ownFilter: ownFilter}
+            );
+
+            var dataview = DataviewFactory.getDataview(query, dataviewDefinition);
+            dataview.getResult(pg, overrideParams, this);
+        },
+        function returnCallback(err, result) {
+            return callback(err, result, timer.getTimes());
+        }
+    );
+};
+
+DataviewBackend.prototype.search = function (mapConfigProvider, user, params, callback) {
+    var self = this;
+
+    var timer = new Timer();
+
+    var dataviewName = params.dataviewName;
+
+    var mapConfig;
+    var dataviewDefinition;
+    step(
+        function getMapConfig() {
+            mapConfigProvider.getMapConfig(this);
+        },
+        function _getDataviewDefinition(err, _mapConfig) {
+            assert.ifError(err);
+
+            mapConfig = _mapConfig;
+
+            var _dataviewDefinition = getDataviewDefinition(mapConfig.obj(), dataviewName);
+            if (!_dataviewDefinition) {
+                throw new Error("Dataview '" + dataviewName + "' does not exists");
+            }
+
+            dataviewDefinition = _dataviewDefinition;
+
+            return dataviewDefinition;
+        },
+        function loadAnalysis(err) {
+            assert.ifError(err);
+
+            var analysisConfiguration = {
+                db: {
+                    host: params.dbhost,
+                    port: params.dbport,
+                    dbname: params.dbname,
+                    user: params.dbuser,
+                    pass: params.dbpassword
+                },
+                batch: {
+                    // TODO load this from configuration
+                    endpoint: 'http://127.0.0.1:8080/api/v1/sql/job',
+                    username: user,
+                    apiKey: params.api_key
+                }
+            };
+
+            var sourceId = dataviewDefinition.source.id;
+            var analysisDefinition = getAnalysisDefinition(mapConfig.obj().analyses, sourceId);
+
+            var next = this;
+
+            self.analysisBackend.create(analysisConfiguration, analysisDefinition, function(err, analysis) {
+                if (err) {
+                    return next(err);
+                }
+
+                var sourceId2Node = {};
+                var rootNode = analysis.getRoot();
+                if (rootNode.params && rootNode.params.id) {
+                    sourceId2Node[rootNode.params.id] = rootNode;
+                }
+
+                analysis.getSortedNodes().forEach(function(node) {
+                    if (node.params && node.params.id) {
+                        sourceId2Node[node.params.id] = node;
+                    }
+                });
+
+                var node = sourceId2Node[sourceId];
+
+                if (!node) {
+                    return next(new Error('Analysis node not found for dataview'));
+                }
+
+                return next(null, node);
+            });
+        },
+        function runDataviewQuery(err, node) {
+            assert.ifError(err);
+
+            var pg = new PSQL(dbParamsFromReqParams(params));
+
+            var ownFilter = +params.own_filter;
+            ownFilter = !!ownFilter;
+
+            var query;
+            if (ownFilter) {
+                query = node.getQuery();
+            } else {
+                var applyFilters = {};
+                applyFilters[dataviewName] = false;
+                query = node.getQuery(applyFilters);
+            }
+
+            if (params.bbox) {
+                var bboxFilter = new BBoxFilter({column: 'the_geom', srid: 4326}, {bbox: params.bbox});
+                query = bboxFilter.sql(query);
+            }
+
+            var userQuery = params.q;
+
+            var dataview = DataviewFactory.getDataview(query, dataviewDefinition);
+            dataview.search(pg, userQuery, this);
+        },
+        function returnCallback(err, result) {
+            return callback(err, result, timer.getTimes());
+        }
+    );
+};
+
+function getAnalysisDefinition(mapConfigAnalyses, sourceId) {
+    mapConfigAnalyses = mapConfigAnalyses || [];
+    for (var i = 0; i < mapConfigAnalyses.length; i++) {
+        var analysisGraph = new camshaft.reference.AnalysisGraph(mapConfigAnalyses[i]);
+        var nodes = analysisGraph.getNodesWithId();
+        if (nodes.hasOwnProperty(sourceId)) {
+            return mapConfigAnalyses[i];
+        }
+    }
+    throw new Error('There is no associated analysis for the dataview source id');
+}
+
+function getDataviewDefinition(mapConfig, dataviewName) {
+    var dataviews = mapConfig.dataviews || {};
+    return dataviews[dataviewName];
+}
+
+function dbParamsFromReqParams(params) {
+    var dbParams = {};
+    if ( params.dbuser ) {
+        dbParams.user = params.dbuser;
+    }
+    if ( params.dbpassword ) {
+        dbParams.pass = params.dbpassword;
+    }
+    if ( params.dbhost ) {
+        dbParams.host = params.dbhost;
+    }
+    if ( params.dbport ) {
+        dbParams.port = params.dbport;
+    }
+    if ( params.dbname ) {
+        dbParams.dbname = params.dbname;
+    }
+    return dbParams;
+}

lib/cartodb/backends/pg_connection.js

@@ -0,0 +1,136 @@
+var assert = require('assert');
+var step = require('step');
+var PSQL = require('cartodb-psql');
+var _ = require('underscore');
+
+function PgConnection(metadataBackend) {
+    this.metadataBackend = metadataBackend;
+}
+
+module.exports = PgConnection;
+
+
+// Set db authentication parameters to those of the given username
+//
+// @param username the cartodb username, mapped to a database username
+//                 via CartodbRedis metadata records
+//
+// @param params the parameters to set auth options into
+//               added params are: "dbuser" and "dbpassword"
+//
+// @param callback function(err)
+//
+PgConnection.prototype.setDBAuth = function(username, params, callback) {
+    var self = this;
+
+    var user_params = {};
+    var auth_user = global.environment.postgres_auth_user;
+    var auth_pass = global.environment.postgres_auth_pass;
+    step(
+        function getId() {
+            self.metadataBackend.getUserId(username, this);
+        },
+        function(err, user_id) {
+            assert.ifError(err);
+            user_params.user_id = user_id;
+            var dbuser = _.template(auth_user, user_params);
+            _.extend(params, {dbuser:dbuser});
+
+            // skip looking up user_password if postgres_auth_pass
+            // doesn't contain the "user_password" label
+            if (!auth_pass || ! auth_pass.match(/\buser_password\b/) ) {
+                return null;
+            }
+
+            self.metadataBackend.getUserDBPass(username, this);
+        },
+        function(err, user_password) {
+            assert.ifError(err);
+            user_params.user_password = user_password;
+            if ( auth_pass ) {
+                var dbpass = _.template(auth_pass, user_params);
+                _.extend(params, {dbpassword:dbpass});
+            }
+            return true;
+        },
+        function finish(err) {
+            callback(err);
+        }
+    );
+};
+
+// Set db connection parameters to those for the given username
+//
+// @param dbowner cartodb username of database owner,
+//                mapped to a database username
+//                via CartodbRedis metadata records
+//
+// @param params the parameters to set connection options into
+//               added params are: "dbname", "dbhost"
+//
+// @param callback function(err)
+//
+PgConnection.prototype.setDBConn = function(dbowner, params, callback) {
+    var self = this;
+    // Add default database connection parameters
+    // if none given
+    _.defaults(params, {
+        dbuser: global.environment.postgres.user,
+        dbpassword: global.environment.postgres.password,
+        dbhost: global.environment.postgres.host,
+        dbport: global.environment.postgres.port
+    });
+    step(
+        function getConnectionParams() {
+            self.metadataBackend.getUserDBConnectionParams(dbowner, this);
+        },
+        function extendParams(err, dbParams){
+            assert.ifError(err);
+            // we don't want null values or overwrite a non public user
+            if (params.dbuser !== 'publicuser' || !dbParams.dbuser) {
+                delete dbParams.dbuser;
+            }
+            if ( dbParams ) {
+                _.extend(params, dbParams);
+            }
+            return null;
+        },
+        function finish(err) {
+            callback(err);
+        }
+    );
+};
+
+
+/**
+ * Returns a `cartodb-psql` object for a given username.
+ * @param {String} username
+ * @param {Function} callback function({Error}, {PSQL})
+ */
+
+PgConnection.prototype.getConnection = function(username, callback) {
+    var self = this;
+
+    var params = {};
+
+    require('debug')('cachechan')("getConn1");
+    step(
+        function setAuth() {
+            self.setDBAuth(username, params, this);
+        },
+        function setConn(err) {
+            assert.ifError(err);
+            self.setDBConn(username, params, this);
+        },
+        function openConnection(err) {
+            assert.ifError(err);
+            return callback(err, new PSQL({
+                user: params.dbuser,
+                pass: params.dbpass,
+                host: params.dbhost,
+                port: params.dbport,
+                dbname: params.dbname
+            }));
+        }
+    );
+};

lib/cartodb/backends/pg_query_runner.js

@@ -0,0 +1,46 @@
+var assert = require('assert');
+var PSQL = require('cartodb-psql');
+var step = require('step');
+
+function PgQueryRunner(pgConnection) {
+    this.pgConnection = pgConnection;
+}
+
+module.exports = PgQueryRunner;
+
+/**
+ * Runs `query` with `username`'s PostgreSQL role, callback receives error and rows array.
+ *
+ * @param {String} username
+ * @param {String} query
+ * @param {Function} callback function({Error}, {Array}) second argument is guaranteed to be an array
+ */
+PgQueryRunner.prototype.run = function(username, query, callback) {
+    var self = this;
+
+    var params = {};
+
+    step(
+        function setAuth() {
+            self.pgConnection.setDBAuth(username, params, this);
+        },
+        function setConn(err) {
+            assert.ifError(err);
+            self.pgConnection.setDBConn(username, params, this);
+        },
+        function executeQuery(err) {
+            assert.ifError(err);
+            var psql = new PSQL({
+                user: params.dbuser,
+                pass: params.dbpass,
+                host: params.dbhost,
+                port: params.dbport,
+                dbname: params.dbname
+            });
+            psql.query(query, function(err, resultSet) {
+                resultSet = resultSet || {};
+                return callback(err, resultSet.rows || []);
+            });
+        }
+    );
+};

lib/cartodb/backends/template_maps.js

@@ -1,7 +1,9 @@
-var crypto  = require('crypto'),
-    Step    = require('step'),
-    _       = require('underscore'),
-    dot     = require('dot');
+var assert = require('assert');
+var crypto = require('crypto');
+var debug = require('debug')('windshaft:templates');
+var step = require('step');
+var _ = require('underscore');
+var dot = require('dot');
 
 
 var EventEmitter = require('events').EventEmitter;
@@ -19,9 +21,11 @@ var util = require('util');
 // @param opts TemplateMap options. Supported elements:
 //    'max_user_templates' limit on the number of per-user
 //
-// 
+//
 function TemplateMaps(redis_pool, opts) {
-  if (!(this instanceof TemplateMaps)) return new TemplateMaps();
+  if (!(this instanceof TemplateMaps)) {
+      return new TemplateMaps();
+  }
 
   EventEmitter.call(this);
 
@@ -37,18 +41,13 @@ function TemplateMaps(redis_pool, opts) {
   //
   // Map templates are owned by a user that specifies access permissions
   // for their instances.
-  // 
+  //
   // We have the following datastores:
   //
   //  1. User templates: set of per-user map templates
 
   // User templates (HASH:tpl_id->tpl_val)
   this.key_usr_tpl = dot.template("map_tpl|{{=it.owner}}");
-
-  // User template locks (HASH:tpl_id->ctime)
-  this.key_usr_tpl_lck = dot.template("map_tpl|{{=it.owner}}|locks");
-
-  this.lock_ttl = this.opts['lock_ttl'] || 5000;
 }
 
 util.inherits(TemplateMaps, EventEmitter);
@@ -61,7 +60,7 @@ var o = TemplateMaps.prototype;
 //--------------- PRIVATE METHODS --------------------------------
 
 o._userTemplateLimit = function() {
-  return this.opts['max_user_templates'] || 0;
+  return this.opts.max_user_templates || 0;
 };
 
 /**
@@ -76,33 +75,37 @@ o._redisCmd = function(redisFunc, redisArgs, callback) {
   var that = this;
   var db = that.db_signatures;
 
-  Step(
+  step(
     function getRedisClient() {
       that.redis_pool.acquire(db, this);
     },
     function executeQuery(err, data) {
-      if ( err ) throw err;
+      assert.ifError(err);
       redisClient = data;
       redisArgs.push(this);
       redisClient[redisFunc.toUpperCase()].apply(redisClient, redisArgs);
     },
     function releaseRedisClient(err, data) {
-      if ( ! _.isUndefined(redisClient) ) that.redis_pool.release(db, redisClient);
+      if ( ! _.isUndefined(redisClient) ) {
+          that.redis_pool.release(db, redisClient);
+      }
       callback(err, data);
     }
   );
 };
 
-var _reValidIdentifier = /^[a-zA-Z][0-9a-zA-Z_]*$/;
+var _reValidNameIdentifier = /^[a-z0-9][0-9a-z_\-]*$/i;
+var _reValidPlaceholderIdentifier = /^[a-z][0-9a-z_]*$/i;
+// jshint maxcomplexity:15
 o._checkInvalidTemplate = function(template) {
-  if ( template.version != '0.0.1' ) {
+  if ( template.version !== '0.0.1' ) {
     return new Error("Unsupported template version " + template.version);
   }
   var tplname = template.name;
   if ( ! tplname ) {
     return new Error("Missing template name");
   }
-  if ( ! tplname.match(_reValidIdentifier) ) {
+  if ( ! tplname.match(_reValidNameIdentifier) ) {
     return new Error("Invalid characters in template name '" + tplname + "'");
   }
 
@@ -117,7 +120,7 @@ o._checkInvalidTemplate = function(template) {
   for (var i = 0, len = placeholderKeys.length; i < len; i++) {
       var placeholderKey = placeholderKeys[i];
 
-      if (!placeholderKey.match(_reValidIdentifier)) {
+      if (!placeholderKey.match(_reValidPlaceholderIdentifier)) {
           return new Error("Invalid characters in placeholder name '" + placeholderKey + "'");
       }
       if ( ! placeholders[placeholderKey].hasOwnProperty('default') ) {
@@ -134,14 +137,15 @@ o._checkInvalidTemplate = function(template) {
         case 'open':
             break;
         case 'token':
-            if ( ! _.isArray(auth.valid_tokens) )
+            if ( ! _.isArray(auth.valid_tokens) ) {
                 return new Error("Invalid 'token' authentication: missing valid_tokens");
-            if ( ! auth.valid_tokens.length )
+            }
+            if ( ! auth.valid_tokens.length ) {
                 return new Error("Invalid 'token' authentication: no valid_tokens");
+            }
             break;
         default:
             return new Error("Unsupported authentication method: " + auth.method);
-            break;
     }
 
     return false;
@@ -193,7 +197,7 @@ function templateDefaults(template) {
 // @param template layergroup template, see
 //        http://github.com/CartoDB/Windshaft-cartodb/wiki/Template-maps#template-format
 //
-// @param callback function(err, tpl_id) 
+// @param callback function(err, tpl_id)
 //        Return template identifier (only valid for given user)
 //
 o.addTemplate = function(owner, template, callback) {
@@ -210,7 +214,7 @@ o.addTemplate = function(owner, template, callback) {
     var userTemplatesKey = this.key_usr_tpl({ owner:owner });
     var limit = this._userTemplateLimit();
 
-    Step(
+    step(
         function checkLimit() {
             if ( ! limit ) {
                 return 0;
@@ -218,19 +222,17 @@ o.addTemplate = function(owner, template, callback) {
             self._redisCmd('HLEN', [ userTemplatesKey ], this);
         },
         function installTemplateIfDoesNotExist(err, numberOfTemplates) {
-            if ( err ) {
-                throw err;
-            }
+            assert.ifError(err);
             if ( limit && numberOfTemplates >= limit ) {
-                throw new Error("User '" + owner + "' reached limit on number of templates " +
-                    "("+ numberOfTemplates + "/" + limit + ")");
+                var limitReachedError = new Error("User '" + owner + "' reached limit on number of templates (" +
+                    numberOfTemplates + "/" + limit + ")");
+                limitReachedError.http_status = 409;
+                throw limitReachedError;
             }
             self._redisCmd('HSETNX', [ userTemplatesKey, templateName, JSON.stringify(template) ], this);
         },
         function validateInstallation(err, wasSet) {
-            if ( err ) {
-                throw err;
-            }
+            assert.ifError(err);
             if ( ! wasSet ) {
                 throw new Error("Template '" + templateName + "' of user '" + owner + "' already exists");
             }
@@ -258,14 +260,12 @@ o.addTemplate = function(owner, template, callback) {
 //
 o.delTemplate = function(owner, tpl_id, callback) {
     var self = this;
-    Step(
+    step(
         function deleteTemplate() {
             self._redisCmd('HDEL', [ self.key_usr_tpl({ owner:owner }), tpl_id ], this);
         },
         function handleDeletion(err, deleted) {
-            if (err) {
-                throw err;
-            }
+            assert.ifError(err);
             if (!deleted) {
                 throw new Error("Template '" + tpl_id + "' of user '" + owner + "' does not exist");
             }
@@ -296,7 +296,7 @@ o.delTemplate = function(owner, tpl_id, callback) {
 //        http://github.com/CartoDB/Windshaft-cartodb/wiki/Template-maps#template-format
 //
 // @param callback function(err)
-//        
+//
 o.updTemplate = function(owner, tpl_id, template, callback) {
     var self = this;
 
@@ -310,37 +310,38 @@ o.updTemplate = function(owner, tpl_id, template, callback) {
 
     var templateName = template.name;
 
-    if ( tpl_id != templateName ) {
+    if ( tpl_id !== templateName ) {
         return callback(new Error("Cannot update name of a map template ('" + tpl_id + "' != '" + templateName + "')"));
     }
 
     var userTemplatesKey = this.key_usr_tpl({ owner:owner });
 
-    Step(
+    var previousTemplate = null;
+
+    step(
         function getExistingTemplate() {
             self._redisCmd('HGET', [ userTemplatesKey, tpl_id ], this);
         },
-        function updateTemplate(err, currentTemplate) {
-            if (err) {
-                throw err;
-            }
-            if (!currentTemplate) {
+        function updateTemplate(err, _currentTemplate) {
+            assert.ifError(err);
+            if (!_currentTemplate) {
                 throw new Error("Template '" + tpl_id + "' of user '" + owner + "' does not exist");
             }
+            previousTemplate = _currentTemplate;
             self._redisCmd('HSET', [ userTemplatesKey, templateName, JSON.stringify(template) ], this);
         },
         function handleTemplateUpdate(err, didSetNewField) {
-            if (err) {
-                throw err;
-            }
+            assert.ifError(err);
             if (didSetNewField) {
-                console.warn('New template created on update operation');
+                debug('New template created on update operation');
             }
             return true;
         },
         function finish(err) {
             if (!err) {
-                self.emit('update', owner, templateName, template);
+                if (self.fingerPrint(JSON.parse(previousTemplate)) !== self.fingerPrint(template)) {
+                    self.emit('update', owner, templateName, template);
+                }
             }
 
             callback(err, template);
@@ -354,7 +355,7 @@ o.updTemplate = function(owner, tpl_id, template, callback) {
 //
 // @param callback function(err, tpl_id_list)
 //        Returns a list of template identifiers
-// 
+//
 o.listTemplates = function(owner, callback) {
     this._redisCmd('HKEYS', [ this.key_usr_tpl({owner:owner}) ], callback);
 };
@@ -371,12 +372,12 @@ o.listTemplates = function(owner, callback) {
 //
 o.getTemplate = function(owner, tpl_id, callback) {
     var self = this;
-    Step(
+    step(
         function getTemplate() {
             self._redisCmd('HGET', [ self.key_usr_tpl({owner:owner}), tpl_id ], this);
         },
         function parseTemplate(err, tpl_val) {
-            if ( err ) throw err;
+            assert.ifError(err);
             return JSON.parse(tpl_val);
         },
         function finish(err, tpl) {
@@ -421,7 +422,7 @@ o.isAuthorized = function(template, authTokens) {
 //        Only the ones found in the template's placeholders object
 //        will be used, with missing ones taking default values.
 //
-// @returns a layergroup configuration 
+// @returns a layergroup configuration
 //
 // @throws Error on malformed template or parameter
 //
@@ -429,14 +430,14 @@ var _reNumber = /^([-+]?[\d\.]?\d+([eE][+-]?\d+)?)$/,
     _reCSSColorName = /^[a-zA-Z]+$/,
     _reCSSColorVal = /^#[0-9a-fA-F]{3,6}$/;
 
-_replaceVars = function(str, params) {
-  //return _.template(str, params); // lazy way, possibly dangerous 
+function _replaceVars (str, params) {
+  //return _.template(str, params); // lazy way, possibly dangerous
   // Construct regular expressions for each param
     Object.keys(params).forEach(function(k) {
         str = str.replace(new RegExp("<%=\\s*" + k + "\\s*%>", "g"), params[k]);
     });
     return str;
-};
+}
 o.instance = function(template, params) {
   var all_params = {};
   var phold = template.placeholders || {};
@@ -455,16 +456,14 @@ o.instance = function(template, params) {
     else if ( type === 'number' ) {
       // check it's a number
       if ( typeof(val) !== 'number' && ! val.match(_reNumber) ) {
-        throw new Error("Invalid number value for template parameter '"
-          + k + "': " + val);
+        throw new Error("Invalid number value for template parameter '" + k + "': " + val);
       }
     }
     else if ( type === 'css_color' ) {
       // check it only contains letters or
       // starts with # and only contains hexdigits
       if ( ! val.match(_reCSSColorName) && ! val.match(_reCSSColorVal) ) {
-        throw new Error("Invalid css_color value for template parameter '"
-          + k + "': " + val);
+        throw new Error("Invalid css_color value for template parameter '" + k + "': " + val);
       }
     }
     else {
@@ -478,8 +477,16 @@ o.instance = function(template, params) {
   var layergroup = JSON.parse(JSON.stringify(template.layergroup));
   for (var i=0; i<layergroup.layers.length; ++i) {
     var lyropt = layergroup.layers[i].options;
-    if ( lyropt.cartocss ) lyropt.cartocss = _replaceVars(lyropt.cartocss, all_params);
-    if ( lyropt.sql) lyropt.sql = _replaceVars(lyropt.sql, all_params);
+
+    if ( params.styles && params.styles[i] ) {
+        // dynamic styling for this layer
+        lyropt.cartocss = params.styles[i];
+    } else if ( lyropt.cartocss ) {
+        lyropt.cartocss = _replaceVars(lyropt.cartocss, all_params);
+    }
+    if ( lyropt.sql) {
+        lyropt.sql = _replaceVars(lyropt.sql, all_params);
+    }
     // Anything else ?
   }
 
@@ -499,3 +506,14 @@ o.fingerPrint = function(template) {
     .digest('hex')
   ;
 };
+
+module.exports.templateName = function templateName(templateId) {
+    var templateIdTokens = templateId.split('@');
+    var name = templateIdTokens[0];
+
+    if (templateIdTokens.length > 1) {
+        name = templateIdTokens[1];
+    }
+
+    return name;
+};

lib/cartodb/cache/backend/fastly.js

@@ -0,0 +1,16 @@
+var FastlyPurge = require('fastly-purge');
+
+function FastlyCacheBackend(apiKey, serviceId) {
+    this.serviceId = serviceId;
+    this.fastlyPurge = new FastlyPurge(apiKey, { softPurge: false });
+}
+
+module.exports = FastlyCacheBackend;
+
+/**
+ * @param cacheObject should respond to `key() -> String` method
+ * @param {Function} callback
+ */
+FastlyCacheBackend.prototype.invalidate = function(cacheObject, callback) {
+    this.fastlyPurge.key(this.serviceId, cacheObject.key(), callback);
+};

lib/cartodb/cache/backend/varnish_http.js

@@ -28,5 +28,3 @@ VarnishHttpCacheBackend.prototype.invalidate = function(cacheObject, callback) {
         }
     );
 };
-
-module.exports = VarnishHttpCacheBackend;

lib/cartodb/cache/layergroup_affected_tables.js

@@ -0,0 +1,24 @@
+var LruCache = require('lru-cache');
+
+function LayergroupAffectedTables() {
+    // dbname + layergroupId -> affected tables cache
+    this.cache = new LruCache({ max: 2000 });
+}
+
+module.exports = LayergroupAffectedTables;
+
+LayergroupAffectedTables.prototype.hasAffectedTables = function(dbName, layergroupId) {
+    return this.cache.has(createKey(dbName, layergroupId));
+};
+
+LayergroupAffectedTables.prototype.set = function(dbName, layergroupId, affectedTables) {
+    this.cache.set(createKey(dbName, layergroupId), affectedTables);
+};
+
+LayergroupAffectedTables.prototype.get = function(dbName, layergroupId) {
+    return this.cache.get(createKey(dbName, layergroupId));
+};
+
+function createKey(dbName, layergroupId) {
+    return dbName + ':' + layergroupId;
+}

lib/cartodb/cache/named_map_provider_cache.js

@@ -0,0 +1,96 @@
+var _ = require('underscore');
+var dot = require('dot');
+var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider');
+var MapConfigNamedLayersAdapter = require('../models/mapconfig_named_layers_adapter');
+var AnalysisMapConfigAdapter = require('../models/analysis-mapconfig-adapter');
+var templateName = require('../backends/template_maps').templateName;
+var queue = require('queue-async');
+
+var LruCache = require("lru-cache");
+
+function NamedMapProviderCache(templateMaps, pgConnection, metadataBackend, userLimitsApi, overviewsAdapter,
+                               turboCartoAdapter) {
+    this.templateMaps = templateMaps;
+    this.pgConnection = pgConnection;
+    this.metadataBackend = metadataBackend;
+    this.userLimitsApi = userLimitsApi;
+
+    this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps);
+    this.analysisMapConfigAdapter = new AnalysisMapConfigAdapter();
+    this.overviewsAdapter = overviewsAdapter;
+    this.turboCartoAdapter = turboCartoAdapter;
+
+    this.providerCache = new LruCache({ max: 2000 });
+}
+
+module.exports = NamedMapProviderCache;
+
+NamedMapProviderCache.prototype.get = function(user, templateId, config, authToken, params, callback) {
+    var namedMapKey = createNamedMapKey(user, templateId);
+    var namedMapProviders = this.providerCache.get(namedMapKey) || {};
+
+    var providerKey = createProviderKey(config, authToken, params);
+    if (!namedMapProviders.hasOwnProperty(providerKey)) {
+        namedMapProviders[providerKey] = new NamedMapMapConfigProvider(
+            this.templateMaps,
+            this.pgConnection,
+            this.metadataBackend,
+            this.userLimitsApi,
+            this.namedLayersAdapter,
+            this.overviewsAdapter,
+            this.turboCartoAdapter,
+            this.analysisMapConfigAdapter,
+            user,
+            templateId,
+            config,
+            authToken,
+            params
+        );
+        this.providerCache.set(namedMapKey, namedMapProviders);
+
+        // early exit, if provider did not exist we just return it
+        return callback(null, namedMapProviders[providerKey]);
+    }
+
+    var namedMapProvider = namedMapProviders[providerKey];
+
+    var self = this;
+    queue(2)
+        .defer(namedMapProvider.getTemplate.bind(namedMapProvider))
+        .defer(this.templateMaps.getTemplate.bind(this.templateMaps), user, templateId)
+        .awaitAll(function templatesQueueDone(err, results) {
+            if (err) {
+                return callback(err);
+            }
+
+            // We want to reset provider its template has changed
+            // Ideally this should be done in a passive mode where this cache gets notified of template changes
+            var uniqueFingerprints = _.uniq(results.map(self.templateMaps.fingerPrint)).length;
+            if (uniqueFingerprints > 1) {
+                namedMapProvider.reset();
+            }
+            return callback(null, namedMapProvider);
+        });
+};
+
+NamedMapProviderCache.prototype.invalidate = function(user, templateId) {
+    this.providerCache.del(createNamedMapKey(user, templateId));
+};
+
+function createNamedMapKey(user, templateId) {
+    return user + ':' + templateName(templateId);
+}
+
+var providerKey = '{{=it.authToken}}:{{=it.configHash}}:{{=it.format}}:{{=it.layer}}:{{=it.scale_factor}}';
+var providerKeyTpl = dot.template(providerKey);
+
+function createProviderKey(config, authToken, params) {
+    var tplValues = _.defaults({}, params, {
+        authToken: authToken || '',
+        configHash: NamedMapMapConfigProvider.configHash(config),
+        layer: '',
+        format: '',
+        scale_factor: 1
+    });
+    return providerKeyTpl(tplValues);
+}

lib/cartodb/cache/surrogate_keys_cache.js

@@ -1,9 +1,11 @@
+var queue = require('queue-async');
+
 /**
- * @param cacheBackend should respond to `invalidate(cacheObject, callback)` method
+ * @param {Array|Object} cacheBackends each backend backend should respond to `invalidate(cacheObject, callback)` method
  * @constructor
  */
-function SurrogateKeysCache(cacheBackend) {
-    this.cacheBackend = cacheBackend;
+function SurrogateKeysCache(cacheBackends) {
+    this.cacheBackends = Array.isArray(cacheBackends) ? cacheBackends : [cacheBackends];
 }
 
 module.exports = SurrogateKeysCache;
@@ -14,13 +16,38 @@ module.exports = SurrogateKeysCache;
  * @param cacheObject should respond to `key() -> String` method
  */
 SurrogateKeysCache.prototype.tag = function(response, cacheObject) {
-    response.header('Surrogate-Key', cacheObject.key());
+    var newKey = cacheObject.key();
+    response.set('Surrogate-Key', appendSurrogateKey(
+        response.get('Surrogate-Key'),
+        Array.isArray(newKey) ? cacheObject.key().join(' ') : newKey
+    ));
+
 };
 
+function appendSurrogateKey(currentKey, newKey) {
+    if (!!currentKey) {
+        newKey = currentKey + ' ' + newKey;
+    }
+    return newKey;
+}
+
 /**
  * @param cacheObject should respond to `key() -> String` method
  * @param {Function} callback
  */
 SurrogateKeysCache.prototype.invalidate = function(cacheObject, callback) {
-    this.cacheBackend.invalidate(cacheObject, callback);
+    var invalidationQueue = queue(this.cacheBackends.length);
+
+    this.cacheBackends.forEach(function(cacheBackend) {
+        invalidationQueue.defer(function(cacheBackend, done) {
+            cacheBackend.invalidate(cacheObject, done);
+        }, cacheBackend);
+    });
+
+    invalidationQueue.awaitAll(function(err, result) {
+        if (err) {
+            return callback(err);
+        }
+        callback(null, result);
+    });
 };

lib/cartodb/cache_validator.js

@@ -1,26 +0,0 @@
-var _             = require('underscore'),
-    Varnish       = require('node-varnish'),
-    varnish_queue = null;
-
-function init(host, port, secret) {
-    varnish_queue = new Varnish.VarnishQueue(host, port, secret);
-    varnish_queue.on('error', function(e) {
-        console.log("[CACHE VALIDATOR ERROR] " + e);
-    });
-}
-
-function invalidate_db(dbname, table) {
-    var cmd = 'purge obj.http.X-Cache-Channel ~ "^' + dbname +
-              ':(.*'+ table +'.*)|(table)$"';
-    try{
-        varnish_queue.run_cmd(cmd, false);
-    } catch (e) {
-        console.log("[CACHE VALIDATOR ERROR] could not queue command " +
-                    cmd + " -- " + e);
-    }
-}
-
-module.exports = {
-    init: init,
-    invalidate_db: invalidate_db
-}

lib/cartodb/cartodb_windshaft.js

@@ -1,262 +0,0 @@
-var _ = require('underscore')
-    , Step       = require('step')
-    , Windshaft = require('windshaft')
-    , TemplateMaps = require('./template_maps.js')
-    , Cache = require('./cache_validator')
-    , os = require('os')
-    , HealthCheck = require('./monitoring/health_check')
-;
-
-if ( ! process.env['PGAPPNAME'] )
-  process.env['PGAPPNAME']='cartodb_tiler';
-
-var CartodbWindshaft = function(serverOptions) {
-    // Perform keyword substitution in statsd
-    // See https://github.com/CartoDB/Windshaft-cartodb/issues/153
-    if ( global.environment.statsd ) {
-      if ( global.environment.statsd.prefix ) {
-        var host_token = os.hostname().split('.').reverse().join('.');
-        global.environment.statsd.prefix = global.environment.statsd.prefix.replace(/:host/, host_token);
-      }
-    }
-
-    var redisPool = serverOptions.redis.pool
-        || require('redis-mpool')(_.extend(global.environment.redis, {name: 'windshaft:cartodb'}));
-
-    var cartoData = require('cartodb-redis')({pool: redisPool});
-
-    if(serverOptions.cache_enabled) {
-        console.log("cache invalidation enabled, varnish on ", serverOptions.varnish_host, ' ', serverOptions.varnish_port);
-        Cache.init(serverOptions.varnish_host, serverOptions.varnish_port, serverOptions.varnish_secret);
-        serverOptions.afterStateChange = function(req, data, callback) {
-            Cache.invalidate_db(req.params.dbname, req.params.table);
-            callback(null, data);
-        }
-    }
-
-    serverOptions.beforeStateChange = function(req, callback) {
-        var err = null;
-        if ( ! req.params.hasOwnProperty('_authorizedByApiKey') ) {
-          err = new Error("map state cannot be changed by unauthenticated request!");
-        }
-        callback(err, req);
-    };
-
-    // This is for Templated maps
-    //
-    // "named" is the official, "template" is for backward compatibility up to 1.6.x
-    //
-    var template_baseurl = global.environment.base_url_templated || '(?:/maps/named|/tiles/template)';
-
-    var templateMapsOpts = {
-      max_user_templates: global.environment.maxUserTemplates
-    };
-    var templateMaps = new TemplateMaps(redisPool, templateMapsOpts);
-    serverOptions.templateMaps = templateMaps;
-
-    var SurrogateKeysCache = require('./cache/surrogate_keys_cache'),
-        NamedMapsCacheEntry = require('./cache/model/named_maps_entry'),
-        VarnishHttpCacheBackend = require('./cache/backend/varnish_http'),
-        varnishHttpCacheBackend = new VarnishHttpCacheBackend(serverOptions.varnish_host, serverOptions.varnish_http_port),
-        surrogateKeysCache = new SurrogateKeysCache(varnishHttpCacheBackend);
-
-    if (serverOptions.varnish_purge_enabled) {
-        function invalidateNamedMap(owner, templateName) {
-            surrogateKeysCache.invalidate(new NamedMapsCacheEntry(owner, templateName), function(err) {
-                if (err) {
-                    console.warn('Cache: surrogate key invalidation failed');
-                }
-            });
-        }
-
-        ['update', 'delete'].forEach(function(eventType) {
-            templateMaps.on(eventType, invalidateNamedMap);
-        });
-    }
-
-    // boot
-    var ws = new Windshaft.Server(serverOptions);
-
-    // Override getVersion to include cartodb-specific versions
-    var wsversion = ws.getVersion;
-    ws.getVersion = function() {
-      var version = wsversion();
-      version.windshaft_cartodb = require('../../package.json').version;
-      return version;
-    };
-
-    var ws_sendResponse = ws.sendResponse;
-    // GET routes for which we don't want to request any caching.
-    // POST/PUT/DELETE requests are never cached anyway.
-    var noCacheGETRoutes = [
-      '/',
-      // See https://github.com/CartoDB/Windshaft-cartodb/issues/176
-      serverOptions.base_url_mapconfig,
-      template_baseurl + '/:template_id/jsonp'
-    ];
-    ws.sendResponse = function(res, args) {
-      var that = this;
-      var thatArgs = arguments;
-      var statusCode;
-      if ( res._windshaftStatusCode ) {
-        // Added by our override of sendError
-        statusCode = res._windshaftStatusCode;
-      } else {
-        if ( args.length > 2 ) statusCode = args[2];
-        else {
-          statusCode = args[1] || 200;
-        }
-      }
-      var req = res.req;
-      Step (
-        function addCacheChannel() {
-          if ( ! req ) {
-            // having no associated request can happen when
-            // using fake response objects for testing layergroup
-            // creation
-            return false;
-          }
-          if ( ! req.params ) {
-            // service requests (/version, /) 
-            // have no need for an X-Cache-Channel
-            return false;
-          }
-          if ( statusCode != 200 ) {
-            // We do not want to cache
-            // unsuccessful responses
-            return false;
-          }
-          if ( _.contains(noCacheGETRoutes, req.route.path) ) {
-//console.log("Skipping cache channel in route:\n" + req.route.path);
-            return false;
-          }
-//console.log("Adding cache channel to route\n" + req.route.path + " not matching any in:\n" + mapCreateRoutes.join("\n"));
-          serverOptions.addCacheChannel(that, req, this);
-        },
-        function sendResponse(err, added) {
-          if ( err ) console.log(err + err.stack);
-          ws_sendResponse.apply(that, thatArgs);
-          return null;
-        },
-        function finish(err) {
-          if ( err ) console.log(err + err.stack);
-        }
-      );
-    };
-
-    var ws_sendError = ws.sendError;
-    ws.sendError = function() {
-      var res = arguments[0];
-      var statusCode = arguments[2];
-      res._windshaftStatusCode = statusCode;
-      ws_sendError.apply(this, arguments);
-    };
-
-    /*******************************************************************************************************************
-     * Routing
-     ******************************************************************************************************************/
-
-    var TemplateMapsController = require('./controllers/template_maps'),
-        templateMapsController = new TemplateMapsController(
-            ws, serverOptions, templateMaps, cartoData, template_baseurl, surrogateKeysCache, NamedMapsCacheEntry
-        );
-    templateMapsController.register(ws);
-
-    /*******************************************************************************************************************
-     * END Routing
-     ******************************************************************************************************************/
-
-    /**
-     * Helper to allow access to the layer to be used in the maps infowindow popup.
-     */
-    ws.get(serverOptions.base_url + '/infowindow', function(req, res){
-        ws.doCORS(res);
-        Step(
-            function(){
-                serverOptions.getInfowindow(req, this);
-            },
-            function(err, data){
-                if (err){
-                    ws.sendError(res, {error: err.message}, 500, 'GET INFOWINDOW', err);
-                    //ws.sendResponse(res, [{error: err.message}, 500]);
-                } else {
-                    ws.sendResponse(res, [{infowindow: data}, 200]);
-                }
-            }
-        );
-    });
-
-
-    /**
-     * Helper to allow access to metadata to be used in embedded maps.
-     */
-    ws.get(serverOptions.base_url + '/map_metadata', function(req, res){
-        ws.doCORS(res);
-        Step(
-            function(){
-                serverOptions.getMapMetadata(req, this);
-            },
-            function(err, data){
-                if (err){
-                    ws.sendError(res, {error: err.message}, 500, 'GET MAP_METADATA', err);
-                    //ws.sendResponse(res, [err.message, 500]);
-                } else {
-                    ws.sendResponse(res, [{map_metadata: data}, 200]);
-                }
-            }
-        );
-    });
-
-    /**
-     * Helper API to allow per table tile cache (and sql cache) to be invalidated remotely.
-     * TODO: Move?
-     */
-    ws.del(serverOptions.base_url + '/flush_cache', function(req, res){
-        if ( req.profiler && req.profiler.statsd_client ) {
-          req.profiler.start('windshaft-cartodb.flush_cache');
-        }
-        ws.doCORS(res);
-        Step(
-            function flushCache(){
-                serverOptions.flushCache(req, serverOptions.cache_enabled ? Cache : null, this);
-            },
-            function sendResponse(err, data){
-                if (err){
-                    ws.sendError(res, {error: err.message}, 500, 'DELETE CACHE', err);
-                    //ws.sendResponse(res, [500]);
-                } else {
-                    ws.sendResponse(res, [{status: 'ok'}, 200]);
-                }
-            }
-        );
-    });
-
-    var healthCheck = new HealthCheck(cartoData, Windshaft.tilelive);
-    ws.get('/health', function(req, res) {
-        var healthConfig = global.environment.health || {};
-
-        if (!!healthConfig.enabled) {
-            var startTime = Date.now();
-            healthCheck.check(healthConfig, function(err, result) {
-                var ok = !err;
-                var response = {
-                    enabled: true,
-                    ok: ok,
-                    elapsed: Date.now() - startTime,
-                    result: result
-                };
-                if (err) {
-                    response.err = err.message;
-                }
-                res.send(response, ok ? 200 : 503);
-
-            });
-        } else {
-            res.send({enabled: false, ok: true}, 200);
-        }
-    });
-
-    return ws;
-};
-
-module.exports = CartodbWindshaft;

lib/cartodb/controllers/base.js

@@ -0,0 +1,266 @@
+var assert = require('assert');
+
+var _ = require('underscore');
+var step = require('step');
+var debug = require('debug')('windshaft:cartodb');
+
+var LZMA = require('lzma').LZMA;
+var lzmaWorker = new LZMA();
+
+// Whitelist query parameters and attach format
+var REQUEST_QUERY_PARAMS_WHITELIST = [
+    'config',
+    'map_key',
+    'api_key',
+    'auth_token',
+    'callback',
+    // widgets & filters
+    'filters', // json
+    'own_filter', // 0, 1
+    'bbox', // w,s,e,n
+    'bins', // number
+    'start', // number
+    'end', // number
+    'column_type', // string
+    // widgets search
+    'q'
+];
+
+function BaseController(authApi, pgConnection) {
+    this.authApi = authApi;
+    this.pgConnection = pgConnection;
+}
+
+module.exports = BaseController;
+
+// jshint maxcomplexity:9
+/**
+ * Whitelist input and get database name & default geometry type from
+ * subdomain/user metadata held in CartoDB Redis
+ * @param req - standard express request obj. Should have host & table
+ * @param callback
+ */
+BaseController.prototype.req2params = function(req, callback){
+    var self = this;
+
+    if ( req.query.lzma ) {
+
+        // Decode (from base64)
+        var lzma = new Buffer(req.query.lzma, 'base64')
+            .toString('binary')
+            .split('')
+            .map(function(c) {
+                return c.charCodeAt(0) - 128;
+            });
+
+
+        // Decompress
+        lzmaWorker.decompress(
+            lzma,
+            function(result) {
+                if (req.profiler) {
+                    req.profiler.done('lzma');
+                }
+                try {
+                    delete req.query.lzma;
+                    _.extend(req.query, JSON.parse(result));
+                    self.req2params(req, callback);
+                } catch (err) {
+                    req.profiler.done('req2params');
+                    callback(new Error('Error parsing lzma as JSON: ' + err));
+                }
+            }
+        );
+        return;
+    }
+
+    req.query = _.pick(req.query, REQUEST_QUERY_PARAMS_WHITELIST);
+    req.params = _.extend({}, req.params); // shuffle things as request is a strange array/object
+
+    var user = req.context.user;
+
+    if ( req.params.token ) {
+        // Token might match the following patterns:
+        // - {user}@{tpl_id}@{token}:{cache_buster}
+        var tksplit = req.params.token.split(':');
+        req.params.token = tksplit[0];
+        if ( tksplit.length > 1 ) {
+            req.params.cache_buster= tksplit[1];
+        }
+        tksplit = req.params.token.split('@');
+        if ( tksplit.length > 1 ) {
+            req.params.signer = tksplit.shift();
+            if ( ! req.params.signer ) {
+                req.params.signer = user;
+            }
+            else if ( req.params.signer !== user ) {
+                var err = new Error(
+                        'Cannot use map signature of user "' + req.params.signer + '" on db of user "' + user + '"'
+                );
+                err.http_status = 403;
+                req.profiler.done('req2params');
+                callback(err);
+                return;
+            }
+            if ( tksplit.length > 1 ) {
+                /*var template_hash = */tksplit.shift(); // unused
+            }
+            req.params.token = tksplit.shift();
+        }
+    }
+
+    // bring all query values onto req.params object
+    _.extend(req.params, req.query);
+
+    if (req.profiler) {
+        req.profiler.done('req2params.setup');
+    }
+
+    step(
+        function getPrivacy(){
+            self.authApi.authorize(req, this);
+        },
+        function validateAuthorization(err, authorized) {
+            if (req.profiler) {
+                req.profiler.done('authorize');
+            }
+            assert.ifError(err);
+            if(!authorized) {
+                err = new Error("Sorry, you are unauthorized (permission denied)");
+                err.http_status = 403;
+                throw err;
+            }
+            return null;
+        },
+        function getDatabase(err){
+            assert.ifError(err);
+            self.pgConnection.setDBConn(user, req.params, this);
+        },
+        function finishSetup(err) {
+            if ( err ) {
+                req.profiler.done('req2params');
+                return callback(err, req);
+            }
+
+            // Add default database connection parameters
+            // if none given
+            _.defaults(req.params, {
+                dbuser: global.environment.postgres.user,
+                dbpassword: global.environment.postgres.password,
+                dbhost: global.environment.postgres.host,
+                dbport: global.environment.postgres.port
+            });
+
+            req.profiler.done('req2params');
+            callback(null, req);
+        }
+    );
+};
+// jshint maxcomplexity:6
+
+// jshint maxcomplexity:9
+BaseController.prototype.send = function(req, res, body, status, headers) {
+    if (req.params.dbhost) {
+        res.set('X-Served-By-DB-Host', req.params.dbhost);
+    }
+
+    if (req.profiler) {
+        res.set('X-Tiler-Profiler', req.profiler.toJSONString());
+    }
+
+    if (headers) {
+        res.set(headers);
+    }
+
+    res.status(status);
+
+    if (!Buffer.isBuffer(body) && typeof body === 'object') {
+        if (req.query && req.query.callback) {
+            res.jsonp(body);
+        } else {
+            res.json(body);
+        }
+    } else {
+        res.send(body);
+    }
+
+    if (req.profiler) {
+        try {
+            // May throw due to dns, see
+            // See http://github.com/CartoDB/Windshaft/issues/166
+            req.profiler.sendStats();
+        } catch (err) {
+            debug("error sending profiling stats: " + err);
+        }
+    }
+};
+// jshint maxcomplexity:6
+
+BaseController.prototype.sendError = function(req, res, err, label) {
+    var allErrors = Array.isArray(err) ? err : [err];
+    label = label || 'UNKNOWN';
+    err = allErrors[0] || new Error(label);
+    allErrors[0] = err;
+
+    var statusCode = findStatusCode(err);
+
+    debug('[%s ERROR] -- %d: %s, %s', label, statusCode, err, err.stack);
+
+    // If a callback was requested, force status to 200
+    if (req.query && req.query.callback) {
+        statusCode = 200;
+    }
+
+    var errorResponseBody = { errors: allErrors.map(errorMessage) };
+
+    this.send(req, res, errorResponseBody, statusCode);
+};
+
+function errorMessage(err) {
+    // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
+    var message = (_.isString(err) ? err : err.message) || 'Unknown error';
+
+    // Strip connection info, if any
+    return message
+        // See https://github.com/CartoDB/Windshaft/issues/173
+        .replace(/Connection string: '[^']*'\n\s/im, '')
+        // See https://travis-ci.org/CartoDB/Windshaft/jobs/20703062#L1644
+        .replace(/is the server.*encountered/im, 'encountered');
+}
+module.exports.errorMessage = errorMessage;
+
+function findStatusCode(err) {
+    var statusCode;
+    if ( err.http_status ) {
+        statusCode = err.http_status;
+    } else {
+        statusCode = statusFromErrorMessage('' + err);
+    }
+    return statusCode;
+}
+module.exports.findStatusCode = findStatusCode;
+
+function statusFromErrorMessage(errMsg) {
+    // Find an appropriate statusCode based on message
+    // jshint maxcomplexity:7
+    var statusCode = 400;
+    if ( -1 !== errMsg.indexOf('permission denied') ) {
+        statusCode = 403;
+    }
+    else if ( -1 !== errMsg.indexOf('authentication failed') ) {
+        statusCode = 403;
+    }
+    else if (errMsg.match(/Postgis Plugin.*[\s|\n].*column.*does not exist/)) {
+        statusCode = 400;
+    }
+    else if ( -1 !== errMsg.indexOf('does not exist') ) {
+        if ( -1 !== errMsg.indexOf(' role ') ) {
+            statusCode = 403; // role 'xxx' does not exist
+        } else if ( errMsg.match(/function .* does not exist/) ) {
+            statusCode = 400; // invalid SQL (SQL function does not exist)
+        } else {
+            statusCode = 404;
+        }
+    }
+    return statusCode;
+}

lib/cartodb/controllers/index.js

@@ -0,0 +1,7 @@
+module.exports = {
+    Layergroup: require('./layergroup'),
+    Map: require('./map'),
+    NamedMaps: require('./named_maps'),
+    NamedMapsAdmin: require('./named_maps_admin'),
+    ServerInfo: require('./server_info')
+};

lib/cartodb/controllers/layergroup.js

@@ -0,0 +1,489 @@
+var assert = require('assert');
+var step = require('step');
+
+var util = require('util');
+var BaseController = require('./base');
+
+var cors = require('../middleware/cors');
+var userMiddleware = require('../middleware/user');
+
+var DataviewBackend = require('../backends/dataview');
+var AnalysisStatusBackend = require('../backends/analysis-status');
+
+var MapStoreMapConfigProvider = require('../models/mapconfig/map_store_provider');
+
+var QueryTables = require('cartodb-query-tables');
+
+/**
+ * @param {AuthApi} authApi
+ * @param {PgConnection} pgConnection
+ * @param {MapStore} mapStore
+ * @param {TileBackend} tileBackend
+ * @param {PreviewBackend} previewBackend
+ * @param {AttributesBackend} attributesBackend
+ * @param {WidgetBackend} widgetBackend
+ * @param {SurrogateKeysCache} surrogateKeysCache
+ * @param {UserLimitsApi} userLimitsApi
+ * @param {LayergroupAffectedTables} layergroupAffectedTables
+ * @param {AnalysisBackend} analysisBackend
+ * @constructor
+ */
+function LayergroupController(authApi, pgConnection, mapStore, tileBackend, previewBackend, attributesBackend,
+                          widgetBackend, surrogateKeysCache, userLimitsApi, layergroupAffectedTables, analysisBackend) {
+    BaseController.call(this, authApi, pgConnection);
+
+    this.pgConnection = pgConnection;
+    this.mapStore = mapStore;
+    this.tileBackend = tileBackend;
+    this.previewBackend = previewBackend;
+    this.attributesBackend = attributesBackend;
+    this.widgetBackend = widgetBackend;
+    this.surrogateKeysCache = surrogateKeysCache;
+    this.userLimitsApi = userLimitsApi;
+    this.layergroupAffectedTables = layergroupAffectedTables;
+
+    this.dataviewBackend = new DataviewBackend(analysisBackend);
+    this.analysisStatusBackend = new AnalysisStatusBackend();
+}
+
+util.inherits(LayergroupController, BaseController);
+
+module.exports = LayergroupController;
+
+
+LayergroupController.prototype.register = function(app) {
+    app.get(app.base_url_mapconfig +
+        '/:token/:z/:x/:y@:scale_factor?x.:format', cors(), userMiddleware,
+        this.tile.bind(this));
+
+    app.get(app.base_url_mapconfig +
+        '/:token/:z/:x/:y.:format', cors(), userMiddleware,
+        this.tile.bind(this));
+
+    app.get(app.base_url_mapconfig +
+        '/:token/:layer/:z/:x/:y.(:format)', cors(), userMiddleware,
+        this.layer.bind(this));
+
+    app.get(app.base_url_mapconfig +
+        '/:token/:layer/attributes/:fid', cors(), userMiddleware,
+        this.attributes.bind(this));
+
+    app.get(app.base_url_mapconfig +
+        '/static/center/:token/:z/:lat/:lng/:width/:height.:format', cors(), userMiddleware,
+        this.center.bind(this));
+
+    app.get(app.base_url_mapconfig +
+        '/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format', cors(), userMiddleware,
+        this.bbox.bind(this));
+
+    // Undocumented/non-supported API endpoint methods.
+    // Use at your own peril.
+    app.get(app.base_url_mapconfig +
+        '/:token/:layer/widget/:widgetName', cors(), userMiddleware,
+        this.widget.bind(this));
+
+    app.get(app.base_url_mapconfig +
+            '/:token/:layer/widget/:widgetName/search', cors(), userMiddleware,
+        this.widgetSearch.bind(this));
+
+    app.get(app.base_url_mapconfig +
+            '/:token/dataview/:dataviewName', cors(), userMiddleware,
+        this.dataview.bind(this));
+
+    app.get(app.base_url_mapconfig +
+            '/:token/dataview/:dataviewName/search', cors(), userMiddleware,
+        this.dataviewSearch.bind(this));
+
+    app.get(app.base_url_mapconfig +
+        '/:token/analysis/node/:nodeId', cors(), userMiddleware,
+        this.analysisNodeStatus.bind(this));
+};
+
+LayergroupController.prototype.analysisNodeStatus = function(req, res) {
+    var self = this;
+
+    step(
+        function setupParams() {
+            self.req2params(req, this);
+        },
+        function retrieveNodeStatus(err) {
+            assert.ifError(err);
+            self.analysisStatusBackend.getNodeStatus(req.params, this);
+        },
+        function finish(err, nodeStatus, stats) {
+            req.profiler.add(stats || {});
+
+            if (err) {
+                self.sendError(req, res, err, 'GET NODE STATUS');
+            } else {
+                self.sendResponse(req, res, nodeStatus, 200, {
+                    'Cache-Control': 'public,max-age=5',
+                    'Last-Modified': new Date().toUTCString()
+                });
+            }
+        }
+    );
+};
+
+LayergroupController.prototype.dataview = function(req, res) {
+    var self = this;
+
+    step(
+        function setupParams() {
+            self.req2params(req, this);
+        },
+        function retrieveDataview(err) {
+            assert.ifError(err);
+
+            var mapConfigProvider = new MapStoreMapConfigProvider(
+                self.mapStore, req.context.user, self.userLimitsApi, req.params
+            );
+            self.dataviewBackend.getDataview(mapConfigProvider, req.context.user, req.params, this);
+        },
+        function finish(err, dataview, stats) {
+            req.profiler.add(stats || {});
+
+            if (err) {
+                self.sendError(req, res, err, 'GET DATAVIEW');
+            } else {
+                self.sendResponse(req, res, dataview, 200);
+            }
+        }
+    );
+
+};
+
+LayergroupController.prototype.dataviewSearch = function(req, res) {
+    var self = this;
+
+    step(
+        function setupParams() {
+            self.req2params(req, this);
+        },
+        function searchDataview(err) {
+            assert.ifError(err);
+
+            var mapConfigProvider = new MapStoreMapConfigProvider(
+                self.mapStore, req.context.user, self.userLimitsApi, req.params
+            );
+            self.dataviewBackend.search(mapConfigProvider, req.context.user, req.params, this);
+        },
+        function finish(err, searchResult, stats) {
+            req.profiler.add(stats || {});
+
+            if (err) {
+                self.sendError(req, res, err, 'GET DATAVIEW SEARCH');
+            } else {
+                self.sendResponse(req, res, searchResult, 200);
+            }
+        }
+    );
+
+};
+
+LayergroupController.prototype.widget = function(req, res) {
+    var self = this;
+
+    step(
+        function setupParams() {
+            self.req2params(req, this);
+        },
+        function retrieveList(err) {
+            assert.ifError(err);
+
+            var mapConfigProvider = new MapStoreMapConfigProvider(
+                self.mapStore, req.context.user, self.userLimitsApi, req.params
+            );
+            self.widgetBackend.getWidget(mapConfigProvider, req.params, this);
+        },
+        function finish(err, widget, stats) {
+            req.profiler.add(stats || {});
+
+            if (err) {
+                self.sendError(req, res, err, 'GET WIDGET');
+            } else {
+                self.sendResponse(req, res, widget, 200);
+            }
+        }
+    );
+
+};
+
+LayergroupController.prototype.widgetSearch = function(req, res) {
+    var self = this;
+
+    step(
+        function setupParams() {
+            self.req2params(req, this);
+        },
+        function retrieveList(err) {
+            assert.ifError(err);
+
+            var mapConfigProvider = new MapStoreMapConfigProvider(
+                self.mapStore, req.context.user, self.userLimitsApi, req.params
+            );
+            self.widgetBackend.search(mapConfigProvider, req.params, this);
+        },
+        function finish(err, searchResult, stats) {
+            req.profiler.add(stats || {});
+
+            if (err) {
+                self.sendError(req, res, err, 'GET WIDGET');
+            } else {
+                self.sendResponse(req, res, searchResult, 200);
+            }
+        }
+    );
+
+};
+
+LayergroupController.prototype.attributes = function(req, res) {
+    var self = this;
+
+    req.profiler.start('windshaft.maplayer_attribute');
+
+    step(
+        function setupParams() {
+            self.req2params(req, this);
+        },
+        function retrieveFeatureAttributes(err) {
+            assert.ifError(err);
+
+            var mapConfigProvider = new MapStoreMapConfigProvider(
+                self.mapStore, req.context.user, self.userLimitsApi, req.params
+            );
+            self.attributesBackend.getFeatureAttributes(mapConfigProvider, req.params, false, this);
+        },
+        function finish(err, tile, stats) {
+            req.profiler.add(stats || {});
+
+            if (err) {
+                self.sendError(req, res, err, 'GET ATTRIBUTES');
+            } else {
+                self.sendResponse(req, res, tile, 200);
+            }
+        }
+    );
+
+};
+
+// Gets a tile for a given token and set of tile ZXY coords. (OSM style)
+LayergroupController.prototype.tile = function(req, res) {
+    req.profiler.start('windshaft.map_tile');
+    this.tileOrLayer(req, res);
+};
+
+// Gets a tile for a given token, layer set of tile ZXY coords. (OSM style)
+LayergroupController.prototype.layer = function(req, res, next) {
+    if (req.params.token === 'static') {
+        return next();
+    }
+    req.profiler.start('windshaft.maplayer_tile');
+    this.tileOrLayer(req, res);
+};
+
+LayergroupController.prototype.tileOrLayer = function (req, res) {
+    var self = this;
+
+    step(
+        function mapController$prepareParams() {
+            self.req2params(req, this);
+        },
+        function mapController$getTileOrGrid(err) {
+            assert.ifError(err);
+            self.tileBackend.getTile(
+                new MapStoreMapConfigProvider(self.mapStore, req.context.user, self.userLimitsApi, req.params),
+                req.params, this
+            );
+        },
+        function mapController$finalize(err, tile, headers, stats) {
+            req.profiler.add(stats);
+            self.finalizeGetTileOrGrid(err, req, res, tile, headers);
+        }
+    );
+};
+
+// This function is meant for being called as the very last
+// step by all endpoints serving tiles or grids
+LayergroupController.prototype.finalizeGetTileOrGrid = function(err, req, res, tile, headers) {
+    var supportedFormats = {
+        grid_json: true,
+        json_torque: true,
+        torque_json: true,
+        png: true
+    };
+
+    var formatStat = 'invalid';
+    if (req.params.format) {
+        var format = req.params.format.replace('.', '_');
+        if (supportedFormats[format]) {
+            formatStat = format;
+        }
+    }
+
+    if (err) {
+        // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
+        var errMsg = err.message ? ( '' + err.message ) : ( '' + err );
+
+        // Rewrite mapnik parsing errors to start with layer number
+        var matches = errMsg.match("(.*) in style 'layer([0-9]+)'");
+        if (matches) {
+            errMsg = 'style'+matches[2]+': ' + matches[1];
+        }
+        err.message = errMsg;
+
+        this.sendError(req, res, err, 'TILE RENDER');
+        global.statsClient.increment('windshaft.tiles.error');
+        global.statsClient.increment('windshaft.tiles.' + formatStat + '.error');
+    } else {
+        this.sendResponse(req, res, tile, 200, headers);
+        global.statsClient.increment('windshaft.tiles.success');
+        global.statsClient.increment('windshaft.tiles.' + formatStat + '.success');
+    }
+};
+
+LayergroupController.prototype.bbox = function(req, res) {
+    this.staticMap(req, res, +req.params.width, +req.params.height, {
+        west: +req.params.west,
+        north: +req.params.north,
+        east: +req.params.east,
+        south: +req.params.south
+    });
+};
+
+LayergroupController.prototype.center = function(req, res) {
+    this.staticMap(req, res, +req.params.width, +req.params.height, +req.params.z, {
+        lng: +req.params.lng,
+        lat: +req.params.lat
+    });
+};
+
+LayergroupController.prototype.staticMap = function(req, res, width, height, zoom /* bounds */, center) {
+    var format = req.params.format === 'jpg' ? 'jpeg' : 'png';
+    req.params.layer = 'all';
+    req.params.format = 'png';
+
+    var self = this;
+
+    step(
+        function reqParams() {
+            self.req2params(req, this);
+        },
+        function getImage(err) {
+            assert.ifError(err);
+            if (center) {
+                self.previewBackend.getImage(
+                    new MapStoreMapConfigProvider(self.mapStore, req.context.user, self.userLimitsApi, req.params),
+                    format, width, height, zoom, center, this);
+            } else {
+                self.previewBackend.getImage(
+                    new MapStoreMapConfigProvider(self.mapStore, req.context.user, self.userLimitsApi, req.params),
+                    format, width, height, zoom /* bounds */, this);
+            }
+        },
+        function handleImage(err, image, headers, stats) {
+            req.profiler.done('render-' + format);
+            req.profiler.add(stats || {});
+
+            if (err) {
+                self.sendError(req, res, err, 'STATIC_MAP');
+            } else {
+                res.set('Content-Type', headers['Content-Type'] || 'image/' + format);
+                self.sendResponse(req, res, image, 200);
+            }
+        }
+    );
+};
+
+LayergroupController.prototype.sendResponse = function(req, res, body, status, headers) {
+    var self = this;
+
+    res.set('Cache-Control', 'public,max-age=31536000');
+
+    // Set Last-Modified header
+    var lastUpdated;
+    if (req.params.cache_buster) {
+        // Assuming cache_buster is a timestamp
+        lastUpdated = new Date(parseInt(req.params.cache_buster));
+    } else {
+        lastUpdated = new Date();
+    }
+    res.set('Last-Modified', lastUpdated.toUTCString());
+
+    var dbName = req.params.dbname;
+    step(
+        function getAffectedTables() {
+            self.getAffectedTables(req.context.user, dbName, req.params.token, this);
+        },
+        function sendResponse(err, affectedTables) {
+            req.profiler.done('affectedTables');
+            if (err) {
+                global.logger.warn('ERROR generating cache channel: ' + err);
+            }
+            if (!!affectedTables) {
+                res.set('X-Cache-Channel', affectedTables.getCacheChannel());
+                self.surrogateKeysCache.tag(res, affectedTables);
+            }
+            self.send(req, res, body, status, headers);
+        }
+    );
+
+};
+
+LayergroupController.prototype.getAffectedTables = function(user, dbName, layergroupId, callback) {
+
+    if (this.layergroupAffectedTables.hasAffectedTables(dbName, layergroupId)) {
+        return callback(null, this.layergroupAffectedTables.get(dbName, layergroupId));
+    }
+
+    var self = this;
+    step(
+        function extractSQL() {
+            step(
+                function loadFromStore() {
+                    self.mapStore.load(layergroupId, this);
+                },
+                function getSQL(err, mapConfig) {
+                    assert.ifError(err);
+
+                    var queries = mapConfig.getLayers()
+                        .map(function(lyr) {
+                            return lyr.options.sql;
+                        })
+                        .filter(function(sql) {
+                            return !!sql;
+                        });
+
+                    return queries.length ? queries.join(';') : null;
+                },
+                this
+            );
+        },
+        function findAffectedTables(err, sql) {
+            assert.ifError(err);
+
+            if ( ! sql ) {
+                throw new Error("this request doesn't need an X-Cache-Channel generated");
+            }
+
+            step(
+                function getConnection() {
+                    self.pgConnection.getConnection(user, this);
+                },
+                function getAffectedTables(err, connection) {
+                    assert.ifError(err);
+
+                    QueryTables.getAffectedTablesFromQuery(connection, sql, this);
+                },
+                this
+            );
+        },
+        function buildCacheChannel(err, tables) {
+            assert.ifError(err);
+            self.layergroupAffectedTables.set(dbName, layergroupId, tables);
+
+            return tables;
+        },
+        callback
+    );
+};

lib/cartodb/controllers/map.js

@@ -0,0 +1,462 @@
+var _ = require('underscore');
+var assert = require('assert');
+var step = require('step');
+var windshaft = require('windshaft');
+var QueryTables = require('cartodb-query-tables');
+
+var util = require('util');
+var BaseController = require('./base');
+
+var cors = require('../middleware/cors');
+var userMiddleware = require('../middleware/user');
+
+var MapConfig = windshaft.model.MapConfig;
+var Datasource = windshaft.model.Datasource;
+
+var NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
+
+var MapConfigNamedLayersAdapter = require('../models/mapconfig_named_layers_adapter');
+var AnalysisMapConfigAdapter = require('../models/analysis-mapconfig-adapter');
+var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider');
+var CreateLayergroupMapConfigProvider = require('../models/mapconfig/create_layergroup_provider');
+
+/**
+ * @param {AuthApi} authApi
+ * @param {PgConnection} pgConnection
+ * @param {TemplateMaps} templateMaps
+ * @param {MapBackend} mapBackend
+ * @param metadataBackend
+ * @param {SurrogateKeysCache} surrogateKeysCache
+ * @param {UserLimitsApi} userLimitsApi
+ * @param {LayergroupAffectedTables} layergroupAffectedTables
+ * @param {MapConfigOverviewsAdapter} overviewsAdapter
+ * @param {TurboCartoAdapter} turboCartoAdapter
+ * @param {AnalysisBackend} analysisBackend
+ * @constructor
+ */
+function MapController(authApi, pgConnection, templateMaps, mapBackend, metadataBackend,
+                       surrogateKeysCache, userLimitsApi, layergroupAffectedTables,
+                       overviewsAdapter, turboCartoAdapter, analysisBackend) {
+
+    BaseController.call(this, authApi, pgConnection);
+
+    this.pgConnection = pgConnection;
+    this.templateMaps = templateMaps;
+    this.mapBackend = mapBackend;
+    this.metadataBackend = metadataBackend;
+    this.surrogateKeysCache = surrogateKeysCache;
+    this.userLimitsApi = userLimitsApi;
+    this.layergroupAffectedTables = layergroupAffectedTables;
+    this.turboCartoAdapter = turboCartoAdapter;
+
+    this.analysisMapConfigAdapter = new AnalysisMapConfigAdapter(analysisBackend);
+    this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps);
+    this.overviewsAdapter = overviewsAdapter;
+}
+
+util.inherits(MapController, BaseController);
+
+module.exports = MapController;
+
+
+MapController.prototype.register = function(app) {
+    app.get(app.base_url_mapconfig, cors(), userMiddleware, this.createGet.bind(this));
+    app.post(app.base_url_mapconfig, cors(), userMiddleware, this.createPost.bind(this));
+    app.get(app.base_url_templated + '/:template_id/jsonp', cors(), userMiddleware, this.jsonp.bind(this));
+    app.post(app.base_url_templated + '/:template_id', cors(), userMiddleware, this.instantiate.bind(this));
+    app.options(app.base_url_mapconfig, cors('Content-Type'));
+};
+
+MapController.prototype.createGet = function(req, res){
+    req.profiler.start('windshaft.createmap_get');
+
+    this.create(req, res, function createGet$prepareConfig(err, req) {
+        assert.ifError(err);
+        if ( ! req.params.config ) {
+            throw new Error('layergroup GET needs a "config" parameter');
+        }
+        return JSON.parse(req.params.config);
+    });
+};
+
+MapController.prototype.createPost = function(req, res) {
+    req.profiler.start('windshaft.createmap_post');
+
+    this.create(req, res, function createPost$prepareConfig(err, req) {
+        assert.ifError(err);
+        if (!req.is('application/json')) {
+            throw new Error('layergroup POST data must be of type application/json');
+        }
+        return req.body;
+    });
+};
+
+MapController.prototype.instantiate = function(req, res) {
+    if (req.profiler) {
+        req.profiler.start('windshaft-cartodb.instance_template_post');
+    }
+
+    this.instantiateTemplate(req, res, function prepareTemplateParams(callback) {
+        if (!req.is('application/json')) {
+            return callback(new Error('Template POST data must be of type application/json'));
+        }
+        return callback(null, req.body);
+    });
+};
+
+MapController.prototype.jsonp = function(req, res) {
+    if (req.profiler) {
+        req.profiler.start('windshaft-cartodb.instance_template_get');
+    }
+
+    this.instantiateTemplate(req, res, function prepareJsonTemplateParams(callback) {
+        var err = null;
+        if ( req.query.callback === undefined || req.query.callback.length === 0) {
+            err = new Error('callback parameter should be present and be a function name');
+        }
+
+        var templateParams = {};
+        if (req.query.config) {
+            try {
+                templateParams = JSON.parse(req.query.config);
+            } catch(e) {
+                err = new Error('Invalid config parameter, should be a valid JSON');
+            }
+        }
+
+        return callback(err, templateParams);
+    });
+};
+
+MapController.prototype.create = function(req, res, prepareConfigFn) {
+    var self = this;
+
+    var mapConfig;
+    var analysesResults = [];
+
+    step(
+        function setupParams(){
+            self.req2params(req, this);
+        },
+        prepareConfigFn,
+        function prepareAnalysisLayers(err, requestMapConfig) {
+            assert.ifError(err);
+            var analysisConfiguration = {
+                db: {
+                    host: req.params.dbhost,
+                    port: req.params.dbport,
+                    dbname: req.params.dbname,
+                    user: req.params.dbuser,
+                    pass: req.params.dbpassword
+                },
+                batch: {
+                    username: req.context.user,
+                    apiKey: req.params.api_key
+                }
+            };
+
+            var filters = {};
+            if (req.params.filters) {
+                try {
+                    filters = JSON.parse(req.params.filters);
+                } catch (e) {
+                    // ignore
+                }
+            }
+            self.analysisMapConfigAdapter.getMapConfig(analysisConfiguration, requestMapConfig, filters, this);
+        },
+        function beforeLayergroupCreate(err, requestMapConfig, _analysesResults) {
+            assert.ifError(err);
+            var next = this;
+            analysesResults = _analysesResults;
+            self.namedLayersAdapter.getLayers(req.context.user, requestMapConfig.layers, self.pgConnection,
+                function(err, layers, datasource) {
+                    if (err) {
+                        return next(err);
+                    }
+
+                    if (layers) {
+                        requestMapConfig.layers = layers;
+                    }
+                    return next(null, requestMapConfig, datasource);
+                }
+            );
+        },
+        function addOverviewsInformation(err, requestMapConfig, datasource) {
+            assert.ifError(err);
+            var next = this;
+            self.overviewsAdapter.getLayers(req.context.user, requestMapConfig.layers, function(err, layers) {
+                if (err) {
+                    return next(err);
+                }
+
+                if (layers) {
+                    requestMapConfig.layers = layers;
+                }
+
+                return next(null, requestMapConfig, datasource);
+            });
+        },
+        function parseTurboCarto(err, requestMapConfig, datasource) {
+            assert.ifError(err);
+
+            var next = this;
+            self.turboCartoAdapter.getLayers(req.context.user, requestMapConfig.layers, function (err, layers) {
+                if (err) {
+                    return next(err);
+                }
+
+                if (layers) {
+                    requestMapConfig.layers = layers;
+                }
+
+                return next(null, requestMapConfig, datasource);
+            });
+        },
+        function createLayergroup(err, requestMapConfig, datasource) {
+            assert.ifError(err);
+            mapConfig = new MapConfig(requestMapConfig, datasource || Datasource.EmptyDatasource());
+            self.mapBackend.createLayergroup(
+                mapConfig, req.params,
+                new CreateLayergroupMapConfigProvider(mapConfig, req.context.user, self.userLimitsApi, req.params),
+                this
+            );
+        },
+        function afterLayergroupCreate(err, layergroup) {
+            assert.ifError(err);
+            self.afterLayergroupCreate(req, res, mapConfig, analysesResults, layergroup, this);
+        },
+        function finish(err, layergroup) {
+            if (err) {
+                self.sendError(req, res, err, 'ANONYMOUS LAYERGROUP');
+            } else {
+                addWidgetsUrl(req.context.user, layergroup);
+
+                res.set('X-Layergroup-Id', layergroup.layergroupid);
+                self.send(req, res, layergroup, 200);
+            }
+        }
+    );
+};
+
+MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn) {
+    var self = this;
+
+    var cdbuser = req.context.user;
+
+    var mapConfigProvider;
+    var mapConfig;
+
+    step(
+        function setupParams(){
+            self.req2params(req, this);
+        },
+        function getTemplateParams() {
+            prepareParamsFn(this);
+        },
+        function getTemplate(err, templateParams) {
+            assert.ifError(err);
+            mapConfigProvider = new NamedMapMapConfigProvider(
+                self.templateMaps,
+                self.pgConnection,
+                self.metadataBackend,
+                self.userLimitsApi,
+                self.namedLayersAdapter,
+                self.overviewsAdapter,
+                self.turboCartoAdapter,
+                self.analysisMapConfigAdapter,
+                cdbuser,
+                req.params.template_id,
+                templateParams,
+                req.query.auth_token,
+                req.params
+            );
+            mapConfigProvider.getMapConfig(this);
+        },
+        function createLayergroup(err, mapConfig_, rendererParams) {
+            assert.ifError(err);
+            mapConfig = mapConfig_;
+            self.mapBackend.createLayergroup(
+                mapConfig, rendererParams,
+                new CreateLayergroupMapConfigProvider(mapConfig, cdbuser, self.userLimitsApi, rendererParams),
+                this
+            );
+        },
+        function afterLayergroupCreate(err, layergroup) {
+            assert.ifError(err);
+            self.afterLayergroupCreate(req, res, mapConfig, [], layergroup, this);
+        },
+        function finishTemplateInstantiation(err, layergroup) {
+            if (err) {
+                self.sendError(req, res, err, 'NAMED MAP LAYERGROUP');
+            } else {
+                var templateHash = self.templateMaps.fingerPrint(mapConfigProvider.template).substring(0, 8);
+                layergroup.layergroupid = cdbuser + '@' + templateHash + '@' + layergroup.layergroupid;
+
+                addWidgetsUrl(cdbuser, layergroup);
+                addDataviewsUrls(cdbuser, layergroup, mapConfig.obj());
+                addAnalysesMetadata(cdbuser, layergroup, mapConfigProvider.analysesResults);
+
+                res.set('X-Layergroup-Id', layergroup.layergroupid);
+                self.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(cdbuser, mapConfigProvider.getTemplateName()));
+
+                self.send(req, res, layergroup, 200);
+            }
+        }
+    );
+};
+
+
+MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, analysesResults, layergroup, callback) {
+    var self = this;
+
+    var username = req.context.user;
+
+    var tasksleft = 2; // redis key and affectedTables
+    var errors = [];
+
+    var done = function(err) {
+        if ( err ) {
+            errors.push('' + err);
+        }
+        if ( ! --tasksleft ) {
+            err = errors.length ? new Error(errors.join('\n')) : null;
+            callback(err, layergroup);
+        }
+    };
+
+    // include in layergroup response the variables in serverMedata
+    // those variables are useful to send to the client information
+    // about how to reach this server or information about it
+    _.extend(layergroup, global.environment.serverMetadata);
+
+    // Don't wait for the mapview count increment to
+    // take place before proceeding. Error will be logged
+    // asynchronously
+    this.metadataBackend.incMapviewCount(username, mapconfig.obj().stat_tag, function(err) {
+        if (req.profiler) {
+            req.profiler.done('incMapviewCount');
+        }
+        if ( err ) {
+            global.logger.log("ERROR: failed to increment mapview count for user '" + username + "': " + err);
+        }
+        done();
+    });
+
+    var sql = mapconfig.getLayers().map(function(layer) {
+        return layer.options.sql;
+    }).join(';');
+
+    var dbName = req.params.dbname;
+    var layergroupId = layergroup.layergroupid;
+
+    step(
+        function getPgConnection() {
+            self.pgConnection.getConnection(username, this);
+        },
+        function getAffectedTablesAndLastUpdatedTime(err, connection) {
+            assert.ifError(err);
+            QueryTables.getAffectedTablesFromQuery(connection, sql, this);
+        },
+        function handleAffectedTablesAndLastUpdatedTime(err, result) {
+            if (req.profiler) {
+                req.profiler.done('queryTablesAndLastUpdated');
+            }
+            assert.ifError(err);
+            // feed affected tables cache so it can be reused from, for instance, layergroup controller
+            self.layergroupAffectedTables.set(dbName, layergroupId, result);
+
+            // last update for layergroup cache buster
+            layergroup.layergroupid = layergroup.layergroupid + ':' + result.getLastUpdatedAt();
+            layergroup.last_updated = new Date(result.getLastUpdatedAt()).toISOString();
+
+            // TODO this should take into account several URL patterns
+            addWidgetsUrl(username, layergroup);
+            addDataviewsUrls(username, layergroup, mapconfig.obj());
+            addAnalysesMetadata(username, layergroup, analysesResults, true);
+            if (req.method === 'GET') {
+                var ttl = global.environment.varnish.layergroupTtl || 86400;
+                res.set('Cache-Control', 'public,max-age='+ttl+',must-revalidate');
+                res.set('Last-Modified', (new Date()).toUTCString());
+                res.set('X-Cache-Channel', result.getCacheChannel());
+                if (result.tables && result.tables.length > 0) {
+                    self.surrogateKeysCache.tag(res, result);
+                }
+            }
+
+            return null;
+        },
+        function finish(err) {
+            done(err);
+        }
+    );
+};
+
+function addAnalysesMetadata(username, layergroup, analysesResults, includeQuery) {
+    includeQuery = includeQuery || false;
+    analysesResults = analysesResults || [];
+    layergroup.metadata.analyses = [];
+
+    analysesResults.forEach(function(analysis) {
+        var nodes = analysis.getSortedNodes();
+        layergroup.metadata.analyses.push({
+            nodes: nodes.reduce(function(nodesIdMap, node) {
+                if (node.params.id) {
+                    var nodeResource = layergroup.layergroupid + '/analysis/node/' + node.id();
+                    nodesIdMap[node.params.id] = {
+                        status: node.getStatus(),
+                        url: getUrls(username, nodeResource)
+                    };
+                    if (includeQuery) {
+                        nodesIdMap[node.params.id].query = node.getQuery();
+                    }
+                }
+
+                return nodesIdMap;
+            }, {})
+        });
+    });
+}
+
+function addDataviewsUrls(username, layergroup, mapConfig) {
+    layergroup.metadata.dataviews = layergroup.metadata.dataviews || {};
+    var dataviews = mapConfig.dataviews || {};
+
+    Object.keys(dataviews).forEach(function(dataviewName) {
+        var resource = layergroup.layergroupid + '/dataview/' + dataviewName;
+        layergroup.metadata.dataviews[dataviewName] = {
+            url: getUrls(username, resource)
+        };
+    });
+}
+
+function addWidgetsUrl(username, layergroup) {
+
+    if (layergroup.metadata && Array.isArray(layergroup.metadata.layers)) {
+        layergroup.metadata.layers = layergroup.metadata.layers.map(function(layer, layerIndex) {
+            if (layer.widgets) {
+                Object.keys(layer.widgets).forEach(function(widgetName) {
+                    var resource = layergroup.layergroupid + '/' + layerIndex + '/widget/' + widgetName;
+                    layer.widgets[widgetName].url = getUrls(username, resource);
+                });
+            }
+            return layer;
+        });
+    }
+
+}
+
+function getUrls(username, resource) {
+    var cdnUrl = global.environment.serverMetadata && global.environment.serverMetadata.cdn_url;
+    if (cdnUrl) {
+        return {
+            http: 'http://' + cdnUrl.http + '/' + username + '/api/v1/map/' + resource,
+            https: 'https://' + cdnUrl.https + '/' + username + '/api/v1/map/' + resource
+        };
+    } else {
+        var port = global.environment.port;
+        return {
+            http: 'http://' + username + '.' + 'localhost.lan:' + port +  '/api/v1/map/' + resource
+        };
+    }
+}

lib/cartodb/controllers/named_maps.js

@@ -0,0 +1,277 @@
+var step = require('step');
+var assert = require('assert');
+var _ = require('underscore');
+var NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
+
+var util = require('util');
+var BaseController = require('./base');
+
+var cors = require('../middleware/cors');
+var userMiddleware = require('../middleware/user');
+
+function NamedMapsController(authApi, pgConnection, namedMapProviderCache, tileBackend, previewBackend,
+                             surrogateKeysCache, tablesExtentApi, metadataBackend) {
+    BaseController.call(this, authApi, pgConnection);
+
+    this.namedMapProviderCache = namedMapProviderCache;
+    this.tileBackend = tileBackend;
+    this.previewBackend = previewBackend;
+    this.surrogateKeysCache = surrogateKeysCache;
+    this.tablesExtentApi = tablesExtentApi;
+    this.metadataBackend = metadataBackend;
+}
+
+util.inherits(NamedMapsController, BaseController);
+
+module.exports = NamedMapsController;
+
+NamedMapsController.prototype.register = function(app) {
+    app.get(app.base_url_templated +
+        '/:template_id/:layer/:z/:x/:y.(:format)', cors(), userMiddleware,
+        this.tile.bind(this));
+
+    app.get(app.base_url_mapconfig +
+        '/static/named/:template_id/:width/:height.:format', cors(), userMiddleware,
+        this.staticMap.bind(this));
+};
+
+NamedMapsController.prototype.sendResponse = function(req, res, resource, headers, namedMapProvider) {
+    this.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(req.context.user, namedMapProvider.getTemplateName()));
+    res.set('Content-Type', headers['content-type'] || headers['Content-Type'] || 'image/png');
+    res.set('Cache-Control', 'public,max-age=7200,must-revalidate');
+
+    var self = this;
+
+    step(
+        function getAffectedTablesAndLastUpdatedTime() {
+            namedMapProvider.getAffectedTablesAndLastUpdatedTime(this);
+        },
+        function sendResponse(err, result) {
+            req.profiler.done('affectedTables');
+            if (err) {
+                global.logger.log('ERROR generating cache channel: ' + err);
+            }
+            if (!result || !!result.tables) {
+                // we increase cache control as we can invalidate it
+                res.set('Cache-Control', 'public,max-age=31536000');
+
+                var lastModifiedDate;
+                if (Number.isFinite(result.lastUpdatedTime)) {
+                    lastModifiedDate = new Date(result.getLastUpdatedAt());
+                } else {
+                    lastModifiedDate = new Date();
+                }
+                res.set('Last-Modified', lastModifiedDate.toUTCString());
+
+                res.set('X-Cache-Channel', result.getCacheChannel());
+                if (result.tables.length > 0) {
+                    self.surrogateKeysCache.tag(res, result);
+                }
+            }
+            self.send(req, res, resource, 200);
+        }
+    );
+};
+
+NamedMapsController.prototype.tile = function(req, res) {
+    var self = this;
+
+    var cdbUser = req.context.user;
+
+    var namedMapProvider;
+    step(
+        function reqParams() {
+            self.req2params(req, this);
+        },
+        function getNamedMapProvider(err) {
+            assert.ifError(err);
+            self.namedMapProviderCache.get(
+                cdbUser,
+                req.params.template_id,
+                req.query.config,
+                req.query.auth_token,
+                req.params,
+                this
+            );
+        },
+        function getTile(err, _namedMapProvider) {
+            assert.ifError(err);
+            namedMapProvider = _namedMapProvider;
+            self.tileBackend.getTile(namedMapProvider, req.params, this);
+        },
+        function handleImage(err, tile, headers, stats) {
+            if (req.profiler) {
+                req.profiler.add(stats);
+            }
+            if (err) {
+                self.sendError(req, res, err, 'NAMED_MAP_TILE');
+            } else {
+                self.sendResponse(req, res, tile, headers, namedMapProvider);
+            }
+        }
+    );
+};
+
+NamedMapsController.prototype.staticMap = function(req, res) {
+    var self = this;
+
+    var cdbUser = req.context.user;
+
+    var format = req.params.format === 'jpg' ? 'jpeg' : 'png';
+    req.params.format = 'png';
+    req.params.layer = 'all';
+
+    var namedMapProvider;
+    step(
+        function reqParams() {
+            self.req2params(req, this);
+        },
+        function getNamedMapProvider(err) {
+            assert.ifError(err);
+            self.namedMapProviderCache.get(
+                cdbUser,
+                req.params.template_id,
+                req.query.config,
+                req.query.auth_token,
+                req.params,
+                this
+            );
+        },
+        function prepareImageOptions(err, _namedMapProvider) {
+            assert.ifError(err);
+            namedMapProvider = _namedMapProvider;
+            self.getStaticImageOptions(cdbUser, namedMapProvider, this);
+        },
+        function getImage(err, imageOpts) {
+            assert.ifError(err);
+
+            var width = +req.params.width;
+            var height = +req.params.height;
+
+            if (!_.isUndefined(imageOpts.zoom) && imageOpts.center) {
+                self.previewBackend.getImage(
+                    namedMapProvider, format, width, height, imageOpts.zoom, imageOpts.center, this);
+            } else {
+                self.previewBackend.getImage(
+                    namedMapProvider, format, width, height, imageOpts.bounds, this);
+            }
+        },
+        function incrementMapViews(err, image, headers, stats) {
+            assert.ifError(err);
+
+            var next = this;
+            namedMapProvider.getMapConfig(function(mapConfigErr, mapConfig) {
+                self.metadataBackend.incMapviewCount(cdbUser, mapConfig.obj().stat_tag, function(sErr) {
+                    if (err) {
+                        global.logger.log("ERROR: failed to increment mapview count for user '%s': %s", cdbUser, sErr);
+                    }
+                    next(err, image, headers, stats);
+                });
+            });
+        },
+        function handleImage(err, image, headers, stats) {
+            if (req.profiler) {
+                req.profiler.done('render-' + format);
+                req.profiler.add(stats || {});
+            }
+
+            if (err) {
+                self.sendError(req, res, err, 'STATIC_VIZ_MAP');
+            } else {
+                self.sendResponse(req, res, image, headers, namedMapProvider);
+            }
+        }
+    );
+};
+
+var DEFAULT_ZOOM_CENTER = {
+    zoom: 1,
+    center: {
+        lng: 0,
+        lat: 0
+    }
+};
+
+NamedMapsController.prototype.getStaticImageOptions = function(cdbUser, namedMapProvider, callback) {
+    var self = this;
+
+    step(
+        function getTemplate() {
+            namedMapProvider.getTemplate(this);
+        },
+        function handleTemplateView(err, template) {
+            assert.ifError(err);
+
+            if (template.view) {
+                var zoomCenter = templateZoomCenter(template.view);
+                if (zoomCenter) {
+                    return zoomCenter;
+                }
+
+                var bounds = templateBounds(template.view);
+                if (bounds) {
+                    return bounds;
+                }
+            }
+
+            return false;
+        },
+        function estimateBoundsIfNoImageOpts(err, imageOpts) {
+            if (imageOpts) {
+                return imageOpts;
+            }
+
+            var next = this;
+            namedMapProvider.getAffectedTablesAndLastUpdatedTime(function(err, affectedTablesAndLastUpdate) {
+                if (err) {
+                    return next(null);
+                }
+
+                var affectedTables = affectedTablesAndLastUpdate.tables || [];
+
+                if (affectedTables.length === 0) {
+                    return next(null);
+                }
+
+                self.tablesExtentApi.getBounds(cdbUser, affectedTables, function(err, result) {
+                    return next(null, result);
+                });
+            });
+
+        },
+        function returnCallback(err, imageOpts) {
+            return callback(err, imageOpts || DEFAULT_ZOOM_CENTER);
+        }
+    );
+};
+
+function templateZoomCenter(view) {
+    if (!_.isUndefined(view.zoom) && view.center) {
+        return {
+            zoom: view.zoom,
+            center: view.center
+        };
+    }
+    return false;
+}
+
+function templateBounds(view) {
+    if (view.bounds) {
+        var hasAllBounds = _.every(['west', 'south', 'east', 'north'], function(prop) {
+            return Number.isFinite(view.bounds[prop]);
+        });
+        if (hasAllBounds) {
+            return {
+                bounds: {
+                    west: view.bounds.west,
+                    south: view.bounds.south,
+                    east: view.bounds.east,
+                    north: view.bounds.north
+                }
+            };
+        } else {
+            return false;
+        }
+    }
+    return false;
+}

lib/cartodb/controllers/named_maps_admin.js

@@ -0,0 +1,203 @@
+var step = require('step');
+var assert = require('assert');
+var templateName = require('../backends/template_maps').templateName;
+
+var util = require('util');
+var BaseController = require('./base');
+
+var cors = require('../middleware/cors');
+var userMiddleware = require('../middleware/user');
+
+
+/**
+ * @param {AuthApi} authApi
+ * @param {PgConnection} pgConnection
+ * @param {TemplateMaps} templateMaps
+ * @constructor
+ */
+function NamedMapsAdminController(authApi, pgConnection, templateMaps) {
+    BaseController.call(this, authApi, pgConnection);
+
+    this.authApi = authApi;
+    this.templateMaps = templateMaps;
+}
+
+util.inherits(NamedMapsAdminController, BaseController);
+
+module.exports = NamedMapsAdminController;
+
+NamedMapsAdminController.prototype.register = function(app) {
+    app.post(app.base_url_templated, cors(), userMiddleware, this.create.bind(this));
+    app.put(app.base_url_templated + '/:template_id', cors(), userMiddleware, this.update.bind(this));
+    app.get(app.base_url_templated + '/:template_id', cors(), userMiddleware, this.retrieve.bind(this));
+    app.delete(app.base_url_templated + '/:template_id', cors(), userMiddleware, this.destroy.bind(this));
+    app.get(app.base_url_templated, cors(), userMiddleware, this.list.bind(this));
+    app.options(app.base_url_templated + '/:template_id', cors('Content-Type'));
+};
+
+NamedMapsAdminController.prototype.create = function(req, res) {
+    var self = this;
+
+    var cdbuser = req.context.user;
+
+    step(
+        function checkPerms(){
+            self.authApi.authorizedByAPIKey(cdbuser, req, this);
+        },
+        function addTemplate(err, authenticated) {
+            assert.ifError(err);
+            ifUnauthenticated(authenticated, 'Only authenticated users can get template maps');
+            ifInvalidContentType(req, 'template POST data must be of type application/json');
+            var cfg = req.body;
+            self.templateMaps.addTemplate(cdbuser, cfg, this);
+        },
+        function prepareResponse(err, tpl_id){
+            assert.ifError(err);
+            return { template_id: tpl_id };
+        },
+        finishFn(self, req, res, 'POST TEMPLATE')
+    );
+};
+
+NamedMapsAdminController.prototype.update = function(req, res) {
+    var self = this;
+
+    var cdbuser = req.context.user;
+    var template;
+    var tpl_id;
+
+    step(
+        function checkPerms(){
+            self.authApi.authorizedByAPIKey(cdbuser, req, this);
+        },
+        function updateTemplate(err, authenticated) {
+            assert.ifError(err);
+            ifUnauthenticated(authenticated, 'Only authenticated user can update templated maps');
+            ifInvalidContentType(req, 'template PUT data must be of type application/json');
+
+            template = req.body;
+            tpl_id = templateName(req.params.template_id);
+            self.templateMaps.updTemplate(cdbuser, tpl_id, template, this);
+        },
+        function prepareResponse(err){
+            assert.ifError(err);
+
+            return { template_id: tpl_id };
+        },
+        finishFn(self, req, res, 'PUT TEMPLATE')
+    );
+};
+
+NamedMapsAdminController.prototype.retrieve = function(req, res) {
+    var self = this;
+
+    if (req.profiler) {
+        req.profiler.start('windshaft-cartodb.get_template');
+    }
+
+    var cdbuser = req.context.user;
+    var tpl_id;
+    step(
+        function checkPerms(){
+            self.authApi.authorizedByAPIKey(cdbuser, req, this);
+        },
+        function getTemplate(err, authenticated) {
+            assert.ifError(err);
+            ifUnauthenticated(authenticated, 'Only authenticated users can get template maps');
+
+            tpl_id = templateName(req.params.template_id);
+            self.templateMaps.getTemplate(cdbuser, tpl_id, this);
+        },
+        function prepareResponse(err, tpl_val) {
+            assert.ifError(err);
+            if ( ! tpl_val ) {
+                err = new Error("Cannot find template '" + tpl_id + "' of user '" + cdbuser + "'");
+                err.http_status = 404;
+                throw err;
+            }
+            // auth_id was added by ourselves,
+            // so we remove it before returning to the user
+            delete tpl_val.auth_id;
+            return { template: tpl_val };
+        },
+        finishFn(self, req, res, 'GET TEMPLATE')
+    );
+};
+
+NamedMapsAdminController.prototype.destroy = function(req, res) {
+    var self = this;
+
+    if (req.profiler) {
+        req.profiler.start('windshaft-cartodb.delete_template');
+    }
+
+    var cdbuser = req.context.user;
+    var tpl_id;
+    step(
+        function checkPerms(){
+            self.authApi.authorizedByAPIKey(cdbuser, req, this);
+        },
+        function deleteTemplate(err, authenticated) {
+            assert.ifError(err);
+            ifUnauthenticated(authenticated, 'Only authenticated users can delete template maps');
+
+            tpl_id = templateName(req.params.template_id);
+            self.templateMaps.delTemplate(cdbuser, tpl_id, this);
+        },
+        function prepareResponse(err/*, tpl_val*/){
+            assert.ifError(err);
+            return '';
+        },
+        finishFn(self, req, res, 'DELETE TEMPLATE', 204)
+    );
+};
+
+NamedMapsAdminController.prototype.list = function(req, res) {
+    var self = this;
+    if ( req.profiler ) {
+        req.profiler.start('windshaft-cartodb.get_template_list');
+    }
+
+    var cdbuser = req.context.user;
+
+    step(
+        function checkPerms(){
+            self.authApi.authorizedByAPIKey(cdbuser, req, this);
+        },
+        function listTemplates(err, authenticated) {
+            assert.ifError(err);
+            ifUnauthenticated(authenticated, 'Only authenticated user can list templated maps');
+
+            self.templateMaps.listTemplates(cdbuser, this);
+        },
+        function prepareResponse(err, tpl_ids){
+            assert.ifError(err);
+            return { template_ids: tpl_ids };
+        },
+        finishFn(self, req, res, 'GET TEMPLATE LIST')
+    );
+};
+
+function finishFn(controller, req, res, description, status) {
+    return function finish(err, response){
+        if (err) {
+            controller.sendError(req, res, err, description);
+        } else {
+            controller.send(req, res, response, status || 200);
+        }
+    };
+}
+
+function ifUnauthenticated(authenticated, description) {
+    if (!authenticated) {
+        var err = new Error(description);
+        err.http_status = 403;
+        throw err;
+    }
+}
+
+function ifInvalidContentType(req, description) {
+    if (!req.is('application/json')) {
+        throw new Error(description);
+    }
+}

lib/cartodb/controllers/server_info.js

@@ -0,0 +1,56 @@
+var windshaft = require('windshaft');
+var HealthCheck = require('../monitoring/health_check');
+
+var WELCOME_MSG = "This is the CartoDB Maps API, " +
+    "see the documentation at http://docs.cartodb.com/cartodb-platform/maps-api.html";
+
+
+var versions = {
+    windshaft: windshaft.version,
+    grainstore: windshaft.grainstore.version(),
+    node_mapnik: windshaft.mapnik.version,
+    mapnik: windshaft.mapnik.versions.mapnik,
+    windshaft_cartodb: require('../../../package.json').version
+};
+
+function ServerInfoController() {
+    this.healthConfig = global.environment.health || {};
+    this.healthCheck = new HealthCheck(global.environment.disabled_file);
+}
+
+module.exports = ServerInfoController;
+
+ServerInfoController.prototype.register = function(app) {
+    app.get('/health', this.health.bind(this));
+    app.get('/', this.welcome.bind(this));
+    app.get('/version', this.version.bind(this));
+};
+
+ServerInfoController.prototype.welcome = function(req, res) {
+    res.status(200).send(WELCOME_MSG);
+};
+
+ServerInfoController.prototype.version = function(req, res) {
+    res.status(200).send(versions);
+};
+
+ServerInfoController.prototype.health = function(req, res) {
+    if (!!this.healthConfig.enabled) {
+        var startTime = Date.now();
+        this.healthCheck.check(function(err) {
+            var ok = !err;
+            var response = {
+                enabled: true,
+                ok: ok,
+                elapsed: Date.now() - startTime
+            };
+            if (err) {
+                response.err = err.message;
+            }
+            res.status(ok ? 200 : 503).send(response);
+
+        });
+    } else {
+        res.status(200).send({enabled: false, ok: true});
+    }
+};

lib/cartodb/controllers/template_maps.js

@@ -1,478 +0,0 @@
-var Step = require('step');
-var _ = require('underscore');
-
-function TemplateMapsController(app, serverOptions, templateMaps, metadataBackend, templateBaseUrl, surrogateKeysCache,
-                                NamedMapsCacheEntry) {
-    this.app = app;
-    this.serverOptions = serverOptions;
-    this.templateMaps = templateMaps;
-    this.metadataBackend = metadataBackend;
-    this.templateBaseUrl = templateBaseUrl;
-    this.surrogateKeysCache = surrogateKeysCache;
-    this.NamedMapsCacheEntry = NamedMapsCacheEntry;
-}
-
-module.exports = TemplateMapsController;
-
-
-TemplateMapsController.prototype.register = function(app) {
-    app.get(this.templateBaseUrl + '/:template_id/jsonp', this.jsonp.bind(this));
-    app.post(this.templateBaseUrl, this.create.bind(this));
-    app.put(this.templateBaseUrl + '/:template_id', this.update.bind(this));
-    app.get(this.templateBaseUrl + '/:template_id', this.retrieve.bind(this));
-    app.del(this.templateBaseUrl + '/:template_id', this.destroy.bind(this));
-    app.get(this.templateBaseUrl, this.list.bind(this));
-    app.options(this.templateBaseUrl + '/:template_id', this.options.bind(this));
-    app.post(this.templateBaseUrl + '/:template_id', this.instantiate.bind(this));
-};
-
-// Add a template
-TemplateMapsController.prototype.create = function(req, res) {
-    var self = this;
-
-    this.app.doCORS(res);
-
-    var cdbuser = self.serverOptions.userByReq(req);
-
-    Step(
-        function checkPerms(){
-            self.serverOptions.authorizedByAPIKey(req, this);
-        },
-        function addTemplate(err, authenticated) {
-            if ( err ) throw err;
-            if (authenticated !== 1) {
-                err = new Error("Only authenticated user can create templated maps");
-                err.http_status = 403;
-                throw err;
-            }
-            if ( ! req.headers['content-type'] || req.headers['content-type'].split(';')[0] != 'application/json' )
-                throw new Error('template POST data must be of type application/json');
-            var cfg = req.body;
-            self.templateMaps.addTemplate(cdbuser, cfg, this);
-        },
-        function prepareResponse(err, tpl_id){
-            if ( err ) throw err;
-            // NOTE: might omit "cdbuser" if == dbowner ...
-            return { template_id: cdbuser + '@' + tpl_id };
-        },
-        function finish(err, response){
-            if ( req.profiler ) {
-                res.header('X-Tiler-Profiler', req.profiler.toJSONString());
-            }
-            if (err){
-                response = { error: ''+err };
-                var statusCode = 400;
-                if ( ! _.isUndefined(err.http_status) ) {
-                    statusCode = err.http_status;
-                }
-                self.app.sendError(res, response, statusCode, 'POST TEMPLATE', err);
-            } else {
-                self.app.sendResponse(res, [response, 200]);
-            }
-        }
-    );
-};
-
-// Update a template
-TemplateMapsController.prototype.update = function(req, res) {
-    var self = this;
-
-    this.app.doCORS(res);
-
-    var cdbuser = this.serverOptions.userByReq(req);
-    var template;
-    var tpl_id;
-    Step(
-        function checkPerms(){
-            self.serverOptions.authorizedByAPIKey(req, this);
-        },
-        function updateTemplate(err, authenticated) {
-            if ( err ) throw err;
-            if (authenticated !== 1) {
-                err = new Error("Only authenticated user can list templated maps");
-                err.http_status = 403;
-                throw err;
-            }
-            if ( ! req.headers['content-type'] || req.headers['content-type'].split(';')[0] != 'application/json' )
-                throw new Error('template PUT data must be of type application/json');
-            template = req.body;
-            tpl_id = req.params.template_id.split('@');
-            if ( tpl_id.length > 1 ) {
-                if ( tpl_id[0] != cdbuser ) {
-                    err = new Error("Invalid template id '"
-                        + req.params.template_id + "' for user '" + cdbuser + "'");
-                    err.http_status = 404;
-                    throw err;
-                }
-                tpl_id = tpl_id[1];
-            }
-            self.templateMaps.updTemplate(cdbuser, tpl_id, template, this);
-        },
-        function prepareResponse(err){
-            if ( err ) throw err;
-            return { template_id: cdbuser + '@' + tpl_id };
-        },
-        function finish(err, response){
-            if ( req.profiler ) {
-                res.header('X-Tiler-Profiler', req.profiler.toJSONString());
-            }
-            if (err){
-                var statusCode = 400;
-                response = { error: ''+err };
-                if ( ! _.isUndefined(err.http_status) ) {
-                    statusCode = err.http_status;
-                }
-                self.app.sendError(res, response, statusCode, 'PUT TEMPLATE', err);
-            } else {
-                self.app.sendResponse(res, [response, 200]);
-            }
-        }
-    );
-};
-
-// Get a specific template
-TemplateMapsController.prototype.retrieve = function(req, res) {
-    var self = this;
-
-    if ( req.profiler && req.profiler.statsd_client ) {
-        req.profiler.start('windshaft-cartodb.get_template');
-    }
-
-    this.app.doCORS(res);
-
-    var cdbuser = this.serverOptions.userByReq(req);
-    var template;
-    var tpl_id;
-    Step(
-        function checkPerms(){
-            self.serverOptions.authorizedByAPIKey(req, this);
-        },
-        function updateTemplate(err, authenticated) {
-            if ( err ) throw err;
-            if (authenticated !== 1) {
-                err = new Error("Only authenticated users can get template maps");
-                err.http_status = 403;
-                throw err;
-            }
-            tpl_id = req.params.template_id.split('@');
-            if ( tpl_id.length > 1 ) {
-                if ( tpl_id[0] != cdbuser ) {
-                    var err = new Error("Cannot get template id '"
-                        + req.params.template_id + "' for user '" + cdbuser + "'");
-                    err.http_status = 404;
-                    throw err;
-                }
-                tpl_id = tpl_id[1];
-            }
-            self.templateMaps.getTemplate(cdbuser, tpl_id, this);
-        },
-        function prepareResponse(err, tpl_val){
-            if ( err ) throw err;
-            if ( ! tpl_val ) {
-                err = new Error("Cannot find template '" + tpl_id + "' of user '" + cdbuser + "'");
-                err.http_status = 404;
-                throw err;
-            }
-            // auth_id was added by ourselves,
-            // so we remove it before returning to the user
-            delete tpl_val.auth_id;
-            return { template: tpl_val };
-        },
-        function finish(err, response){
-            if (err){
-                var statusCode = 400;
-                response = { error: ''+err };
-                if ( ! _.isUndefined(err.http_status) ) {
-                    statusCode = err.http_status;
-                }
-                self.app.sendError(res, response, statusCode, 'GET TEMPLATE', err);
-            } else {
-                self.app.sendResponse(res, [response, 200]);
-            }
-        }
-    );
-};
-
-// Delete a specific template
-TemplateMapsController.prototype.destroy = function(req, res) {
-    var self = this;
-
-    if ( req.profiler && req.profiler.statsd_client ) {
-        req.profiler.start('windshaft-cartodb.delete_template');
-    }
-    this.app.doCORS(res);
-
-    var cdbuser = this.serverOptions.userByReq(req);
-    var template;
-    var tpl_id;
-    Step(
-        function checkPerms(){
-            self.serverOptions.authorizedByAPIKey(req, this);
-        },
-        function updateTemplate(err, authenticated) {
-            if ( err ) throw err;
-            if (authenticated !== 1) {
-                err = new Error("Only authenticated users can delete template maps");
-                err.http_status = 403;
-                throw err;
-            }
-            tpl_id = req.params.template_id.split('@');
-            if ( tpl_id.length > 1 ) {
-                if ( tpl_id[0] != cdbuser ) {
-                    var err = new Error("Cannot find template id '"
-                        + req.params.template_id + "' for user '" + cdbuser + "'");
-                    err.http_status = 404;
-                    throw err;
-                }
-                tpl_id = tpl_id[1];
-            }
-            self.templateMaps.delTemplate(cdbuser, tpl_id, this);
-        },
-        function prepareResponse(err, tpl_val){
-            if ( err ) throw err;
-            return { status: 'ok' };
-        },
-        function finish(err, response){
-            if (err){
-                var statusCode = 400;
-                response = { error: ''+err };
-                if ( ! _.isUndefined(err.http_status) ) {
-                    statusCode = err.http_status;
-                }
-                self.app.sendError(res, response, statusCode, 'DELETE TEMPLATE', err);
-            } else {
-                self.app.sendResponse(res, ['', 204]);
-            }
-        }
-    );
-};
-
-// Get a list of owned templates
-TemplateMapsController.prototype.list = function(req, res) {
-    var self = this;
-
-    if ( req.profiler && req.profiler.statsd_client ) {
-        req.profiler.start('windshaft-cartodb.get_template_list');
-    }
-    this.app.doCORS(res);
-
-    var cdbuser = this.serverOptions.userByReq(req);
-
-    Step(
-        function checkPerms(){
-            self.serverOptions.authorizedByAPIKey(req, this);
-        },
-        function listTemplates(err, authenticated) {
-            if ( err ) throw err;
-            if (authenticated !== 1) {
-                err = new Error("Only authenticated user can list templated maps");
-                err.http_status = 403;
-                throw err;
-            }
-            self.templateMaps.listTemplates(cdbuser, this);
-        },
-        function prepareResponse(err, tpl_ids){
-            if ( err ) throw err;
-            // NOTE: might omit "cbduser" if == dbowner ...
-            var ids = _.map(tpl_ids, function(id) { return cdbuser + '@' + id; });
-            return { template_ids: ids };
-        },
-        function finish(err, response){
-            var statusCode = 200;
-            if (err){
-                response = { error: ''+err };
-                if ( ! _.isUndefined(err.http_status) ) {
-                    statusCode = err.http_status;
-                }
-                self.app.sendError(res, response, statusCode, 'GET TEMPLATE LIST', err);
-            } else {
-                self.app.sendResponse(res, [response, statusCode]);
-            }
-        }
-    );
-};
-
-TemplateMapsController.prototype.instantiate = function(req, res) {
-    var self = this;
-
-    if ( req.profiler && req.profiler.statsd_client) {
-        req.profiler.start('windshaft-cartodb.instance_template_post');
-    }
-    Step(
-        function() {
-            if ( ! req.headers['content-type'] || req.headers['content-type'].split(';')[0] != 'application/json') {
-                throw new Error('template POST data must be of type application/json, it is instead ');
-            }
-            self.instantiateTemplate(req, res, req.body, this);
-        }, function(err, response) {
-            self.finish_instantiation(err, response, res, req);
-        }
-    );
-};
-
-TemplateMapsController.prototype.options = function(req, res) {
-    this.app.doCORS(res, "Content-Type");
-    return next();
-};
-
-/**
- * jsonp endpoint, allows to instantiate a template with a json call.
- * callback query argument is mandatory
- */
-TemplateMapsController.prototype.jsonp = function(req, res) {
-    var self = this;
-
-    if ( req.profiler && req.profiler.statsd_client) {
-        req.profiler.start('windshaft-cartodb.instance_template_get');
-    }
-    Step(
-        function() {
-            if ( req.query.callback === undefined || req.query.callback.length === 0) {
-                throw new Error('callback parameter should be present and be a function name');
-            }
-            var config = {};
-            if(req.query.config) {
-                try {
-                    config = JSON.parse(req.query.config);
-                } catch(e) {
-                    throw new Error('badformed config parameter, should be a valid JSON');
-                }
-            }
-            self.instantiateTemplate(req, res, config, this);
-        }, function(err, response) {
-            self.finish_instantiation(err, response, res, req);
-        }
-    );
-};
-
-
-// Instantiate a template
-TemplateMapsController.prototype.instantiateTemplate = function(req, res, template_params, callback) {
-    var self = this;
-
-    this.app.doCORS(res);
-
-    var template;
-    var layergroup;
-    var fakereq; // used for call to createLayergroup
-    var cdbuser = self.serverOptions.userByReq(req);
-    // Format of template_id: [<template_owner>]@<template_id>
-    var tpl_id = req.params.template_id.split('@');
-    if ( tpl_id.length > 1 ) {
-        if ( tpl_id[0] && tpl_id[0] != cdbuser ) {
-            var err = new Error('Cannot instanciate map of user "'
-                + tpl_id[0] + '" on database of user "'
-                + cdbuser + '"');
-            err.http_status = 403;
-            callback(err);
-            return;
-        }
-        tpl_id = tpl_id[1];
-    }
-    var auth_token = req.query.auth_token;
-    Step(
-        function getTemplate(){
-            self.templateMaps.getTemplate(cdbuser, tpl_id, this);
-        },
-        function checkAuthorized(err, templateValue) {
-            if ( req.profiler ) req.profiler.done('getTemplate');
-            if ( err ) throw err;
-            if ( ! templateValue ) {
-                err = new Error("Template '" + tpl_id + "' of user '" + cdbuser + "' not found");
-                err.http_status = 404;
-                throw err;
-            }
-
-            template = templateValue;
-
-            var authorized = false;
-            try {
-                authorized = self.templateMaps.isAuthorized(template, auth_token);
-            } catch (err) {
-                // we catch to add http_status
-                err.http_status = 403;
-                throw err;
-            }
-            if ( ! authorized ) {
-                err = new Error('Unauthorized template instanciation');
-                err.http_status = 403;
-                throw err;
-            }
-
-            if (req.profiler) {
-                req.profiler.done('authorizedByCert');
-            }
-
-            return self.templateMaps.instance(template, template_params);
-        },
-        function prepareParams(err, instance){
-            if ( req.profiler ) req.profiler.done('TemplateMaps_instance');
-            if ( err ) throw err;
-            layergroup = instance;
-            fakereq = { query: {}, params: {}, headers: _.clone(req.headers),
-                method: req.method,
-                res: res,
-                profiler: req.profiler
-            };
-            self.setDBParams(cdbuser, fakereq.params, this);
-        },
-        function setApiKey(err){
-            if ( req.profiler ) req.profiler.done('setDBParams');
-            if ( err ) throw err;
-            self.metadataBackend.getUserMapKey(cdbuser, this);
-        },
-        function createLayergroup(err, val) {
-            if ( req.profiler ) req.profiler.done('getUserMapKey');
-            if ( err ) throw err;
-            fakereq.params.api_key = val;
-            self.app.createLayergroup(layergroup, fakereq, this);
-        },
-        function prepareResponse(err, layergroup) {
-            if ( err ) {
-                throw err;
-            }
-            var tplhash = self.templateMaps.fingerPrint(template).substring(0,8);
-            layergroup.layergroupid = cdbuser + '@' + tplhash + '@' + layergroup.layergroupid;
-
-            self.surrogateKeysCache.tag(res, new self.NamedMapsCacheEntry(cdbuser, template.name));
-
-            return layergroup;
-        },
-        callback
-    );
-};
-
-TemplateMapsController.prototype.finish_instantiation = function(err, response, res, req) {
-    if ( req.profiler ) {
-        res.header('X-Tiler-Profiler', req.profiler.toJSONString());
-    }
-    if (err) {
-        var statusCode = 400;
-        response = { error: ''+err };
-        if ( ! _.isUndefined(err.http_status) ) {
-            statusCode = err.http_status;
-        }
-        if(global.environment.debug) {
-            response.stack = err.stack;
-        }
-        this.app.sendError(res, response, statusCode, 'POST INSTANCE TEMPLATE', err);
-    } else {
-        this.app.sendResponse(res, [response, 200]);
-    }
-};
-
-TemplateMapsController.prototype.setDBParams = function(cdbuser, params, callback) {
-    var self = this;
-    Step(
-        function setAuth() {
-            self.serverOptions.setDBAuth(cdbuser, params, this);
-        },
-        function setConn(err) {
-            if ( err ) throw err;
-            self.serverOptions.setDBConn(cdbuser, params, this);
-        },
-        function finish(err) {
-            callback(err);
-        }
-    );
-};

lib/cartodb/log4js_rollbar.js

@@ -1,49 +0,0 @@
-var rollbar = require("rollbar");
-
-/**
- * Rollbar Appender. Sends logging events to Rollbar using node-rollbar 
- *
- * @param config object with rollbar configuration data
- * {
- *   token: 'your-secret-token',
- *   options: node-rollbar options
- * }
- */
-function rollbarAppender(config) {
-
-  var opt = config.options;
-	rollbar.init(opt.token, opt.options);
-	
-	return function(loggingEvent) {
-/*
-For logger.trace('one','two','three'):
-{ startTime: Wed Mar 12 2014 16:27:40 GMT+0100 (CET),
-  categoryName: '[default]',
-  data: [ 'one', 'two', 'three' ],
-  level: { level: 5000, levelStr: 'TRACE' },
-  logger: { category: '[default]', _events: { log: [Object] } } }
-*/
-
-    // Levels:
-    // TRACE  5000
-    // DEBUG 10000
-    // INFO  20000
-    // WARN  30000
-    // ERROR 40000
-    // FATAL 50000
-    //
-    // We only log error and higher errors
-    //
-    if ( loggingEvent.level.level < 40000 ) return;
-
-    rollbar.reportMessage(loggingEvent.data);
-	};
-}
-
-function configure(config) {
-	return rollbarAppender(config);
-}
-
-exports.name      = "rollbar";
-exports.appender  = rollbarAppender;
-exports.configure = configure;

lib/cartodb/middleware/cors.js

@@ -0,0 +1,11 @@
+module.exports = function cors(extraHeaders) {
+    return function(req, res, next) {
+        var baseHeaders = "X-Requested-With, X-Prototype-Version, X-CSRF-Token";
+        if(extraHeaders) {
+            baseHeaders += ", " + extraHeaders;
+        }
+        res.set("Access-Control-Allow-Origin", "*");
+        res.set("Access-Control-Allow-Headers", baseHeaders);
+        next();
+    };
+};

lib/cartodb/middleware/user.js

@@ -0,0 +1,7 @@
+var CdbRequest = require('../models/cdb_request');
+var cdbRequest = new CdbRequest();
+
+module.exports = function userMiddleware(req, res, next) {
+    req.context.user = cdbRequest.userByReq(req);
+    next();
+};

lib/cartodb/models/analysis-mapconfig-adapter.js

@@ -0,0 +1,220 @@
+var queue = require('queue-async');
+var debug = require('debug')('windshaft:analysis');
+
+var camshaft = require('camshaft');
+var dot = require('dot');
+dot.templateSettings.strip = false;
+
+function AnalysisMapConfigAdapter(analysisBackend) {
+    this.analysisBackend = analysisBackend;
+}
+
+module.exports = AnalysisMapConfigAdapter;
+
+var SKIP_COLUMNS = {
+    'the_geom': true,
+    'the_geom_webmercator': true
+};
+
+function skipColumns(columnNames) {
+    return columnNames
+        .filter(function(columnName) { return !SKIP_COLUMNS[columnName]; });
+}
+
+var layerQueryTemplate = dot.template([
+    'SELECT {{=it._columns}}',
+    'FROM ({{=it._query}}) _cdb_analysis_query'
+].join('\n'));
+
+function layerQuery(query, columnNames) {
+    var _columns = ['ST_Transform(the_geom, 3857) the_geom_webmercator'].concat(skipColumns(columnNames));
+    return layerQueryTemplate({ _query: query, _columns: _columns.join(', ') });
+}
+
+function appendFiltersToNodes(requestMapConfig, dataviewsFiltersBySourceId) {
+    var analyses = requestMapConfig.analyses || [];
+
+    requestMapConfig.analyses = analyses.map(function(analysisDefinition) {
+        var analysisGraph = new camshaft.reference.AnalysisGraph(analysisDefinition);
+        var definition = analysisDefinition;
+        Object.keys(dataviewsFiltersBySourceId).forEach(function(sourceId) {
+            definition = analysisGraph.getDefinitionWith(sourceId, {filters: dataviewsFiltersBySourceId[sourceId] });
+        });
+
+        return definition;
+    });
+
+    return requestMapConfig;
+}
+
+function shouldAdaptLayers(requestMapConfig) {
+    return Array.isArray(requestMapConfig.layers) &&
+        Array.isArray(requestMapConfig.analyses) && requestMapConfig.analyses.length > 0;
+}
+
+var DATAVIEW_TYPE_2_FILTER_TYPE = {
+    aggregation: 'category',
+    histogram: 'range'
+};
+function getFilter(dataview, params) {
+    var type = dataview.type;
+
+    return {
+        type: DATAVIEW_TYPE_2_FILTER_TYPE[type],
+        column: dataview.options.column,
+        params: params
+    };
+}
+
+AnalysisMapConfigAdapter.prototype.getMapConfig = function(analysisConfiguration, requestMapConfig, filters, callback) {
+    // jshint maxcomplexity:7
+    var self = this;
+    filters = filters || {};
+
+    if (!shouldAdaptLayers(requestMapConfig)) {
+        return callback(null, requestMapConfig);
+    }
+
+    var dataviewsFilters = filters.dataviews || {};
+    debug(dataviewsFilters);
+    var dataviews = requestMapConfig.dataviews || {};
+
+    var errors = getDataviewsErrors(dataviews);
+    if (errors.length > 0) {
+        return callback(errors);
+    }
+
+    var dataviewsFiltersBySourceId = Object.keys(dataviewsFilters).reduce(function(bySourceId, dataviewName) {
+        var dataview = dataviews[dataviewName];
+        if (dataview) {
+            var sourceId = dataview.source.id;
+            if (!bySourceId.hasOwnProperty(sourceId)) {
+                bySourceId[sourceId] = {};
+            }
+
+            bySourceId[sourceId][dataviewName] = getFilter(dataview, dataviewsFilters[dataviewName]);
+        }
+        return bySourceId;
+    }, {});
+
+    debug(dataviewsFiltersBySourceId);
+
+    debug('mapconfig input', JSON.stringify(requestMapConfig, null, 4));
+
+    requestMapConfig = appendFiltersToNodes(requestMapConfig, dataviewsFiltersBySourceId);
+
+    function createAnalysis(analysisDefinition, done) {
+        self.analysisBackend.create(analysisConfiguration, analysisDefinition, done);
+    }
+
+    var analysesQueue = queue(requestMapConfig.analyses.length);
+    requestMapConfig.analyses.forEach(function(analysis) {
+        analysesQueue.defer(createAnalysis, analysis);
+    });
+
+    analysesQueue.awaitAll(function(err, analysesResults) {
+        if (err) {
+            return callback(err);
+        }
+
+        var sourceId2Node = analysesResults.reduce(function(sourceId2Query, analysis) {
+            var rootNode = analysis.getRoot();
+            if (rootNode.params && rootNode.params.id) {
+                sourceId2Query[rootNode.params.id] = rootNode;
+            }
+
+            analysis.getSortedNodes().forEach(function(node) {
+                if (node.params && node.params.id) {
+                    sourceId2Query[node.params.id] = node;
+                }
+            });
+
+            return sourceId2Query;
+        }, {});
+
+        var missingNodesErrors = [];
+
+        requestMapConfig.layers = requestMapConfig.layers.map(function(layer, layerIndex) {
+            if (getLayerSourceId(layer)) {
+                var layerSourceId = getLayerSourceId(layer);
+                var layerNode = sourceId2Node[layerSourceId];
+                if (layerNode) {
+                    layer.options.sql = layerQuery(layerNode.getQuery(), layerNode.getColumns());
+                    var layerDataviews = getLayerDataviews(layer, dataviews);
+                    layer.options.columns = layerDataviews.reduce(function(columns, dataview) {
+                        return columns.concat(getDataviewColumns(dataview));
+                    }, []);
+                } else {
+                    missingNodesErrors.push(
+                        new Error('Missing analysis node.id="' + layerSourceId +'" for layer='+layerIndex)
+                    );
+                }
+            }
+            return layer;
+        });
+
+        debug('mapconfig output', JSON.stringify(requestMapConfig, null, 4));
+
+        if (missingNodesErrors.length > 0) {
+            return callback(missingNodesErrors);
+        }
+
+        return callback(null, requestMapConfig, analysesResults);
+    });
+};
+
+function getLayerSourceId(layer) {
+    return layer.options.source && layer.options.source.id;
+}
+
+function getDataviewSourceId(dataview) {
+    return dataview.source && dataview.source.id;
+}
+
+function getLayerDataviews(layer, dataviews) {
+    var layerDataviews = [];
+
+    var layerSourceId = getLayerSourceId(layer);
+    if (layerSourceId) {
+        var dataviewsList = getDataviewsList(dataviews);
+        dataviewsList.forEach(function(dataview) {
+            if (getDataviewSourceId(dataview) === layerSourceId) {
+                layerDataviews.push(dataview);
+            }
+        });
+    }
+
+    return layerDataviews;
+}
+
+function getDataviewColumns(dataview) {
+    var columns = [];
+    var options = dataview.options;
+    ['column', 'aggregationColumn'].forEach(function(opt) {
+        if (options.hasOwnProperty(opt)) {
+            columns.push(options[opt]);
+        }
+    });
+    return columns;
+}
+
+function getDataviewsList(dataviews) {
+    return Object.keys(dataviews).map(function(dataviewKey) { return dataviews[dataviewKey]; });
+}
+
+function getDataviewsErrors(dataviews) {
+    var errors = [];
+
+    Object.keys(dataviews).forEach(function(dataviewName) {
+        var dataview = dataviews[dataviewName];
+        if (!dataview.hasOwnProperty('source') || !dataview.source.id) {
+            errors.push(new Error('Dataview "' + dataviewName + '" is missing `source.id` attribute'));
+        }
+
+        if (!dataview.type) {
+            errors.push(new Error('Dataview "' + dataviewName + '" is missing `type` attribute'));
+        }
+    });
+
+    return errors;
+}

lib/cartodb/models/cdb_request.js

@@ -0,0 +1,25 @@
+function CdbRequest() {
+    this.RE_USER_FROM_HOST = new RegExp(global.environment.user_from_host ||
+            '^([^\\.]+)\\.' // would extract "strk" from "strk.cartodb.com"
+    );
+}
+
+module.exports = CdbRequest;
+
+
+CdbRequest.prototype.userByReq = function(req) {
+    var host = req.headers.host || '';
+    if (req.params.user) {
+        return req.params.user;
+    }
+    var mat = host.match(this.RE_USER_FROM_HOST);
+    if ( ! mat ) {
+        global.logger.error("Pattern '%s' does not match hostname '%s'", this.RE_USER_FROM_HOST, host);
+        return;
+    }
+    if ( mat.length !== 2 ) {
+        global.logger.error("Pattern '%s' gave unexpected matches against '%s': %s", this.RE_USER_FROM_HOST, host, mat);
+        return;
+    }
+    return mat[1];
+};

lib/cartodb/models/dataview/aggregation.js

@@ -0,0 +1,278 @@
+var _ = require('underscore');
+var BaseWidget = require('./base');
+var debug = require('debug')('windshaft:widget:aggregation');
+
+var dot = require('dot');
+dot.templateSettings.strip = false;
+
+var summaryQueryTpl = dot.template([
+    'summary AS (',
+    '  SELECT',
+    '  count(1) AS count,',
+    '  sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count',
+    '  FROM ({{=it._query}}) _cdb_aggregation_nulls',
+    ')'
+].join('\n'));
+
+var rankedCategoriesQueryTpl = dot.template([
+    'categories AS(',
+    '  SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,',
+    '    row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank',
+    '  FROM ({{=it._query}}) _cdb_aggregation_all',
+    '  GROUP BY {{=it._column}}',
+    '  ORDER BY 2 DESC',
+    ')'
+].join('\n'));
+
+var categoriesSummaryQueryTpl = dot.template([
+    'categories_summary AS(',
+    '  SELECT count(1) categories_count, max(value) max_val, min(value) min_val',
+    '  FROM categories',
+    ')'
+].join('\n'));
+
+var rankedAggregationQueryTpl = dot.template([
+    'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count',
+    '  FROM categories, summary, categories_summary',
+    '  WHERE rank < {{=it._limit}}',
+    'UNION ALL',
+    'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val, count, categories_count',
+    '  FROM categories, summary, categories_summary',
+    '  WHERE rank >= {{=it._limit}}',
+    'GROUP BY nulls_count, min_val, max_val, count, categories_count'
+].join('\n'));
+
+var aggregationQueryTpl = dot.template([
+    'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,',
+    '  nulls_count, min_val, max_val, count, categories_count',
+    'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary',
+    'GROUP BY category, nulls_count, min_val, max_val, count, categories_count',
+    'ORDER BY value DESC'
+].join('\n'));
+
+var CATEGORIES_LIMIT = 6;
+
+var VALID_OPERATIONS = {
+    count: [],
+    sum: ['aggregationColumn']
+};
+
+var TYPE = 'aggregation';
+
+/**
+ {
+     type: 'aggregation',
+     options: {
+         column: 'name',
+         aggregation: 'count' // it could be, e.g., sum if column is numeric
+     }
+ }
+ */
+function Aggregation(query, options) {
+    if (!_.isString(options.column)) {
+        throw new Error('Aggregation expects `column` in widget options');
+    }
+
+    if (!_.isString(options.aggregation)) {
+        throw new Error('Aggregation expects `aggregation` operation in widget options');
+    }
+
+    if (!VALID_OPERATIONS[options.aggregation]) {
+        throw new Error("Aggregation does not support '" + options.aggregation + "' operation");
+    }
+
+    var requiredOptions = VALID_OPERATIONS[options.aggregation];
+    var missingOptions = _.difference(requiredOptions, Object.keys(options));
+    if (missingOptions.length > 0) {
+        throw new Error(
+            "Aggregation '" + options.aggregation + "' is missing some options: " + missingOptions.join(',')
+        );
+    }
+
+    BaseWidget.apply(this);
+
+    this.query = query;
+    this.column = options.column;
+    this.aggregation = options.aggregation;
+    this.aggregationColumn = options.aggregationColumn;
+}
+
+Aggregation.prototype = new BaseWidget();
+Aggregation.prototype.constructor = Aggregation;
+
+module.exports = Aggregation;
+
+Aggregation.prototype.sql = function(psql, filters, override, callback) {
+    if (!callback) {
+        callback = override;
+        override = {};
+    }
+
+    var _query = this.query;
+
+    var aggregationSql;
+    if (!!override.ownFilter) {
+        aggregationSql = [
+            "WITH",
+            [
+                summaryQueryTpl({
+                    _query: _query,
+                    _column: this.column
+                }),
+                rankedCategoriesQueryTpl({
+                    _query: _query,
+                    _column: this.column,
+                    _aggregation: this.getAggregationSql()
+                }),
+                categoriesSummaryQueryTpl({
+                    _query: _query,
+                    _column: this.column
+                })
+            ].join(',\n'),
+            aggregationQueryTpl({
+                _query: _query,
+                _column: this.column,
+                _aggregation: this.getAggregationSql(),
+                _limit: CATEGORIES_LIMIT
+            })
+        ].join('\n');
+    } else {
+        aggregationSql = [
+            "WITH",
+            [
+                summaryQueryTpl({
+                    _query: _query,
+                    _column: this.column
+                }),
+                rankedCategoriesQueryTpl({
+                    _query: _query,
+                    _column: this.column,
+                    _aggregation: this.getAggregationSql()
+                }),
+                categoriesSummaryQueryTpl({
+                    _query: _query,
+                    _column: this.column
+                })
+            ].join(',\n'),
+            rankedAggregationQueryTpl({
+                _query: _query,
+                _column: this.column,
+                _limit: CATEGORIES_LIMIT
+            })
+        ].join('\n');
+    }
+
+    debug(aggregationSql);
+
+    return callback(null, aggregationSql);
+};
+
+var aggregationFnQueryTpl = dot.template('{{=it._aggregationFn}}({{=it._aggregationColumn}})');
+Aggregation.prototype.getAggregationSql = function() {
+    return aggregationFnQueryTpl({
+        _aggregationFn: this.aggregation,
+        _aggregationColumn: this.aggregationColumn || 1
+    });
+};
+
+Aggregation.prototype.format = function(result) {
+    var categories = [];
+    var count = 0;
+    var nulls = 0;
+    var minValue = 0;
+    var maxValue = 0;
+    var categoriesCount = 0;
+
+
+    if (result.rows.length) {
+        var firstRow = result.rows[0];
+        count = firstRow.count;
+        nulls = firstRow.nulls_count;
+        minValue = firstRow.min_val;
+        maxValue = firstRow.max_val;
+        categoriesCount = firstRow.categories_count;
+
+        result.rows.forEach(function(row) {
+            categories.push(_.omit(row, 'count', 'nulls_count', 'min_val', 'max_val', 'categories_count'));
+        });
+    }
+
+    return {
+        count: count,
+        nulls: nulls,
+        min: minValue,
+        max: maxValue,
+        categoriesCount: categoriesCount,
+        categories: categories
+    };
+};
+
+var filterCategoriesQueryTpl = dot.template([
+    'SELECT {{=it._column}} AS category, {{=it._value}} AS value',
+    'FROM ({{=it._query}}) _cdb_aggregation_search',
+    'WHERE CAST({{=it._column}} as text) ILIKE {{=it._userQuery}}',
+    'GROUP BY {{=it._column}}'
+].join('\n'));
+
+var searchQueryTpl = dot.template([
+    'WITH',
+    'search_unfiltered AS (',
+    '  {{=it._searchUnfiltered}}',
+    '),',
+    'search_filtered AS (',
+    '  {{=it._searchFiltered}}',
+    '),',
+    'search_union AS (',
+    '  SELECT * FROM search_unfiltered',
+    '  UNION ALL',
+    '  SELECT * FROM search_filtered',
+    ')',
+    'SELECT category, sum(value) AS value',
+    'FROM search_union',
+    'GROUP BY category',
+    'ORDER BY value desc'
+].join('\n'));
+
+
+Aggregation.prototype.search = function(psql, userQuery, callback) {
+    var self = this;
+
+    var _userQuery = psql.escapeLiteral('%' + userQuery + '%');
+
+    // TODO unfiltered will be wrong as filters are already applied at this point
+    var query = searchQueryTpl({
+        _searchUnfiltered: filterCategoriesQueryTpl({
+            _query: this.query,
+            _column: this.column,
+            _value: '0',
+            _userQuery: _userQuery
+        }),
+        _searchFiltered: filterCategoriesQueryTpl({
+            _query: this.query,
+            _column: this.column,
+            _value: 'count(1)',
+            _userQuery: _userQuery
+        })
+    });
+
+    psql.query(query, function(err, result) {
+        if (err) {
+            return callback(err, result);
+        }
+
+        return callback(null, {type: self.getType(), categories: result.rows });
+    }, true); // use read-only transaction
+};
+
+Aggregation.prototype.getType = function() {
+    return TYPE;
+};
+
+Aggregation.prototype.toString = function() {
+    return JSON.stringify({
+        _type: TYPE,
+        _query: this.query,
+        _column: this.column,
+        _aggregation: this.aggregation
+    });
+};

lib/cartodb/models/dataview/base.js

@@ -0,0 +1,26 @@
+function BaseDataview() {}
+
+module.exports = BaseDataview;
+
+BaseDataview.prototype.getResult = function(psql, override, callback) {
+    var self = this;
+    this.sql(psql, override, function(err, query) {
+        psql.query(query, function(err, result) {
+
+            if (err) {
+                return callback(err, result);
+            }
+
+            result = self.format(result, override);
+            result.type = self.getType();
+
+            return callback(null, result);
+
+        }, true); // use read-only transaction
+    });
+
+};
+
+BaseDataview.prototype.search = function(psql, userQuery, callback) {
+    return callback(null, this.format({ rows: [] }));
+};

lib/cartodb/models/dataview/factory.js

@@ -0,0 +1,18 @@
+var dataviews = require('./');
+
+var DataviewFactory = {
+    dataviews: Object.keys(dataviews).reduce(function(allDataviews, dataviewClassName) {
+        allDataviews[dataviewClassName.toLowerCase()] = dataviews[dataviewClassName];
+        return allDataviews;
+    }, {}),
+
+    getDataview: function(query, dataviewDefinition) {
+        var type = dataviewDefinition.type;
+        if (!this.dataviews[type]) {
+            throw new Error('Invalid dataview type: "' + type + '"');
+        }
+        return new this.dataviews[type](query, dataviewDefinition.options);
+    }
+};
+
+module.exports = DataviewFactory;

lib/cartodb/models/dataview/formula.js

@@ -0,0 +1,104 @@
+var _ = require('underscore');
+var BaseWidget = require('./base');
+var debug = require('debug')('windshaft:widget:formula');
+
+var dot = require('dot');
+dot.templateSettings.strip = false;
+
+var formulaQueryTpl = dot.template([
+    'SELECT',
+    '{{=it._operation}}({{=it._column}}) AS result,',
+    '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
+    'FROM ({{=it._query}}) _cdb_formula'
+].join('\n'));
+
+var VALID_OPERATIONS = {
+    count: true,
+    avg: true,
+    sum: true,
+    min: true,
+    max: true
+};
+
+var TYPE = 'formula';
+
+/**
+ {
+     type: 'formula',
+     options: {
+         column: 'name',
+         operation: 'count' // count, sum, avg
+     }
+ }
+ */
+function Formula(query, options) {
+    if (!_.isString(options.operation)) {
+        throw new Error('Formula expects `operation` in widget options');
+    }
+
+    if (!VALID_OPERATIONS[options.operation]) {
+        throw new Error("Formula does not support '" + options.operation + "' operation");
+    }
+
+    if (options.operation !== 'count' && !_.isString(options.column)) {
+        throw new Error('Formula expects `column` in widget options');
+    }
+
+    BaseWidget.apply(this);
+
+    this.query = query;
+    this.column = options.column || '1';
+    this.operation = options.operation;
+}
+
+Formula.prototype = new BaseWidget();
+Formula.prototype.constructor = Formula;
+
+module.exports = Formula;
+
+Formula.prototype.sql = function(psql, filters, override, callback) {
+    if (!callback) {
+        callback = override;
+        override = {};
+    }
+
+    var _query = this.query;
+    var formulaSql = formulaQueryTpl({
+        _query: _query,
+        _operation: this.operation,
+        _column: this.column
+    });
+
+    debug(formulaSql);
+
+    return callback(null, formulaSql);
+};
+
+Formula.prototype.format = function(result) {
+    var formattedResult = {
+        operation: this.operation,
+        result: 0,
+        nulls: 0
+    };
+
+    if (result.rows.length) {
+        formattedResult.operation = this.operation;
+        formattedResult.result = result.rows[0].result;
+        formattedResult.nulls = result.rows[0].nulls_count;
+    }
+
+    return formattedResult;
+};
+
+Formula.prototype.getType = function() {
+    return TYPE;
+};
+
+Formula.prototype.toString = function() {
+    return JSON.stringify({
+        _type: TYPE,
+        _query: this.query,
+        _column: this.column,
+        _operation: this.operation
+    });
+};

lib/cartodb/models/dataview/histogram.js

@@ -0,0 +1,297 @@
+var _ = require('underscore');
+var BaseWidget = require('./base');
+var debug = require('debug')('windshaft:dataview:histogram');
+
+var dot = require('dot');
+dot.templateSettings.strip = false;
+
+var columnTypeQueryTpl = dot.template(
+    'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1'
+);
+var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})");
+
+var BIN_MIN_NUMBER = 6;
+var BIN_MAX_NUMBER = 48;
+
+var basicsQueryTpl = dot.template([
+    'basics AS (',
+    '  SELECT',
+    '    max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,',
+    '    avg({{=it._column}}) AS avg_val, count(1) AS total_rows',
+    '  FROM ({{=it._query}}) _cdb_basics',
+    ')'
+].join(' \n'));
+
+var overrideBasicsQueryTpl = dot.template([
+    'basics AS (',
+    '  SELECT',
+    '    max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,',
+    '    avg({{=it._column}}) AS avg_val, count(1) AS total_rows',
+    '  FROM ({{=it._query}}) _cdb_basics',
+    ')'
+].join('\n'));
+
+var iqrQueryTpl = dot.template([
+    'iqrange AS (',
+    '  SELECT max(quartile_max) - min(quartile_max) AS iqr',
+    '  FROM (',
+    '    SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (',
+    '      SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}',
+    '    ) AS quartile',
+    '    FROM ({{=it._query}}) _cdb_rank) _cdb_quartiles',
+    '    WHERE quartile = 1 or quartile = 3',
+    '    GROUP BY quartile',
+    '  ) _cdb_iqr',
+    ')'
+].join('\n'));
+
+var binsQueryTpl = dot.template([
+    'bins AS (',
+    '  SELECT CASE WHEN total_rows = 0 OR iqr = 0',
+    '      THEN 1',
+    '      ELSE GREATEST(',
+    '        LEAST({{=it._minBins}}, CAST(total_rows AS INT)),',
+    '        LEAST(',
+    '          CAST(((max_val - min_val) / (2 * iqr * power(total_rows, 1/3))) AS INT),',
+    '          {{=it._maxBins}}',
+    '        )',
+    '      )',
+    '    END AS bins_number',
+    '  FROM basics, iqrange, ({{=it._query}}) _cdb_bins',
+    '  LIMIT 1',
+    ')'
+].join('\n'));
+
+var overrideBinsQueryTpl = dot.template([
+    'bins AS (',
+    '  SELECT {{=it._bins}} AS bins_number',
+    ')'
+].join('\n'));
+
+var nullsQueryTpl = dot.template([
+    'nulls AS (',
+    '  SELECT',
+    '  count(*) AS nulls_count',
+    '  FROM ({{=it._query}}) _cdb_histogram_nulls',
+    '  WHERE {{=it._column}} IS NULL',
+    ')'
+].join('\n'));
+
+var histogramQueryTpl = dot.template([
+    'SELECT',
+    '    (max_val - min_val) / cast(bins_number as float) AS bin_width,',
+    '    bins_number,',
+    '    nulls_count,',
+    '    avg_val,',
+    '    CASE WHEN min_val = max_val',
+    '      THEN 0',
+    '      ELSE GREATEST(1, LEAST(WIDTH_BUCKET({{=it._column}}, min_val, max_val, bins_number), bins_number)) - 1',
+    '    END AS bin,',
+    '    min({{=it._column}})::numeric AS min,',
+    '    max({{=it._column}})::numeric AS max,',
+    '    avg({{=it._column}})::numeric AS avg,',
+    '    count(*) AS freq',
+    'FROM ({{=it._query}}) _cdb_histogram, basics, nulls, bins',
+    'WHERE {{=it._column}} IS NOT NULL',
+    'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val',
+    'ORDER BY bin'
+].join('\n'));
+
+
+var TYPE = 'histogram';
+
+/**
+ {
+     type: 'histogram',
+     options: {
+         column: 'name',
+         bins: 10 // OPTIONAL
+     }
+ }
+ */
+function Histogram(query, options) {
+    if (!_.isString(options.column)) {
+        throw new Error('Histogram expects `column` in widget options');
+    }
+
+    this.query = query;
+    this.column = options.column;
+    this.bins = options.bins;
+
+    this._columnType = null;
+}
+
+Histogram.prototype = new BaseWidget();
+Histogram.prototype.constructor = Histogram;
+
+module.exports = Histogram;
+
+var DATE_OIDS = {
+    1082: true,
+    1114: true,
+    1184: true
+};
+
+Histogram.prototype.sql = function(psql, override, callback) {
+    if (!callback) {
+        callback = override;
+        override = {};
+    }
+
+    var self = this;
+
+    var _column = this.column;
+
+    var columnTypeQuery = columnTypeQueryTpl({
+        column: _column, query: this.query
+    });
+
+    if (this._columnType === null) {
+        psql.query(columnTypeQuery, function(err, result) {
+            // assume numeric, will fail later
+            self._columnType = 'numeric';
+            if (!err && !!result.rows[0]) {
+                var pgType = result.rows[0].pg_typeof;
+                if (DATE_OIDS.hasOwnProperty(pgType)) {
+                    self._columnType = 'date';
+                }
+            }
+            self.sql(psql, override, callback);
+        }, true); // use read-only transaction
+        return null;
+    }
+
+    if (this._columnType === 'date') {
+        _column = columnCastTpl({column: _column});
+    }
+
+    var _query = this.query;
+
+    var basicsQuery, binsQuery;
+
+    if (override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins')) {
+        debug('overriding with %j', override);
+        basicsQuery = overrideBasicsQueryTpl({
+            _query: _query,
+            _column: _column,
+            _start: override.start,
+            _end: override.end
+        });
+
+        binsQuery = [
+            overrideBinsQueryTpl({
+                _bins: override.bins
+            })
+        ].join(',\n');
+    } else {
+        basicsQuery = basicsQueryTpl({
+            _query: _query,
+            _column: _column
+        });
+
+        if (override && _.has(override, 'bins')) {
+            binsQuery = [
+                overrideBinsQueryTpl({
+                    _bins: override.bins
+                })
+            ].join(',\n');
+        } else {
+            binsQuery = [
+                iqrQueryTpl({
+                    _query: _query,
+                    _column: _column
+                }),
+                binsQueryTpl({
+                    _query: _query,
+                    _minBins: BIN_MIN_NUMBER,
+                    _maxBins: BIN_MAX_NUMBER
+                })
+            ].join(',\n');
+        }
+    }
+
+
+    var histogramSql = [
+        "WITH",
+        [
+            basicsQuery,
+            binsQuery,
+            nullsQueryTpl({
+                _query: _query,
+                _column: _column
+            })
+        ].join(',\n'),
+        histogramQueryTpl({
+            _query: _query,
+            _column: _column
+        })
+    ].join('\n');
+
+    debug(histogramSql);
+
+    return callback(null, histogramSql);
+};
+
+Histogram.prototype.format = function(result, override) {
+    override = override || {};
+    var buckets = [];
+
+    var binsCount = getBinsCount(override);
+    var width = getWidth(override);
+    var binsStart = getBinStart(override);
+    var nulls = 0;
+    var avg;
+
+    if (result.rows.length) {
+        var firstRow = result.rows[0];
+        binsCount = firstRow.bins_number;
+        width = firstRow.bin_width || width;
+        avg = firstRow.avg_val;
+        nulls = firstRow.nulls_count;
+        binsStart = override.hasOwnProperty('start') ? override.start : firstRow.min;
+
+        buckets = result.rows.map(function(row) {
+            return _.omit(row, 'bins_number', 'bin_width', 'nulls_count', 'avg_val');
+        });
+    }
+
+    return {
+        bin_width: width,
+        bins_count: binsCount,
+        bins_start: binsStart,
+        nulls: nulls,
+        avg: avg,
+        bins: buckets
+    };
+};
+
+function getBinStart(override) {
+    return override.start || 0;
+}
+
+function getBinsCount(override) {
+    return override.bins || 0;
+}
+
+function getWidth(override) {
+    var width = 0;
+    var binsCount = override.bins;
+
+    if (binsCount && Number.isFinite(override.start) && Number.isFinite(override.end)) {
+        width = (override.end - override.start) / binsCount;
+    }
+
+    return width;
+}
+
+Histogram.prototype.getType = function() {
+    return TYPE;
+};
+
+Histogram.prototype.toString = function() {
+    return JSON.stringify({
+        _type: TYPE,
+        _column: this.column,
+        _query: this.query
+    });
+};

lib/cartodb/models/dataview/index.js

@@ -0,0 +1,6 @@
+module.exports = {
+    Aggregation: require('./aggregation'),
+    Formula: require('./formula'),
+    Histogram: require('./histogram'),
+    List: require('./list')
+};

lib/cartodb/models/dataview/list.js

@@ -0,0 +1,66 @@
+var dot = require('dot');
+dot.templateSettings.strip = false;
+
+var BaseWidget = require('./base');
+
+var TYPE = 'list';
+
+var listSqlTpl = dot.template('select {{=it._columns}} from ({{=it._query}}) as _cdb_list');
+
+/**
+{
+    type: 'list',
+    options: {
+        columns: ['name', 'description']
+    }
+}
+*/
+
+function List(query, options) {
+    options = options || {};
+
+    if (!Array.isArray(options.columns)) {
+        throw new Error('List expects `columns` array in widget options');
+    }
+
+    BaseWidget.apply(this);
+
+    this.query = query;
+    this.columns = options.columns;
+}
+
+List.prototype = new BaseWidget();
+List.prototype.constructor = List;
+
+module.exports = List;
+
+List.prototype.sql = function(psql, filters, override, callback) {
+    if (!callback) {
+        callback = override;
+    }
+
+    var listSql = listSqlTpl({
+        _query: this.query,
+        _columns: this.columns.join(', ')
+    });
+
+    return callback(null, listSql);
+};
+
+List.prototype.format = function(result) {
+    return {
+        rows: result.rows
+    };
+};
+
+List.prototype.getType = function() {
+    return TYPE;
+};
+
+List.prototype.toString = function() {
+    return JSON.stringify({
+        _type: TYPE,
+        _query: this.query,
+        _columns: this.columns.join(', ')
+    });
+};

lib/cartodb/models/filter/bbox.js

@@ -0,0 +1,122 @@
+var debug = require('debug')('windshaft:filter:bbox');
+var dot = require('dot');
+dot.templateSettings.strip = false;
+
+var filterQueryTpl = dot.template([
+    'SELECT * FROM ({{=it._sql}}) _cdb_bbox_filter',
+    'WHERE {{=it._filters}}'
+].join('\n'));
+
+var bboxFilterTpl = dot.template(
+    '{{=it._column}} && ST_Transform(ST_MakeEnvelope({{=it._bbox}}, 4326), {{=it._srid}})'
+);
+
+var LATITUDE_MAX_VALUE = 85.0511287798066;
+var LONGITUDE_LOWER_BOUND = -180;
+var LONGITUDE_UPPER_BOUND = 180;
+var LONGITUDE_RANGE = LONGITUDE_UPPER_BOUND - LONGITUDE_LOWER_BOUND;
+
+/**
+ Definition
+ {
+     "type”: "bbox",
+     "options": {
+        "column": "the_geom_webmercator",
+        "srid": 3857
+     }
+ }
+
+ Params
+ {
+     “bbox”: "west,south,east,north"
+ }
+ */
+function BBox(filterDefinition, filterParams) {
+    var bbox = filterParams.bbox;
+
+    if (!bbox) {
+        throw new Error('BBox filter expects to have a bbox param');
+    }
+
+    var bboxElements = bbox.split(',').map(function(e) { return +e; });
+
+    validateBboxElements(bboxElements);
+
+    this.column = filterDefinition.column || 'the_geom_webmercator';
+    this.srid = filterDefinition.srid || 3857;
+
+    // Latitudes must be within max extent
+    var south = Math.max(bboxElements[1], -LATITUDE_MAX_VALUE);
+    var north = Math.min(bboxElements[3], LATITUDE_MAX_VALUE);
+
+    // Longitudes crossing 180º need another approach
+    var adjustedLongitudeRange = adjustLongitudeRange([bboxElements[0], bboxElements[2]]);
+    var west = adjustedLongitudeRange[0];
+    var east = adjustedLongitudeRange[1];
+
+    this.bboxes = getBoundingBoxes(west, south, east, north);
+}
+
+function getBoundingBoxes(west, south, east, north) {
+    var bboxes = [];
+
+    if (east - west >= 360) {
+        bboxes.push([-180, south, 180, north]);
+    } else if (west >= -180 && east <= 180) {
+        bboxes.push([west, south, east, north]);
+    } else {
+        bboxes.push([west, south, 180, north]);
+        bboxes.push([-180, south, east % 180, north]);
+    }
+
+    return bboxes;
+}
+
+function validateBboxElements(bboxElements) {
+    var isNumericBbox = bboxElements
+        .map(function(n) { return Number.isFinite(n); })
+        .reduce(function(allFinite, isFinite) {
+            if (!allFinite) {
+                return false;
+            }
+            return isFinite;
+        }, true);
+
+    if (bboxElements.length !== 4 || !isNumericBbox) {
+        throw new Error('Invalid bbox filter, expected format="west,south,east,north"');
+    }
+}
+
+function adjustLongitudeRange(we) {
+    var west = we[0];
+    west -= LONGITUDE_LOWER_BOUND;
+    west = west - (LONGITUDE_RANGE * Math.floor(west / LONGITUDE_RANGE)) + LONGITUDE_LOWER_BOUND;
+
+    var longitudeRange = Math.min(we[1] - we[0], 360);
+
+    return [west, west + longitudeRange];
+}
+
+module.exports = BBox;
+
+module.exports.adjustLongitudeRange = adjustLongitudeRange;
+module.exports.LATITUDE_MAX_VALUE = LATITUDE_MAX_VALUE;
+module.exports.LONGITUDE_MAX_VALUE = LONGITUDE_UPPER_BOUND;
+
+
+BBox.prototype.sql = function(rawSql) {
+    var bboxSql = filterQueryTpl({
+        _sql: rawSql,
+        _filters: this.bboxes.map(function(bbox) {
+            return bboxFilterTpl({
+                _column: this.column,
+                _bbox: bbox.join(','),
+                _srid: this.srid
+            });
+        }.bind(this)).join(' OR ')
+    });
+
+    debug(bboxSql);
+
+    return bboxSql;
+};

lib/cartodb/models/layergroup_token.js

@@ -0,0 +1,29 @@
+/**
+ * @param {String} token might match the following pattern: {user}@{tpl_id}@{token}:{cache_buster}
+ */
+function parse(token) {
+    var signer, cacheBuster;
+
+    var tokenSplit = token.split(':');
+
+    token = tokenSplit[0];
+    if (tokenSplit.length > 1) {
+        cacheBuster = tokenSplit[1];
+    }
+
+    tokenSplit = token.split('@');
+    if ( tokenSplit.length > 1 ) {
+        signer = tokenSplit.shift();
+        if ( tokenSplit.length > 1 ) {
+            /*var template_hash = */tokenSplit.shift(); // unused
+        }
+        token = tokenSplit.shift();
+    }
+
+    return {
+        token: token,
+        signer: signer,
+        cacheBuster: cacheBuster
+    };
+}
+module.exports.parse = parse;

lib/cartodb/models/mapconfig/create_layergroup_provider.js

@@ -0,0 +1,48 @@
+var assert = require('assert');
+var step = require('step');
+
+var MapStoreMapConfigProvider = require('./map_store_provider');
+
+/**
+ * @param {MapConfig} mapConfig
+ * @param {String} user
+ * @param {UserLimitsApi} userLimitsApi
+ * @param {Object} params
+ * @constructor
+ * @type {CreateLayergroupMapConfigProvider}
+ */
+function CreateLayergroupMapConfigProvider(mapConfig, user, userLimitsApi, params) {
+    this.mapConfig = mapConfig;
+    this.user = user;
+    this.userLimitsApi = userLimitsApi;
+    this.params = params;
+    this.cacheBuster = params.cache_buster || 0;
+}
+
+module.exports = CreateLayergroupMapConfigProvider;
+
+CreateLayergroupMapConfigProvider.prototype.getMapConfig = function(callback) {
+    var self = this;
+    var context = {};
+    step(
+        function prepareContextLimits() {
+            self.userLimitsApi.getRenderLimits(self.user, this);
+        },
+        function handleRenderLimits(err, renderLimits) {
+            assert.ifError(err);
+            context.limits = renderLimits;
+            return null;
+        },
+        function finish(err) {
+            return callback(err, self.mapConfig, self.params, context);
+        }
+    );
+};
+
+CreateLayergroupMapConfigProvider.prototype.getKey = MapStoreMapConfigProvider.prototype.getKey;
+
+CreateLayergroupMapConfigProvider.prototype.getCacheBuster = MapStoreMapConfigProvider.prototype.getCacheBuster;
+
+CreateLayergroupMapConfigProvider.prototype.filter = MapStoreMapConfigProvider.prototype.filter;
+
+CreateLayergroupMapConfigProvider.prototype.createKey = MapStoreMapConfigProvider.prototype.createKey;

lib/cartodb/models/mapconfig/map_store_provider.js

@@ -0,0 +1,77 @@
+var _ = require('underscore');
+var assert = require('assert');
+var dot = require('dot');
+var step = require('step');
+
+/**
+ * @param {MapStore} mapStore
+ * @param {String} user
+ * @param {UserLimitsApi} userLimitsApi
+ * @param {Object} params
+ * @constructor
+ * @type {MapStoreMapConfigProvider}
+ */
+function MapStoreMapConfigProvider(mapStore, user, userLimitsApi, params) {
+    this.mapStore = mapStore;
+    this.user = user;
+    this.userLimitsApi = userLimitsApi;
+    this.params = params;
+    this.token = params.token;
+    this.cacheBuster = params.cache_buster || 0;
+}
+
+module.exports = MapStoreMapConfigProvider;
+
+MapStoreMapConfigProvider.prototype.getMapConfig = function(callback) {
+    var self = this;
+    var context = {};
+    step(
+        function prepareContextLimits() {
+            self.userLimitsApi.getRenderLimits(self.user, this);
+        },
+        function handleRenderLimits(err, renderLimits) {
+            assert.ifError(err);
+            context.limits = renderLimits;
+            return null;
+        },
+        function loadMapConfig(err) {
+            assert.ifError(err);
+            self.mapStore.load(self.token, this);
+        },
+        function finish(err, mapConfig) {
+            return callback(err, mapConfig, self.params, context);
+        }
+    );
+};
+
+MapStoreMapConfigProvider.prototype.getKey = function() {
+    return this.createKey(false);
+};
+
+MapStoreMapConfigProvider.prototype.getCacheBuster = function() {
+    return this.cacheBuster;
+};
+
+MapStoreMapConfigProvider.prototype.filter = function(key) {
+    var regex = new RegExp('^' + this.createKey(true) + '.*');
+    return key && key.match(regex);
+};
+
+// Configure bases for cache keys suitable for string interpolation
+var baseKey   = '{{=it.dbname}}:{{=it.token}}';
+var rendererKey = baseKey + ':{{=it.dbuser}}:{{=it.format}}:{{=it.layer}}:{{=it.scale_factor}}';
+
+var baseKeyTpl = dot.template(baseKey);
+var rendererKeyTpl = dot.template(rendererKey);
+
+MapStoreMapConfigProvider.prototype.createKey = function(base) {
+    var tplValues = _.defaults({}, this.params, {
+        dbname: '',
+        token: '',
+        dbuser: '',
+        format: '',
+        layer: '',
+        scale_factor: 1
+    });
+    return (base) ? baseKeyTpl(tplValues) : rendererKeyTpl(tplValues);
+};

lib/cartodb/models/mapconfig/named_map_provider.js

@@ -0,0 +1,347 @@
+var _ = require('underscore');
+var assert = require('assert');
+var crypto = require('crypto');
+var dot = require('dot');
+var step = require('step');
+var MapConfig = require('windshaft').model.MapConfig;
+var templateName = require('../../backends/template_maps').templateName;
+var QueryTables = require('cartodb-query-tables');
+
+/**
+ * @constructor
+ * @type {NamedMapMapConfigProvider}
+ */
+function NamedMapMapConfigProvider(templateMaps, pgConnection, metadataBackend, userLimitsApi,
+                                   namedLayersAdapter, overviewsAdapter, turboCartoAdapter, analysisMapConfigAdapter,
+                                   owner, templateId, config, authToken, params) {
+    this.templateMaps = templateMaps;
+    this.pgConnection = pgConnection;
+    this.metadataBackend = metadataBackend;
+    this.userLimitsApi = userLimitsApi;
+    this.namedLayersAdapter = namedLayersAdapter;
+    this.turboCartoAdapter = turboCartoAdapter;
+    this.analysisMapConfigAdapter = analysisMapConfigAdapter;
+    this.overviewsAdapter = overviewsAdapter;
+
+    this.owner = owner;
+    this.templateName = templateName(templateId);
+    this.config = config;
+    this.authToken = authToken;
+    this.params = params;
+
+    this.cacheBuster = Date.now();
+
+    // use template after call to mapConfig
+    this.template = null;
+
+    this.affectedTablesAndLastUpdate = null;
+
+    // providing
+    this.err = null;
+    this.mapConfig = null;
+    this.rendererParams = null;
+    this.context = {};
+    this.analysesResults = [];
+}
+
+module.exports = NamedMapMapConfigProvider;
+
+NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) {
+    if (!!this.err || this.mapConfig !== null) {
+        return callback(this.err, this.mapConfig, this.rendererParams, this.context);
+    }
+
+    var self = this;
+
+    var mapConfig = null;
+    var datasource = null;
+    var rendererParams;
+    var apiKey;
+
+    step(
+        function getTemplate() {
+            self.getTemplate(this);
+        },
+        function prepareDbParams(err, tpl) {
+            assert.ifError(err);
+            self.template = tpl;
+
+            rendererParams = _.extend({}, self.params, {
+                user: self.owner
+            });
+            self.setDBParams(self.owner, rendererParams, this);
+        },
+        function getUserApiKey(err) {
+            assert.ifError(err);
+            self.metadataBackend.getUserMapKey(self.owner, this);
+        },
+        function prepareParams(err, _apiKey) {
+            assert.ifError(err);
+
+            apiKey = _apiKey;
+
+            var templateParams = {};
+            if (self.config) {
+                try {
+                    templateParams = _.isString(self.config) ? JSON.parse(self.config) : self.config;
+                } catch (e) {
+                    throw new Error('malformed config parameter, should be a valid JSON');
+                }
+            }
+
+            return templateParams;
+        },
+        function instantiateTemplate(err, templateParams) {
+            assert.ifError(err);
+            return self.templateMaps.instance(self.template, templateParams);
+        },
+        function prepareAnalysisLayers(err, requestMapConfig) {
+            assert.ifError(err);
+            var analysisConfiguration = {
+                db: {
+                    host: rendererParams.dbhost,
+                    port: rendererParams.dbport,
+                    dbname: rendererParams.dbname,
+                    user: rendererParams.dbuser,
+                    pass: rendererParams.dbpassword
+                },
+                batch: {
+                    username: self.owner,
+                    apiKey: apiKey
+                }
+            };
+
+            var filters = {};
+            if (self.params.filters) {
+                try {
+                    filters = JSON.parse(self.params.filters);
+                } catch (e) {
+                    // ignore
+                }
+            }
+            self.analysisMapConfigAdapter.getMapConfig(analysisConfiguration, requestMapConfig, filters, this);
+        },
+        function prepareLayergroup(err, _mapConfig, analysesResults) {
+            assert.ifError(err);
+            var next = this;
+            self.analysesResults = analysesResults || [];
+            self.namedLayersAdapter.getLayers(self.owner, _mapConfig.layers, self.pgConnection,
+                function(err, layers, datasource) {
+                    if (err) {
+                        return next(err);
+                    }
+
+                    if (layers) {
+                        _mapConfig.layers = layers;
+                    }
+                    return next(null, _mapConfig, datasource);
+                }
+            );
+        },
+        function addOverviewsInformation(err, _mapConfig, datasource) {
+            assert.ifError(err);
+            var next = this;
+
+            self.overviewsAdapter.getLayers(self.owner, _mapConfig.layers, function(err, layers) {
+                if (err) {
+                    return next(err);
+                }
+
+                if (layers) {
+                    _mapConfig.layers = layers;
+                }
+
+                return next(null, _mapConfig, datasource);
+            });
+        },
+        function parseTurboCarto(err, _mapConfig, datasource) {
+            assert.ifError(err);
+            var next = this;
+
+            self.turboCartoAdapter.getLayers(self.owner, _mapConfig.layers, function (err, layers) {
+                if (err) {
+                    return next(err);
+                }
+
+                if (layers) {
+                    _mapConfig.layers = layers;
+                }
+
+                return next(null, _mapConfig, datasource);
+            });
+        },
+        function prepareContextLimits(err, _mapConfig, _datasource) {
+            assert.ifError(err);
+            mapConfig = _mapConfig;
+            datasource = _datasource;
+            self.userLimitsApi.getRenderLimits(self.owner, this);
+        },
+        function cacheAndReturnMapConfig(err, renderLimits) {
+            self.err = err;
+            self.mapConfig = (mapConfig === null) ? null : new MapConfig(mapConfig, datasource);
+            self.rendererParams = rendererParams;
+            self.context.limits = renderLimits || {};
+            return callback(self.err, self.mapConfig, self.rendererParams, self.context);
+        }
+    );
+};
+
+NamedMapMapConfigProvider.prototype.getTemplate = function(callback) {
+    var self = this;
+
+    if (!!this.err || this.template !== null) {
+        return callback(this.err, this.template);
+    }
+
+    step(
+        function getTemplate() {
+            self.templateMaps.getTemplate(self.owner, self.templateName, this);
+        },
+        function checkExists(err, tpl) {
+            assert.ifError(err);
+            if (!tpl) {
+                var notFoundErr = new Error(
+                        "Template '" + self.templateName + "' of user '" + self.owner + "' not found"
+                );
+                notFoundErr.http_status = 404;
+                throw notFoundErr;
+            }
+            return tpl;
+        },
+        function checkAuthorized(err, tpl) {
+            assert.ifError(err);
+
+            var authorized = false;
+            try {
+                authorized = self.templateMaps.isAuthorized(tpl, self.authToken);
+            } catch (err) {
+                // we catch to add http_status
+                var authorizationFailedErr = new Error('Failed to authorize template');
+                authorizationFailedErr.http_status = 403;
+                throw authorizationFailedErr;
+            }
+            if ( ! authorized ) {
+                var unauthorizedErr = new Error('Unauthorized template instantiation');
+                unauthorizedErr.http_status = 403;
+                throw unauthorizedErr;
+            }
+
+            return tpl;
+        },
+        function cacheAndReturnTemplate(err, template) {
+            self.err = err;
+            self.template = template;
+            return callback(self.err, self.template);
+        }
+    );
+};
+
+NamedMapMapConfigProvider.prototype.getKey = function() {
+    return this.createKey(false);
+};
+
+NamedMapMapConfigProvider.prototype.getCacheBuster = function() {
+    return this.cacheBuster;
+};
+
+NamedMapMapConfigProvider.prototype.reset = function() {
+    this.template = null;
+
+    this.affectedTablesAndLastUpdate = null;
+
+    this.err = null;
+    this.mapConfig = null;
+
+    this.cacheBuster = Date.now();
+};
+
+NamedMapMapConfigProvider.prototype.filter = function(key) {
+    var regex = new RegExp('^' + this.createKey(true) + '.*');
+    return key && key.match(regex);
+};
+
+// Configure bases for cache keys suitable for string interpolation
+var baseKey = '{{=it.dbname}}:{{=it.owner}}:{{=it.templateName}}';
+var rendererKey = baseKey + ':{{=it.authToken}}:{{=it.configHash}}:{{=it.format}}:{{=it.layer}}:{{=it.scale_factor}}';
+
+var baseKeyTpl = dot.template(baseKey);
+var rendererKeyTpl = dot.template(rendererKey);
+
+NamedMapMapConfigProvider.prototype.createKey = function(base) {
+    var tplValues = _.defaults({}, this.params, {
+        dbname: '',
+        owner: this.owner,
+        templateName: this.templateName,
+        authToken: this.authToken || '',
+        configHash: configHash(this.config),
+        layer: '',
+        scale_factor: 1
+    });
+    return (base) ? baseKeyTpl(tplValues) : rendererKeyTpl(tplValues);
+};
+
+function configHash(config) {
+    if (!config) {
+        return '';
+    }
+    return crypto.createHash('md5').update(JSON.stringify(config)).digest('hex').substring(0,8);
+}
+
+module.exports.configHash = configHash;
+
+NamedMapMapConfigProvider.prototype.setDBParams = function(cdbuser, params, callback) {
+    var self = this;
+    step(
+        function setAuth() {
+            self.pgConnection.setDBAuth(cdbuser, params, this);
+        },
+        function setConn(err) {
+            assert.ifError(err);
+            self.pgConnection.setDBConn(cdbuser, params, this);
+        },
+        function finish(err) {
+            callback(err);
+        }
+    );
+};
+
+NamedMapMapConfigProvider.prototype.getTemplateName = function() {
+    return this.templateName;
+};
+
+NamedMapMapConfigProvider.prototype.getAffectedTablesAndLastUpdatedTime = function(callback) {
+    var self = this;
+
+    if (this.affectedTablesAndLastUpdate !== null) {
+        return callback(null, this.affectedTablesAndLastUpdate);
+    }
+
+    step(
+        function getMapConfig() {
+            self.getMapConfig(this);
+        },
+        function getSql(err, mapConfig) {
+            assert.ifError(err);
+            return mapConfig.getLayers().map(function(layer) {
+                return layer.options.sql;
+            }).join(';');
+        },
+        function getAffectedTables(err, sql) {
+            assert.ifError(err);
+            step(
+                function getConnection() {
+                    self.pgConnection.getConnection(self.owner, this);
+                },
+                function getAffectedTables(err, connection) {
+                    assert.ifError(err);
+                    QueryTables.getAffectedTablesFromQuery(connection, sql, this);
+                },
+                this
+            );
+        },
+        function finish(err, result) {
+            self.affectedTablesAndLastUpdate = result;
+            return callback(err, result);
+        }
+    );
+};

lib/cartodb/models/mapconfig_named_layers_adapter.js

@@ -0,0 +1,124 @@
+var queue = require('queue-async');
+var _ = require('underscore');
+var Datasource = require('windshaft').model.Datasource;
+
+function MapConfigNamedLayersAdapter(templateMaps) {
+    this.templateMaps = templateMaps;
+}
+
+module.exports = MapConfigNamedLayersAdapter;
+
+MapConfigNamedLayersAdapter.prototype.getLayers = function(username, layers, dbMetadata, callback) {
+    var self = this;
+
+    if (!layers) {
+        return callback(null);
+    }
+
+    var adaptLayersQueue = queue(layers.length);
+
+    function adaptLayer(layer, done) {
+        if (isNamedTypeLayer(layer)) {
+
+            if (!layer.options.name) {
+                return done(new Error('Missing Named Map `name` in layer options'));
+            }
+
+            var templateName = layer.options.name;
+            var templateConfigParams = layer.options.config || {};
+            var templateAuthTokens = layer.options.auth_tokens;
+
+            self.templateMaps.getTemplate(username, templateName, function(err,  template) {
+                if (err || !template) {
+                    return done(new Error("Template '" + templateName + "' of user '" + username + "' not found"));
+                }
+
+                if (self.templateMaps.isAuthorized(template, templateAuthTokens)) {
+                    var nestedNamedLayers = template.layergroup.layers.filter(function(layer) {
+                        return layer.type === 'named';
+                    });
+
+                    if (nestedNamedLayers.length > 0) {
+                        var nestedNamedMapsError = new Error('Nested named layers are not allowed');
+                        // nestedNamedMapsError.http_status = 400;
+                        return done(nestedNamedMapsError);
+                    }
+
+                    try {
+                        var templateLayergroupConfig = self.templateMaps.instance(template, templateConfigParams);
+                        return done(null, {
+                            datasource: true,
+                            layers: templateLayergroupConfig.layers
+                        });
+                    } catch (err) {
+                        return done(err);
+                    }
+                } else {
+                    var unauthorizedError = new Error("Unauthorized '" + templateName + "' template instantiation");
+                    unauthorizedError.http_status = 403;
+                    return done(unauthorizedError);
+                }
+            });
+
+        } else {
+            return done(null, {
+                datasource: false,
+                layers: [layer]
+            });
+        }
+    }
+
+    var datasourceBuilder = new Datasource.Builder();
+
+    function layersAdaptQueueFinish(err, layersResults) {
+        if (err) {
+            return callback(err);
+        }
+
+        if (!layersResults || layersResults.length === 0) {
+            return callback(new Error('Missing layers array from layergroup config'));
+        }
+
+        var layers = [],
+            currentLayerIndex = 0;
+
+        layersResults.forEach(function(layersResult) {
+
+            layersResult.layers.forEach(function(layer) {
+                layers.push(layer);
+                if (layersResult.datasource) {
+                    datasourceBuilder.withLayerDatasource(currentLayerIndex, {
+                        user: dbAuth.dbuser
+                    });
+                }
+                currentLayerIndex++;
+            });
+
+        });
+
+        return callback(null, layers, datasourceBuilder.build());
+    }
+
+
+    var dbAuth = {};
+
+    if (_.some(layers, isNamedTypeLayer)) {
+        // Lazy load dbAuth
+        dbMetadata.setDBAuth(username, dbAuth, function(err) {
+            if (err) {
+                return callback(err);
+            }
+            layers.forEach(function(layer) {
+                adaptLayersQueue.defer(adaptLayer, layer);
+            });
+            adaptLayersQueue.awaitAll(layersAdaptQueueFinish);
+        });
+    } else {
+        return callback(null, layers, datasourceBuilder.build());
+    }
+
+};
+
+function isNamedTypeLayer(layer) {
+    return layer.type === 'named';
+}

lib/cartodb/models/mapconfig_overviews_adapter.js

@@ -0,0 +1,53 @@
+var queue = require('queue-async');
+var _ = require('underscore');
+
+function MapConfigOverviewsAdapter(overviewsMetadataApi) {
+    this.overviewsMetadataApi = overviewsMetadataApi;
+}
+
+module.exports = MapConfigOverviewsAdapter;
+
+MapConfigOverviewsAdapter.prototype.getLayers = function(username, layers, callback) {
+  var self = this;
+
+  if (!layers || layers.length === 0) {
+      return callback(null, layers);
+  }
+
+  var augmentLayersQueue = queue(layers.length);
+
+  function augmentLayer(layer, done) {
+      if ( layer.type !== 'mapnik' && layer.type !== 'cartodb' ) {
+          return done(null, layer);
+      }
+      self.overviewsMetadataApi.getOverviewsMetadata(username, layer.options.sql, function(err, metadata){
+           if (err) {
+               done(err, layer);
+           } else {
+               if ( !_.isEmpty(metadata) ) {
+                   layer = _.extend({}, layer);
+                   layer.options = _.extend({}, layer.options, { query_rewrite_data: { overviews: metadata } });
+               }
+               done(null, layer);
+           }
+      });
+  }
+
+  function layersAugmentQueueFinish(err, layers) {
+    if (err) {
+        return callback(err);
+    }
+
+    if (!layers || layers.length === 0) {
+        return callback(new Error('Missing layers array from layergroup config'));
+    }
+
+    return callback(null, layers);
+  }
+
+  layers.forEach(function(layer) {
+      augmentLayersQueue.defer(augmentLayer, layer);
+  });
+  augmentLayersQueue.awaitAll(layersAugmentQueueFinish);
+
+};

lib/cartodb/monitoring/health_check.js

@@ -1,90 +1,19 @@
-var _    = require('underscore'),
-    dot  = require('dot'),
-    fs   = require('fs'),
-    path = require('path'),
-    Step = require('step');
+var fs = require('fs');
 
-function HealthCheck(metadataBackend, tilelive) {
-    this.metadataBackend = metadataBackend;
-    this.tilelive = tilelive;
+function HealthCheck(disableFile) {
+    this.disableFile = disableFile;
 }
 
 module.exports = HealthCheck;
 
 
-var mapnikOptions = {
-    query: {
-        metatile: 1,
-        poolSize: 4,
-        bufferSize: 64
-    },
-    protocol: 'mapnik:',
-    slashes: true,
-    xml: null
-};
-
-var xmlTemplate = dot.template(fs.readFileSync(path.resolve(__dirname, 'map-config.xml'), 'utf-8'));
-
-HealthCheck.prototype.check = function(config, callback) {
-
-    var self = this,
-        startTime,
-        result = {
-            redis: {
-                ok: false
-            },
-            mapnik: {
-                ok: false
-            },
-            tile: {
-                ok: false
-            }
-        };
-    mapnikXmlParams = config;
-
-    Step(
-        function getDBParams() {
-            startTime = Date.now();
-            self.metadataBackend.getAllUserDBParams(config.username, this);
-        },
-        function loadMapnik(err, dbParams) {
-            if (err) {
-                throw err;
-            }
-            result.redis = {
-                ok: !err,
-                elapsed: Date.now() - startTime,
-                size: Object.keys(dbParams).length
-            };
-            mapnikOptions.xml = xmlTemplate(mapnikXmlParams);
-
-            startTime = Date.now();
-            self.tilelive.load(mapnikOptions, this);
-        },
-        function getTile(err, source) {
-            if (err) {
-                throw err;
-            }
-
-            result.mapnik = {
-                ok: !err,
-                elapsed: Date.now() - startTime
-            };
-
-            startTime = Date.now();
-            source.getTile(config.z, config.x, config.y, this);
-        },
-        function handleTile(err, tile) {
-            result.tile = {
-                ok: !err
-            };
-
-            if (tile) {
-                result.tile.elapsed = Date.now() - startTime;
-                result.tile.size = tile.length;
-            }
-
-            callback(err, result);
+HealthCheck.prototype.check = function(callback) {
+    fs.readFile(this.disableFile, function handleDisabledFile(err, data) {
+        var disabledError = null;
+        if (!err) {
+            disabledError = new Error(data || 'Unknown error');
+            disabledError.http_status = 503;
         }
-    );
+        return callback(disabledError);
+    });
 };

lib/cartodb/monitoring/map-config.xml

@@ -1,4 +0,0 @@
-<Map
-    background-color="#c33"
-    srs="+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs">
-</Map>

lib/cartodb/server.js

@@ -0,0 +1,335 @@
+var express = require('express');
+var bodyParser = require('body-parser');
+var RedisPool = require('redis-mpool');
+var cartodbRedis = require('cartodb-redis');
+var _ = require('underscore');
+var debug = require('debug')('windshaft:cartodb');
+
+var controller = require('./controllers');
+
+var SurrogateKeysCache = require('./cache/surrogate_keys_cache');
+var NamedMapsCacheEntry = require('./cache/model/named_maps_entry');
+var VarnishHttpCacheBackend = require('./cache/backend/varnish_http');
+var FastlyCacheBackend = require('./cache/backend/fastly');
+
+var StatsClient = require('./stats/client');
+var Profiler = require('./stats/profiler_proxy');
+var RendererStatsReporter = require('./stats/reporter/renderer');
+
+var windshaft = require('windshaft');
+var mapnik = windshaft.mapnik;
+
+var TemplateMaps = require('./backends/template_maps.js');
+var OverviewsMetadataApi = require('./api/overviews_metadata_api');
+var UserLimitsApi = require('./api/user_limits_api');
+var AuthApi = require('./api/auth_api');
+var LayergroupAffectedTablesCache = require('./cache/layergroup_affected_tables');
+var NamedMapProviderCache = require('./cache/named_map_provider_cache');
+var PgQueryRunner = require('./backends/pg_query_runner');
+var PgConnection = require('./backends/pg_connection');
+
+var AnalysisBackend = require('./backends/analysis');
+
+var timeoutErrorTilePath = __dirname + '/../../assets/render-timeout-fallback.png';
+var timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, {encoding: null});
+
+var MapConfigOverviewsAdapter = require('./models/mapconfig_overviews_adapter');
+
+var TurboCartoParser = require('./utils/style/turbo-carto-parser');
+var TurboCartoAdapter = require('./utils/style/turbo-carto-adapter');
+
+module.exports = function(serverOptions) {
+    // Make stats client globally accessible
+    global.statsClient = StatsClient.getInstance(serverOptions.statsd);
+
+    var redisPool = new RedisPool(_.defaults(global.environment.redis, {
+        name: 'windshaft-server',
+        unwatchOnRelease: false,
+        noReadyCheck: true
+    }));
+
+    redisPool.on('status', function(status) {
+        var keyPrefix = 'windshaft.redis-pool.' + status.name + '.db' + status.db + '.';
+        global.statsClient.gauge(keyPrefix + 'count', status.count);
+        global.statsClient.gauge(keyPrefix + 'unused', status.unused);
+        global.statsClient.gauge(keyPrefix + 'waiting', status.waiting);
+    });
+
+    var metadataBackend = cartodbRedis({pool: redisPool});
+    var pgConnection = new PgConnection(metadataBackend);
+    var pgQueryRunner = new PgQueryRunner(pgConnection);
+    var overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner);
+    var userLimitsApi = new UserLimitsApi(metadataBackend, {
+        limits: {
+            cacheOnTimeout: serverOptions.renderer.mapnik.limits.cacheOnTimeout || false,
+            render: serverOptions.renderer.mapnik.limits.render || 0
+        }
+    });
+
+    var templateMaps = new TemplateMaps(redisPool, {
+        max_user_templates: global.environment.maxUserTemplates
+    });
+
+    var surrogateKeysCache = new SurrogateKeysCache(surrogateKeysCacheBackends(serverOptions));
+
+    function invalidateNamedMap (owner, templateName) {
+        var startTime = Date.now();
+        surrogateKeysCache.invalidate(new NamedMapsCacheEntry(owner, templateName), function(err) {
+            var logMessage = JSON.stringify({
+                username: owner,
+                type: 'named_map_invalidation',
+                elapsed: Date.now() - startTime,
+                error: !!err ? JSON.stringify(err.message) : undefined
+            });
+            if (err) {
+                global.logger.warn(logMessage);
+            } else {
+                global.logger.info(logMessage);
+            }
+        });
+    }
+    ['update', 'delete'].forEach(function(eventType) {
+        templateMaps.on(eventType, invalidateNamedMap);
+    });
+
+    serverOptions.grainstore.mapnik_version = mapnikVersion(serverOptions);
+
+    validateOptions(serverOptions);
+
+    bootstrapFonts(serverOptions);
+
+    // initialize express server
+    var app = bootstrap(serverOptions);
+    // Extend windshaft with all the elements of the options object
+    _.extend(app, serverOptions);
+
+    var mapStore  = new windshaft.storage.MapStore({
+        pool: redisPool,
+        expire_time: serverOptions.grainstore.default_layergroup_ttl
+    });
+
+    var onTileErrorStrategy;
+    if (global.environment.enabledFeatures.onTileErrorStrategy !== false) {
+        onTileErrorStrategy = function onTileErrorStrategy$TimeoutTile(err, tile, headers, stats, format, callback) {
+            if (err && err.message === 'Render timed out' && format === 'png') {
+                return callback(null, timeoutErrorTile, { 'Content-Type': 'image/png' }, {});
+            } else {
+                return callback(err, tile, headers, stats);
+            }
+        };
+    }
+
+    var rendererFactory = new windshaft.renderer.Factory({
+        onTileErrorStrategy: onTileErrorStrategy,
+        mapnik: {
+            redisPool: redisPool,
+            grainstore: serverOptions.grainstore,
+            mapnik: serverOptions.renderer.mapnik
+        },
+        http: serverOptions.renderer.http
+    });
+
+    // initialize render cache
+    var rendererCacheOpts = _.defaults(serverOptions.renderCache || {}, {
+        ttl: 60000, // 60 seconds TTL by default
+        statsInterval: 60000 // reports stats every milliseconds defined here
+    });
+    var rendererCache = new windshaft.cache.RendererCache(rendererFactory, rendererCacheOpts);
+    var rendererStatsReporter = new RendererStatsReporter(rendererCache, rendererCacheOpts.statsInterval);
+    rendererStatsReporter.start();
+
+    var attributesBackend = new windshaft.backend.Attributes();
+    var previewBackend = new windshaft.backend.Preview(rendererCache);
+    var tileBackend = new windshaft.backend.Tile(rendererCache);
+    var mapValidatorBackend = new windshaft.backend.MapValidator(tileBackend, attributesBackend);
+    var mapBackend = new windshaft.backend.Map(rendererCache, mapStore, mapValidatorBackend);
+    var analysisBackend = new AnalysisBackend(serverOptions.analysis);
+
+    var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache();
+    app.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
+
+    var overviewsAdapter = new MapConfigOverviewsAdapter(overviewsMetadataApi);
+
+    var turboCartoParser = new TurboCartoParser(pgQueryRunner);
+    var turboCartoAdapter = new TurboCartoAdapter(turboCartoParser);
+
+    var namedMapProviderCache = new NamedMapProviderCache(
+        templateMaps,
+        pgConnection,
+        metadataBackend,
+        userLimitsApi,
+        overviewsAdapter,
+        turboCartoAdapter
+    );
+
+    ['update', 'delete'].forEach(function(eventType) {
+        templateMaps.on(eventType, namedMapProviderCache.invalidate.bind(namedMapProviderCache));
+    });
+
+    var authApi = new AuthApi(pgConnection, metadataBackend, mapStore, templateMaps);
+
+    var TablesExtentApi = require('./api/tables_extent_api');
+    var tablesExtentApi = new TablesExtentApi(pgQueryRunner);
+
+    /*******************************************************************************************************************
+     * Routing
+     ******************************************************************************************************************/
+
+    new controller.Layergroup(
+        authApi,
+        pgConnection,
+        mapStore,
+        tileBackend,
+        previewBackend,
+        attributesBackend,
+        new windshaft.backend.Widget(),
+        surrogateKeysCache,
+        userLimitsApi,
+        layergroupAffectedTablesCache,
+        analysisBackend
+    ).register(app);
+
+    new controller.Map(
+        authApi,
+        pgConnection,
+        templateMaps,
+        mapBackend,
+        metadataBackend,
+        surrogateKeysCache,
+        userLimitsApi,
+        layergroupAffectedTablesCache,
+        overviewsAdapter,
+        turboCartoAdapter,
+        analysisBackend
+    ).register(app);
+
+    new controller.NamedMaps(
+        authApi,
+        pgConnection,
+        namedMapProviderCache,
+        tileBackend,
+        previewBackend,
+        surrogateKeysCache,
+        tablesExtentApi,
+        metadataBackend
+    ).register(app);
+
+    new controller.NamedMapsAdmin(authApi, pgConnection, templateMaps).register(app);
+
+    new controller.ServerInfo().register(app);
+
+    /*******************************************************************************************************************
+     * END Routing
+     ******************************************************************************************************************/
+
+    return app;
+};
+
+function validateOptions(opts) {
+    if (!_.isString(opts.base_url) || !_.isString(opts.base_url_mapconfig) || !_.isString(opts.base_url_templated)) {
+        throw new Error("Must initialise server with: 'base_url'/'base_url_mapconfig'/'base_url_templated' URLs");
+    }
+
+    // Be nice and warn if configured mapnik version is != instaled mapnik version
+    if (mapnik.versions.mapnik !== opts.grainstore.mapnik_version) {
+        debug('WARNING: detected mapnik version (' + mapnik.versions.mapnik + ')' +
+            ' != configured mapnik version (' + opts.grainstore.mapnik_version + ')');
+    }
+}
+
+function bootstrapFonts(opts) {
+    // Set carto renderer configuration for MMLStore
+    opts.grainstore.carto_env = opts.grainstore.carto_env || {};
+    var cenv = opts.grainstore.carto_env;
+    cenv.validation_data = cenv.validation_data || {};
+    if ( ! cenv.validation_data.fonts ) {
+        mapnik.register_system_fonts();
+        mapnik.register_default_fonts();
+        cenv.validation_data.fonts = _.keys(mapnik.fontFiles());
+    }
+}
+
+function bootstrap(opts) {
+    var app;
+    if (_.isObject(opts.https)) {
+        // use https if possible
+        app = express.createServer(opts.https);
+    } else {
+        // fall back to http by default
+        app = express();
+    }
+    app.enable('jsonp callback');
+    app.disable('x-powered-by');
+    app.disable('etag');
+    app.use(bodyParser.json());
+
+    app.use(function bootstrap$prepareRequestResponse(req, res, next) {
+        req.context = req.context || {};
+        req.profiler = new Profiler({
+            statsd_client: global.statsClient,
+            profile: opts.useProfiler
+        });
+
+        if (global.environment && global.environment.api_hostname) {
+            res.set('X-Served-By-Host', global.environment.api_hostname);
+        }
+
+        next();
+    });
+
+    // temporary measure until we upgrade to newer version expressjs so we can check err.status
+    app.use(function(err, req, res, next) {
+        if (err) {
+            if (err.name === 'SyntaxError') {
+                res.status(400).json({ errors: [err.name + ': ' + err.message] });
+            } else {
+                next(err);
+            }
+        } else {
+            next();
+        }
+    });
+
+    setupLogger(app, opts);
+
+    return app;
+}
+
+function setupLogger(app, opts) {
+    if (global.log4js && opts.log_format) {
+        var loggerOpts = {
+            // Allowing for unbuffered logging is mainly
+            // used to avoid hanging during unit testing.
+            // TODO: provide an explicit teardown function instead,
+            //       releasing any event handler or timer set by
+            //       this component.
+            buffer: !opts.unbuffered_logging,
+            // optional log format
+            format: opts.log_format
+        };
+        app.use(global.log4js.connectLogger(global.log4js.getLogger(), _.defaults(loggerOpts, {level: 'info'})));
+    }
+}
+
+function surrogateKeysCacheBackends(serverOptions) {
+    var cacheBackends = [];
+
+    if (serverOptions.varnish_purge_enabled) {
+        cacheBackends.push(
+            new VarnishHttpCacheBackend(serverOptions.varnish_host, serverOptions.varnish_http_port)
+        );
+    }
+
+    if (serverOptions.fastly &&
+        !!serverOptions.fastly.enabled && !!serverOptions.fastly.apiKey && !!serverOptions.fastly.serviceId) {
+        cacheBackends.push(
+            new FastlyCacheBackend(serverOptions.fastly.apiKey, serverOptions.fastly.serviceId)
+        );
+    }
+
+    return cacheBackends;
+}
+
+function mapnikVersion(opts) {
+    return opts.grainstore.mapnik_version || mapnik.versions.mapnik;
+}

lib/cartodb/server_options.js

@@ -1,856 +1,112 @@
-var   _          = require('underscore')
-    , Step       = require('step')
-    , Cache      = require('./cache_validator')
-    , QueryTablesApi   = require('./api/query_tables_api')
-    , crypto     = require('crypto')
-    , LZMA       = require('lzma').LZMA
-  ;
+var os = require('os');
+var _ = require('underscore');
+var OverviewsQueryRewriter = require('./utils/overviews_query_rewriter');
 
-// This is for backward compatibility with 1.3.3
-if ( _.isUndefined(global.environment.sqlapi.domain) ) {
-  // Only use "host" as "domain" if it contains alphanumeric characters
-  var host = global.environment.sqlapi.host;
-  if ( host && host.match(/[a-zA-Z]/) ) {
-    global.environment.sqlapi.domain = host;
-  }
+var overviewsQueryRewriter = new OverviewsQueryRewriter({
+    zoom_level: 'CDB_ZoomFromScale(!scale_denominator!)'
+});
+
+var rendererConfig = _.defaults(global.environment.renderer || {}, {
+    cache_ttl: 60000, // milliseconds
+    statsInterval: 60000,
+    mapnik: {
+        poolSize: 8,
+        metatile: 2,
+        bufferSize: 64,
+        snapToGrid: false,
+        clipByBox2d: false,
+        limits: {}
+    },
+    http: {}
+});
+
+rendererConfig.mapnik.queryRewriter = overviewsQueryRewriter;
+
+// Perform keyword substitution in statsd
+// See https://github.com/CartoDB/Windshaft-cartodb/issues/153
+if (global.environment.statsd) {
+    if (global.environment.statsd.prefix) {
+        var host_token = os.hostname().split('.').reverse().join('.');
+        global.environment.statsd.prefix = global.environment.statsd.prefix.replace(/:host/, host_token);
+    }
 }
 
-// Whitelist query parameters and attach format
-var REQUEST_QUERY_PARAMS_WHITELIST = [
-    'sql',
-    'geom_type',
-    'cache_buster',
-    'cache_policy',
-    'callback',
-    'interactivity',
-    'map_key',
-    'api_key',
-    'auth_token',
-    'style',
-    'style_version',
-    'style_convert',
-    'config',
-    'scale_factor'
-];
+var analysisConfig = _.defaults(global.environment.analysis || {}, {
+    batch: {
+        inlineExecution: false,
+        endpoint: 'http://127.0.0.1:8080/api/v2/sql/job',
+        hostHeaderTemplate: '{{=it.username}}.localhost.lan'
+    }
+});
 
-module.exports = function(redisPool) {
-    var redisOpts = redisPool ? {pool: redisPool} : global.environment.redis;
-    var cartoData = require('cartodb-redis')(redisOpts),
-        lzmaWorker = new LZMA(),
-        queryTablesApi = new QueryTablesApi();
+module.exports = {
+    bind: {
+        port: global.environment.port,
+        host: global.environment.host
+    },
+    // This is for inline maps and table maps
+    base_url: global.environment.base_url_legacy || '/tiles/:table',
 
-    var rendererConfig = _.defaults(global.environment.renderer || {}, {
-        cache_ttl: 60000, // milliseconds
-        metatile: 4,
-        bufferSize: 64,
-        statsInterval: 60000
-    });
+    /// @deprecated with Windshaft-0.17.0
+    ///base_url_notable: '/tiles',
 
-    var me = {
-        // This is for inline maps and table maps
-        base_url: global.environment.base_url_legacy || '/tiles/:table',
+    // This is for Detached maps
+    //
+    // "maps" is the official, while
+    // "tiles/layergroup" is for backward compatibility up to 1.6.x
+    //
+    base_url_mapconfig: global.environment.base_url_detached || '(?:/maps|/tiles/layergroup)',
 
-        /// @deprecated with Windshaft-0.17.0
-        ///base_url_notable: '/tiles',
+    base_url_templated: global.environment.base_url_templated || '(?:/maps/named|/tiles/template)',
 
-        // This is for Detached maps
-        //
-        // "maps" is the official, while
-        // "tiles/layergroup" is for backward compatibility up to 1.6.x
-        //
-        base_url_mapconfig: global.environment.base_url_detached || '(?:/maps|/tiles/layergroup)',
-
-        grainstore: {
-          map: {
+    grainstore: {
+        map: {
             // TODO: allow to specify in configuration
             srid: 3857
-          },
-          datasource: global.environment.postgres,
-          cachedir: global.environment.millstone.cache_basedir,
-          mapnik_version: global.environment.mapnik_version,
-          mapnik_tile_format: global.environment.mapnik_tile_format || 'png',
-          default_layergroup_ttl: global.environment.mapConfigTTL || 7200, 
-          gc_prob: 0.01 // @deprecated since Windshaft-1.8.0
         },
-        mapnik: {
-          poolSize: rendererConfig.poolSize,
-          metatile: rendererConfig.metatile,
-          bufferSize: rendererConfig.bufferSize
-        },
-        statsd: global.environment.statsd,
-        renderCache: {
-            ttl: rendererConfig.cache_ttl,
-            statsInterval: rendererConfig.statsInterval
-        },
-        renderer: {
-            http: rendererConfig.http
-        },
-        redis: global.environment.redis,
-        enable_cors: global.environment.enable_cors,
-        varnish_host: global.environment.varnish.host,
-        varnish_port: global.environment.varnish.port,
-        varnish_http_port: global.environment.varnish.http_port,
-        varnish_secret: global.environment.varnish.secret,
-        varnish_purge_enabled: global.environment.varnish.purge_enabled,
-        cache_enabled: global.environment.cache_enabled,
-        log_format: global.environment.log_format,
-        useProfiler: global.environment.useProfiler
-    };
-    
-    // Do not send unwatch on release
-    // See http://github.com/CartoDB/Windshaft-cartodb/issues/161
-    me.redis.unwatchOnRelease = false;
-
-    // Re-use redisPool
-    me.redis.pool = redisPool;
-
-/* This whole block is about generating X-Cache-Channel { */
-
-    // TODO: review lifetime of elements of this cache
-    // NOTE: by-token indices should only be dropped when
-    //       the corresponding layegroup is dropped, because
-    //       we have no SQL after layer creation.
-    me.channelCache = {};
-
-    me.buildCacheChannel = function (dbName, tableNames){
-      return dbName + ':' + tableNames.join(',');
-    };
-
-    me.generateMD5 = function(data){
-        var hash = crypto.createHash('md5');
-        hash.update(data);
-        return hash.digest('hex');
-    };
-
-    me.generateCacheChannel = function(app, req, callback){
-
-      // Build channelCache key
-      var dbName = req.params.dbname;
-      var cacheKey = [ dbName ];
-      if ( req.params.token ) cacheKey.push(req.params.token);
-      else if ( req.params.sql ) cacheKey.push( me.generateMD5(req.params.sql) );
-      cacheKey = cacheKey.join(':');
-
-      var that = this;
-
-      Step (
-        function checkCached() {
-          if ( me.channelCache.hasOwnProperty(cacheKey) ) {
-            callback(null, me.channelCache[cacheKey]);
-            return;
-          }
-          return null;
-        },
-        function extractSQL(err) {
-          if ( err ) throw err;
-
-          if ( req.params.token ) {
-            // TODO: cached cache channel for token-based access should
-            //       be constructed at renderer cache creation time
-            // See http://github.com/CartoDB/Windshaft-cartodb/issues/152
-            if ( ! app.mapStore ) {
-              throw new Error('missing channel cache for token ' + req.params.token);
+        datasource: global.environment.postgres,
+        cachedir: global.environment.millstone.cache_basedir,
+        mapnik_version: global.environment.mapnik_version,
+        mapnik_tile_format: global.environment.mapnik_tile_format || 'png',
+        default_layergroup_ttl: global.environment.mapConfigTTL || 7200
+    },
+    statsd: global.environment.statsd,
+    renderCache: {
+        ttl: rendererConfig.cache_ttl,
+        statsInterval: rendererConfig.statsInterval
+    },
+    renderer: {
+        mapnik: _.defaults(rendererConfig.mapnik, {
+            geojson: {
+                dbPoolParams: {
+                    size: 16,
+                    idleTimeout: 3000,
+                    reapInterval: 1000
+                },
+                clipByBox2d: false
             }
-            var next = this;
-            var mapStore = app.mapStore;
-            Step(
-              function loadFromStore() {
-                mapStore.load(req.params.token, this);
-              },
-              function getSQL(err, mapConfig) {
-                if (req.profiler) req.profiler.done('mapStore_load');
-                if ( err ) throw err;
-                var sql = [];
-                _.each(mapConfig.obj().layers, function(lyr) {
-                  sql.push(lyr.options.sql);
-                });
-                sql = sql.join(';');
-                return sql;
-              },
-              function finish(err, sql) {
-                next(err, sql);
-              }
-            );
-            return;
-          }
+        }),
+        torque: rendererConfig.torque,
+        http: rendererConfig.http
+    },
 
-          if ( ! req.params.sql ) {
-            return null; // no sql
-          }
-
-          // We have sql, and no token...
-
-          // strip out windshaft/mapnik inserted sql if present
-          var sql = req.params.sql.match(/^\((.*)\)\sas\scdbq$/);
-          sql = (sql != null) ? sql[1] : req.params.sql;
-
-          return sql;
-        },
-        function findAffectedTables(err, sql) {
-          if ( err ) throw err;
-          if ( ! sql ) {
-            if ( ! req.params.table ) {
-              throw new Error("this request doesn't need an X-Cache-Channel generated");
-            }
-            return [req.params.table];
-          }
-          var user, key;
-          var next = this;
-          Step (
-            function findUserKey() {
-              if ( req.params.hasOwnProperty('_authorizedBySigner') ) {
-                user = req.params._authorizedBySigner;
-                cartoData.getUserMapKey(user, this);
-              } else {
-                user = that.userByReq(req);
-                key = req.params.map_key || req.params.api_key;
-                return null;
-              }
-            },
-            function getAffected(err, data) {
-              if ( err ) throw err;
-              if ( data ) {
-                if ( req.profiler ) req.profiler.done('getSignerMapKey');
-                key = data;
-              }
-              queryTablesApi.getAffectedTablesInQuery(user, {
-                  user: req.params.dbuser,
-                  pass: req.params.dbpass,
-                  host: req.params.dbhost,
-                  port: req.params.dbport,
-                  dbname: req.params.dbname,
-                  api_key: key
-              }, sql, this); // in addCacheChannel
-            },
-            function finish(err, data) {
-              next(err,data);
-            }
-          );
-        },
-        function buildCacheChannel(err, tableNames) {
-          if ( err ) throw err;
-          if (req.profiler && ! req.params.table ) {
-            req.profiler.done('affectedTables');
-          }
-
-          var dbName     = req.params.dbname;
-          var cacheChannel = me.buildCacheChannel(dbName,tableNames);
-          // store for caching from me.generateCacheChannel
-          // (not worth when table was specified in params)
-          if ( ! req.params.table ) {
-            me.channelCache[cacheKey] = cacheChannel;
-          }
-          return cacheChannel;
-        },
-        function finish(err, cacheChannel) {
-          callback(err, cacheChannel);
+    analysis: {
+        batch: {
+            inlineExecution: analysisConfig.batch.inlineExecution,
+            endpoint: analysisConfig.batch.endpoint,
+            hostHeaderTemplate: analysisConfig.batch.hostHeaderTemplate
         }
-      );
-    };
-
-    // Set the cache chanel info to invalidate the cache on the frontend server
-    //
-    // @param req The request object.
-    //            The function will have no effect unless req.res exists.
-    //            It is expected that req.params contains 'table' and 'dbname'
-    //
-    // @param cb function(err, channel) will be called when ready.
-    //           the channel parameter will be null if nothing was added
-    //
-    me.addCacheChannel = function(app, req, cb) {
-        // skip non-GET requests, or requests for which there's no response
-        if ( req.method != 'GET' || ! req.res ) { cb(null, null); return; }
-        if (req.profiler) req.profiler.start('addCacheChannel');
-        var res = req.res;
-        var cache_policy = req.query.cache_policy;
-        if ( req.params.token ) cache_policy = 'persist';
-        if ( cache_policy == 'persist' ) {
-          res.header('Cache-Control', 'public,max-age=31536000'); // 1 year
-        } else {
-          var ttl = global.environment.varnish.ttl || 86400;
-          res.header('Cache-Control', 'no-cache,max-age='+ttl+',must-revalidate, public');
-        }
-
-        // Set Last-Modified header
-        var lastUpdated;
-        if ( req.params.cache_buster ) {
-          // Assuming cache_buster is a timestamp
-          // FIXME: store lastModified in the cache channel instead
-          lastUpdated = new Date(parseInt(req.params.cache_buster));
-        } else {
-          lastUpdated = new Date();
-        }
-        res.header('Last-Modified', lastUpdated.toUTCString());
-
-        me.generateCacheChannel(app, req, function(err, channel){
-            if (req.profiler) req.profiler.done('generateCacheChannel');
-            if (req.profiler) req.profiler.end();
-            if ( ! err ) {
-              res.header('X-Cache-Channel', channel);
-              cb(null, channel);
-            } else {
-              console.log('ERROR generating cache channel: ' + ( err.message ? err.message : err ));
-              // TODO: evaluate if we should bubble up the error instead
-              cb(null, 'ERROR');
-            }
-        });
-    };
-
-    me.afterLayergroupCreate = function(req, mapconfig, response, callback) {
-        var token = response.layergroupid;
-
-        var username = this.userByReq(req);
-
-        var tasksleft = 2; // redis key and affectedTables
-        var errors = [];
-
-        var done = function(err) {
-          if ( err ) {
-            errors.push('' + err);
-          }
-          if ( ! --tasksleft ) {
-            err = errors.length ? new Error(errors.join('\n')) : null;
-            callback(err);
-          }
-        };
-
-        // include in layergroup response the variables in serverMedata
-        // those variables are useful to send to the client information
-        // about how to reach this server or information about it
-        var serverMetadata = global.environment.serverMetadata;
-        if (serverMetadata) {
-          _.extend(response, serverMetadata);
-        }
-
-        // Don't wait for the mapview count increment to
-        // take place before proceeding. Error will be logged
-        // asyncronously
-        cartoData.incMapviewCount(username, mapconfig.stat_tag, function(err) {
-          if (req.profiler) req.profiler.done('incMapviewCount');
-          if ( err ) console.log("ERROR: failed to increment mapview count for user '" + username + "': " + err);
-          done();
-        });
-
-        var sql = [];
-        _.each(mapconfig.layers, function(lyr) {
-          sql.push(lyr.options.sql);
-        });
-        sql = sql.join(';');
-
-        var dbName = req.params.dbname;
-        var usr    = this.userByReq(req);
-        var key    = req.params.map_key || req.params.api_key;
-
-        var cacheKey = dbName + ':' + token;
-
-        Step(
-            function getAffectedTablesAndLastUpdatedTime() {
-                queryTablesApi.getAffectedTablesAndLastUpdatedTime(usr, {
-                    user: req.params.dbuser,
-                    pass: req.params.dbpass,
-                    host: req.params.dbhost,
-                    port: req.params.dbport,
-                    dbname: req.params.dbname,
-                    api_key: key
-                }, sql, this);
-            },
-            function handleAffectedTablesAndLastUpdatedTime(err, result) {
-                if (req.profiler) req.profiler.done('queryTablesAndLastUpdated');
-                if ( err ) throw err;
-                var cacheChannel = me.buildCacheChannel(dbName, result.affectedTables);
-                me.channelCache[cacheKey] = cacheChannel;
-
-                if (req.res && req.method == 'GET') {
-                    var res = req.res;
-                    if ( req.query && req.query.cache_policy == 'persist' ) {
-                        res.header('Cache-Control', 'public,max-age=31536000'); // 1 year
-                    } else {
-                        var ttl = global.environment.varnish.layergroupTtl || 86400;
-                        res.header('Cache-Control', 'public,max-age='+ttl+',must-revalidate');
-                    }
-                    res.header('Last-Modified', (new Date()).toUTCString());
-                    res.header('X-Cache-Channel', cacheChannel);
-                }
-
-                // last update for layergroup cache buster
-                response.layergroupid = response.layergroupid + ':' + result.lastUpdatedTime;
-                response.last_updated = new Date(result.lastUpdatedTime).toISOString();
-                return null;
-            },
-            function finish(err) {
-                done(err);
-            }
-        );
-    };
-
-/* X-Cache-Channel generation } */
-
-    me.re_userFromHost = new RegExp(
-      global.environment.user_from_host ||
-      '^([^\\.]+)\\.' // would extract "strk" from "strk.cartodb.com"
-    );
-
-    me.userByReq = function(req) {
-      var host = req.headers.host;
-      var mat = host.match(this.re_userFromHost);
-      if ( ! mat ) {
-        console.error("ERROR: user pattern '" + this.re_userFromHost
-          + "' does not match hostname '" + host + "'");
-        return;
-      }
-      // console.log("Matches: "); console.dir(mat);
-      if ( ! mat.length === 2 ) {
-        console.error("ERROR: pattern '" + this.re_userFromHost
-          + "' gave unexpected matches against '" + host + "': " + mat);
-        return;
-      }
-      return mat[1];
-    };
-
-    // Set db authentication parameters to those of the given username
-    //
-    // @param username the cartodb username, mapped to a database username
-    //                 via CartodbRedis metadata records
-    //
-    // @param params the parameters to set auth options into
-    //               added params are: "dbuser" and "dbpassword"
-    //
-    // @param callback function(err) 
-    //
-    me.setDBAuth = function(username, params, callback) {
-
-      var user_params = {};
-      var auth_user = global.environment.postgres_auth_user;
-      var auth_pass = global.environment.postgres_auth_pass;
-      Step(
-        function getId() {
-          cartoData.getUserId(username, this);
-        },
-        function(err, user_id) {
-          if (err) throw err;
-          user_params['user_id'] = user_id;
-          var dbuser = _.template(auth_user, user_params);
-          _.extend(params, {dbuser:dbuser});
-
-          // skip looking up user_password if postgres_auth_pass
-          // doesn't contain the "user_password" label 
-          if (!auth_pass || ! auth_pass.match(/\buser_password\b/) ) return null;
-
-          cartoData.getUserDBPass(username, this);
-        },
-        function(err, user_password) {
-          if (err) throw err;
-          user_params['user_password'] = user_password;
-          if ( auth_pass ) {
-            var dbpass = _.template(auth_pass, user_params);
-            _.extend(params, {dbpassword:dbpass});
-          }
-          return true;
-        },
-        function finish(err) {
-          callback(err); 
-        }
-      );
-    };
-
-    // Set db connection parameters to those for the given username
-    //
-    // @param dbowner cartodb username of database owner,
-    //                mapped to a database username
-    //                via CartodbRedis metadata records
-    //
-    // @param params the parameters to set connection options into
-    //               added params are: "dbname", "dbhost"
-    //
-    // @param callback function(err) 
-    //
-    me.setDBConn = function(dbowner, params, callback) {
-      // Add default database connection parameters
-      // if none given
-      _.defaults(params, {
-        dbuser: global.environment.postgres.user,
-        dbpassword: global.environment.postgres.password,
-        dbhost: global.environment.postgres.host,
-        dbport: global.environment.postgres.port
-      });
-      Step(
-        function getConnectionParams() {
-            cartoData.getUserDBConnectionParams(dbowner, this);
-        },
-        function extendParams(err, dbParams){
-            if (err) throw err;
-            // we don't want null values or overwrite a non public user
-            if (params.dbuser != 'publicuser' || !dbParams.dbuser) {
-                delete dbParams.dbuser;
-            }
-            if ( dbParams ) _.extend(params, dbParams);
-            return null;
-        },
-        function finish(err) {
-          callback(err);
-        }
-      );
-    };
-
-
-    // Check if a request is authorized by a signer
-    //
-    // @param req express request object
-    // @param callback function(err, signed_by) signed_by will be
-    //                 null if the request is not signed by anyone
-    //                 or will be a string cartodb username otherwise.
-    //                 
-    me.authorizedBySigner = function(req, callback)
-    {
-      if ( ! req.params.token || ! req.params.signer ) {
-        //console.log("No signature provided"); // debugging
-        callback(null, null); // no signer requested
-        return;
-      }
-
-      var signer = req.params.signer;
-      var layergroup_id = req.params.token;
-      var auth_token = req.params.auth_token;
-
-      //console.log("Checking authorization from signer " + signer + " for resource " + layergroup_id + " with auth_token " + auth_token);
-      var mapStore = req.app.mapStore;
-      if (!mapStore) {
-          throw new Error('Unable to retrieve map configuration token');
-      }
-
-      mapStore.load(layergroup_id, function(err, mapConfig) {
-        if (err) {
-          throw err;
-        }
-
-        var authorized = me.templateMaps.isAuthorized(mapConfig.obj().template, auth_token);
-        callback(null, authorized ? signer : null);
-      });
-
-    };
-
-    // Check if a request is authorized by api_key
-    //
-    // @param req express request object
-    // @param callback function(err, authorized) 
-    //        NOTE: authorized is expected to be 0 or 1 (integer)
-    //                 
-    me.authorizedByAPIKey = function(req, callback)
-    {
-        var givenKey = req.query.api_key || req.query.map_key;
-        if ( ! givenKey && req.body ) {
-          // check also in request body
-          givenKey = req.body.api_key || req.body.map_key;
-        }
-        if ( ! givenKey ) {
-          callback(null, 0); // no api key, no authorization...
-          return;
-        }
-        //console.log("given ApiKey: " + givenKey);
-        var user = me.userByReq(req);
-        Step(
-          function (){
-              cartoData.getUserMapKey(user, this);
-          },
-          function checkApiKey(err, val){
-              if (err) throw err;
-              return ( val && givenKey == val ) ? 1 : 0;
-          },
-          function finish(err, authorized) {
-              callback(err, authorized);
-          }
-        );
-    };
-
-    /**
-     * Check access authorization
-     *
-     * @param req - standard req object. Importantly contains table and host information
-     * @param callback function(err, allowed) is access allowed not?
-     */
-    me.authorize = function(req, callback) {
-        var that = this;
-        var user = me.userByReq(req);
-
-        Step(
-            function (){
-                that.authorizedByAPIKey(req, this);
-            },
-            function checkApiKey(err, authorized){
-                if (req.profiler) req.profiler.done('authorizedByAPIKey');
-                if (err) throw err;
-
-                // if not authorized by api_key, continue 
-                if (authorized !== 1)  {
-                  // not authorized by api_key, 
-                  // check if authorized by signer
-                  that.authorizedBySigner(req, this);
-                  return;
-                }
-
-                _.extend(req.params, { _authorizedByApiKey: true });
-
-                // authorized by api key, login as the given username and stop
-                that.setDBAuth(user, req.params, function(err) {
-                  callback(err, true); // authorized (or error)
-                });
-            },
-            function checkSignAuthorized(err, signed_by){
-                if (err) throw err;
-                if (req.profiler) {
-                  if ( req.params._authorizedByApiKey ) {
-                    req.profiler.done('setDBAuth');
-                  } else {
-                    req.profiler.done('authorizedBySigner');
-                  }
-                }
-
-                if ( ! signed_by ) {
-                  // request not authorized by signer.
-  
-                  // if table was given, continue to check table privacy
-                  if ( req.params.table ) return null;
-
-                  // if no signer name was given, let dbparams and
-                  // PostgreSQL do the rest.
-                  // 
-                  if ( ! req.params.signer ) {
-                    callback(null, true); // authorized so far
-                    return;
-                  }
-
-                  // if signer name was given, return no authorization
-                  callback(null, false); 
-                  return;
-                }
-
-                // Authorized by "signed_by" !
-                _.extend(req.params, { _authorizedBySigner: signed_by });
-                that.setDBAuth(signed_by, req.params, function(err) {
-                  if (req.profiler) req.profiler.done('setDBAuth');
-                  callback(err, true); // authorized (or error)
-                });
-            },
-            function getDatabase(err){
-                if (err) throw err;
-                // NOTE: only used to get to table privacy
-                cartoData.getUserDBName(user, this);
-            },
-            function getPrivacy(err, dbname){
-                if (err) throw err;
-                if (req.profiler) req.profiler.done('tablePrivacy_getUserDBName');
-                cartoData.getTablePrivacy(dbname, req.params.table, this);
-            },
-            function(err, privacy){
-                if (req.profiler) req.profiler.done('getTablePrivacy');
-                callback(err, privacy !== "0");
-            }
-        );
-    };
-
-    /**
-     * Whitelist input and get database name & default geometry type from
-     * subdomain/user metadata held in CartoDB Redis
-     * @param req - standard express request obj. Should have host & table
-     * @param callback
-     */
-    me.req2params = function(req, callback){
-
-        if ( req.query.lzma ) {
-
-          // TODO: check ?
-          //console.log("type of req.query.lzma is " + typeof(req.query.lzma));
-
-          // Decode (from base64)
-          var lzma = (new Buffer(req.query.lzma, 'base64').toString('binary')).split('').map(function(c) { return c.charCodeAt(0) - 128 });
-
-          // Decompress
-          lzmaWorker.decompress(
-            lzma,
-            function(result) {
-              if (req.profiler) req.profiler.done('LZMA decompress');
-              try {
-                delete req.query.lzma;
-                _.extend(req.query, JSON.parse(result));
-                me.req2params(req, callback);
-              } catch (err) {
-                callback(new Error('Error parsing lzma as JSON: ' + err));
-              }
-            },
-            function(percent) { // progress
-              //console.log("LZMA decompression " + percent + "%");
-            }
-          );
-          return;
-        }
-
-        var bad_query  = _.difference(_.keys(req.query), REQUEST_QUERY_PARAMS_WHITELIST);
-
-        _.each(bad_query, function(key){ delete req.query[key]; });
-        req.params =  _.extend({}, req.params); // shuffle things as request is a strange array/object
-
-        var user = me.userByReq(req);
-
-        if ( req.params.token ) {
-          //console.log("Request parameters include token " + req.params.token);
-          var tksplit = req.params.token.split(':');
-          req.params.token = tksplit[0];
-          if ( tksplit.length > 1 ) req.params.cache_buster= tksplit[1];
-          tksplit = req.params.token.split('@');
-          if ( tksplit.length > 1 ) {
-            req.params.signer = tksplit.shift();
-            if ( ! req.params.signer ) req.params.signer = user;
-            else if ( req.params.signer != user ) {
-              var err = new Error('Cannot use map signature of user "' + req.params.signer + '" on database of user "' + user + '"');
-              err.http_status = 403;
-              callback(err);
-              return;
-            }
-            if ( tksplit.length > 1 ) {
-              var template_hash = tksplit.shift(); // unused
-            }
-            req.params.token = tksplit.shift(); 
-            //console.log("Request for token " + req.params.token + " with signature from " + req.params.signer);
-          }
-        }
-
-        // bring all query values onto req.params object
-        _.extend(req.params, req.query);
-
-        // for cartodb, ensure interactivity is cartodb_id or user specified
-        req.params.interactivity = req.params.interactivity || 'cartodb_id';
-
-        var that = this;
-
-        if (req.profiler) req.profiler.done('req2params.setup');
-
-        Step(
-            function getPrivacy(){
-                me.authorize(req, this);
-            },
-            function gatekeep(err, authorized){
-                if (req.profiler) req.profiler.done('authorize');
-                if(err) throw err;
-                if(!authorized) {
-                  err = new Error("Sorry, you are unauthorized (permission denied)");
-                  err.http_status = 403;
-                  throw err;
-                }
-                return null;
-            },
-            function getDatabase(err){
-                if(err) throw err;
-                that.setDBConn(user, req.params, this);
-            },
-            function getGeometryType(err){
-                if (req.profiler) req.profiler.done('setDBConn');
-                if (err) throw err;
-                if ( ! req.params.table ) return null;
-                cartoData.getTableGeometryType(req.params.dbname, req.params.table, this);
-            },
-            function finishSetup(err, data){
-                if ( err ) { callback(err, req); return; }
-
-                if (!_.isNull(data))
-                    _.extend(req.params, {geom_type: data});
-
-                // Add default database connection parameters
-                // if none given
-                _.defaults(req.params, {
-                  dbuser: global.environment.postgres.user,
-                  dbpassword: global.environment.postgres.password,
-                  dbhost: global.environment.postgres.host,
-                  dbport: global.environment.postgres.port
-                });
-
-                callback(null, req);
-            }
-        );
-    };
-
-    /**
-     * Little helper method to get the current list of infowindow variables and return to client
-     * @param req
-     * @param callback
-     */
-    me.getInfowindow = function(req, callback){
-        var that = this;
-        var user = me.userByReq(req);
-
-        Step(
-            function(){
-                // TODO: if this step really needed ?
-                that.req2params(req, this);
-            },
-            function getDatabase(err){
-                if (err) throw err;
-                cartoData.getUserDBName(user, this);
-            },
-            function getInfowindow(err, dbname){
-                if (err) throw err;
-                cartoData.getTableInfowindow(dbname, req.params.table, this);
-            },
-            function(err, data){
-                callback(err, data);
-            }
-        );
-    };
-
-    /**
-     * Little helper method to get map metadata and return to client
-     * @param req
-     * @param callback
-     */
-    me.getMapMetadata = function(req, callback){
-        var that = this;
-        var user = me.userByReq(req);
-
-        Step(
-            function(){
-                // TODO: if this step really needed ?
-                that.req2params(req, this);
-            },
-            function getDatabase(err){
-                if (err) throw err;
-                cartoData.getUserDBName(user, this);
-            },
-            function getMapMetadata(err, dbname){
-                if (err) throw err;
-                cartoData.getTableMapMetadata(dbname, req.params.table, this);
-            },
-            function(err, data){
-                callback(err, data);
-            }
-        );
-    };
-
-    /**
-     * Helper to clear out tile cache on request
-     * @param req
-     * @param callback
-     */
-    me.flushCache = function(req, Cache, callback){
-        var that = this;
-
-        Step(
-            function getParams(){
-                // this is mostly to compute req.params.dbname
-                that.req2params(req, this);
-            },
-            function flushInternalCache(err){
-                // TODO: implement this, see
-                // http://github.com/Vizzuality/Windshaft-cartodb/issues/73
-                return true;
-            },
-            function flushVarnishCache(err){
-                if (err) { callback(err); return; }
-                if(Cache) {
-                  Cache.invalidate_db(req.params.dbname, req.params.table);
-                }
-                callback(null, true);
-            }
-        );
-    };
-
-    return me;
+    },
+    // Do not send unwatch on release. See http://github.com/CartoDB/Windshaft-cartodb/issues/161
+    redis: _.extend(global.environment.redis, {unwatchOnRelease: false}),
+    enable_cors: global.environment.enable_cors,
+    varnish_host: global.environment.varnish.host,
+    varnish_port: global.environment.varnish.port,
+    varnish_http_port: global.environment.varnish.http_port,
+    varnish_secret: global.environment.varnish.secret,
+    varnish_purge_enabled: global.environment.varnish.purge_enabled,
+    fastly: global.environment.fastly || {},
+    cache_enabled: global.environment.cache_enabled,
+    log_format: global.environment.log_format,
+    useProfiler: global.environment.useProfiler
 };

lib/cartodb/sql/sql_api.js

@@ -1,66 +0,0 @@
-var _       = require('underscore'),
-    request = require('request');
-
-module.exports.query = function (username, api_key, sql, callback) {
-    var api = global.environment.sqlapi;
-
-    // build up api string
-    var sqlapihostname = username;
-    if ( api.domain ) sqlapihostname += '.' + api.domain;
-
-    var sqlapi = api.protocol + '://';
-    if ( api.host && api.host != api.domain ) sqlapi += api.host;
-    else sqlapi += sqlapihostname;
-    sqlapi += ':' + api.port + '/api/' + api.version + '/sql';
-
-    var qs  = { q: sql };
-
-    // add api_key if given
-    if (_.isString(api_key) && api_key != '') { qs.api_key = api_key; }
-
-    // call sql api
-    //
-    // NOTE: using POST to avoid size limits:
-    // See http://github.com/CartoDB/Windshaft-cartodb/issues/111
-    //
-    // NOTE: uses "host" header to allow IP based specification
-    //       of sqlapi address (and avoid a DNS lookup)
-    //
-    // NOTE: allows for keeping up to "maxConnections" concurrent
-    //       sockets opened per SQL-API host.
-    // See http://nodejs.org/api/http.html#http_agent_maxsockets
-    //
-    var maxSockets = global.environment.maxConnections || 128;
-    var maxGetLen = api.max_get_sql_length || 2048;
-    var maxSQLTime = api.timeout || 100; // 1/10 of a second by default
-    var reqSpec = {
-        url:sqlapi,
-        json:true,
-        headers:{host: sqlapihostname}
-        // http://nodejs.org/api/http.html#http_agent_maxsockets
-        ,pool:{maxSockets:maxSockets}
-        // timeout in milliseconds
-        ,timeout:maxSQLTime
-    };
-    if ( sql.length > maxGetLen ) {
-        reqSpec.method = 'POST';
-        reqSpec.body = qs;
-    } else {
-        reqSpec.method = 'GET';
-        reqSpec.qs = qs;
-    }
-    request(reqSpec, function(err, res, body) {
-        if (err){
-            console.log('ERROR connecting to SQL API on ' + sqlapi + ': ' + err);
-            callback(err);
-            return;
-        }
-        if (res.statusCode != 200) {
-            var msg = res.body.error ? res.body.error : res.body;
-            callback(new Error(msg));
-            console.log('unexpected response status (' + res.statusCode + ') for sql query: ' + sql + ': ' + msg);
-            return;
-        }
-        callback(null, body.rows);
-    });
-};

lib/cartodb/stats/client.js

@@ -0,0 +1,73 @@
+var _ = require('underscore');
+var debug = require('debug')('windshaft:stats_client');
+var StatsD = require('node-statsd').StatsD;
+
+module.exports = {
+    /**
+     * Returns an StatsD instance or an stub object that replicates the StatsD public interface so there is no need to
+     * keep checking if the stats_client is instantiated or not.
+     *
+     * The first call to this method implies all future calls will use the config specified in the very first call.
+     *
+     * TODO: It's far from ideal to use make this a singleton, improvement desired.
+     * We proceed this way to be able to use StatsD from several places sharing one single StatsD instance.
+     *
+     * @param config Configuration for StatsD, if undefined it will return an stub
+     * @returns {StatsD|Object}
+     */
+    getInstance: function(config) {
+
+        if (!this.instance) {
+
+            var instance;
+
+            if (config) {
+                instance = new StatsD(config);
+                instance.last_error = { msg: '', count: 0 };
+                instance.socket.on('error', function (err) {
+                    var last_err = instance.last_error;
+                    var last_msg = last_err.msg;
+                    var this_msg = '' + err;
+                    if (this_msg !== last_msg) {
+                        debug("statsd client socket error: " + err);
+                        instance.last_error.count = 1;
+                        instance.last_error.msg = this_msg;
+                    } else {
+                        ++last_err.count;
+                        if (!last_err.interval) {
+                            instance.last_error.interval = setInterval(function () {
+                                var count = instance.last_error.count;
+                                if (count > 1) {
+                                    debug("last statsd client socket error repeated " + count + " times");
+                                    instance.last_error.count = 1;
+                                    clearInterval(instance.last_error.interval);
+                                    instance.last_error.interval = null;
+                                }
+                            }, 1000);
+                        }
+                    }
+                });
+            } else {
+                var stubFunc = function (stat, value, sampleRate, callback) {
+                    if (_.isFunction(callback)) {
+                        callback(null, 0);
+                    }
+                };
+                instance = {
+                    timing: stubFunc,
+                    increment: stubFunc,
+                    decrement: stubFunc,
+                    gauge: stubFunc,
+                    unique: stubFunc,
+                    set: stubFunc,
+                    sendAll: stubFunc,
+                    send: stubFunc
+                };
+            }
+
+            this.instance = instance;
+        }
+
+        return this.instance;
+    }
+};

lib/cartodb/stats/profiler_proxy.js

@@ -0,0 +1,53 @@
+var Profiler = require('step-profiler');
+
+/**
+ * Proxy to encapsulate node-step-profiler module so there is no need to check if there is an instance
+ */
+function ProfilerProxy(opts) {
+    this.profile = !!opts.profile;
+
+    this.profiler = null;
+    if (!!opts.profile) {
+        this.profiler = new Profiler({statsd_client: opts.statsd_client});
+    }
+}
+
+ProfilerProxy.prototype.done = function(what) {
+    if (this.profile) {
+        this.profiler.done(what);
+    }
+};
+
+ProfilerProxy.prototype.end = function() {
+    if (this.profile) {
+        this.profiler.end();
+    }
+};
+
+ProfilerProxy.prototype.start = function(what) {
+    if (this.profile) {
+        this.profiler.start(what);
+    }
+};
+
+ProfilerProxy.prototype.add = function(what) {
+    if (this.profile) {
+        this.profiler.add(what || {});
+    }
+};
+
+ProfilerProxy.prototype.sendStats = function() {
+    if (this.profile) {
+        this.profiler.sendStats();
+    }
+};
+
+ProfilerProxy.prototype.toString = function() {
+    return this.profile ? this.profiler.toString() : "";
+};
+
+ProfilerProxy.prototype.toJSONString = function() {
+    return this.profile ? this.profiler.toJSONString() : "{}";
+};
+
+module.exports = ProfilerProxy;

lib/cartodb/stats/reporter/renderer.js

@@ -0,0 +1,82 @@
+//  - Reports stats about:
+//    * Total number of renderers
+//    * For mapnik renderers:
+//      - the mapnik-pool status: count, unused and waiting
+//      - the internally cached objects: png and grid
+
+var _ = require('underscore');
+
+function RendererStatsReporter(rendererCache, statsInterval) {
+    this.rendererCache = rendererCache;
+    this.statsInterval = statsInterval || 6e4;
+    this.renderersStatsIntervalId = null;
+}
+
+module.exports = RendererStatsReporter;
+
+RendererStatsReporter.prototype.start = function() {
+    var self = this;
+    this.renderersStatsIntervalId = setInterval(function() {
+        var rendererCacheEntries = self.rendererCache.renderers;
+
+        if (!rendererCacheEntries) {
+            return null;
+        }
+
+        global.statsClient.gauge('windshaft.rendercache.count', _.keys(rendererCacheEntries).length);
+
+        var renderersStats = _.reduce(rendererCacheEntries, function(_rendererStats, cacheEntry) {
+                var stats = cacheEntry.renderer && cacheEntry.renderer.getStats && cacheEntry.renderer.getStats();
+                if (!stats) {
+                    return _rendererStats;
+                }
+
+                _rendererStats.pool.count += stats.pool.count;
+                _rendererStats.pool.unused += stats.pool.unused;
+                _rendererStats.pool.waiting += stats.pool.waiting;
+
+                _rendererStats.cache.grid += stats.cache.grid;
+                _rendererStats.cache.png += stats.cache.png;
+
+                return _rendererStats;
+            },
+            {
+                pool: {
+                    count: 0,
+                    unused: 0,
+                    waiting: 0
+                },
+                cache: {
+                    png: 0,
+                    grid: 0
+                }
+            }
+        );
+
+        global.statsClient.gauge('windshaft.mapnik-cache.png', renderersStats.cache.png);
+        global.statsClient.gauge('windshaft.mapnik-cache.grid', renderersStats.cache.grid);
+
+        global.statsClient.gauge('windshaft.mapnik-pool.count', renderersStats.pool.count);
+        global.statsClient.gauge('windshaft.mapnik-pool.unused', renderersStats.pool.unused);
+        global.statsClient.gauge('windshaft.mapnik-pool.waiting', renderersStats.pool.waiting);
+    }, this.statsInterval);
+
+    this.rendererCache.on('err', rendererCacheErrorListener);
+    this.rendererCache.on('gc', gcTimingListener);
+};
+
+function rendererCacheErrorListener() {
+    global.statsClient.increment('windshaft.rendercache.error');
+}
+
+function gcTimingListener(gcTime) {
+    global.statsClient.timing('windshaft.rendercache.gc', gcTime);
+}
+
+RendererStatsReporter.prototype.stop = function() {
+    this.rendererCache.removeListener('err', rendererCacheErrorListener);
+    this.rendererCache.removeListener('gc', gcTimingListener);
+
+    clearInterval(this.renderersStatsIntervalId);
+    this.renderersStatsIntervalId = null;
+};

lib/cartodb/stats/timer.js

@@ -0,0 +1,32 @@
+function Timer() {
+    this.times = {};
+}
+
+module.exports = Timer;
+
+Timer.prototype.start = function(label) {
+    this.timeIt(label, 'start');
+};
+
+Timer.prototype.end = function(label) {
+    this.timeIt(label, 'end');
+};
+
+Timer.prototype.timeIt = function(label, what) {
+    this.times[label] = this.times[label] || {};
+    this.times[label][what] = Date.now();
+};
+
+Timer.prototype.getTimes = function() {
+    var self = this;
+    var times = {};
+
+    Object.keys(this.times).forEach(function(label) {
+        var stat = self.times[label];
+        if (stat.start && stat.end) {
+            times[label] = Math.max(0, stat.end - stat.start);
+        }
+    });
+
+    return times;
+};

lib/cartodb/utils/overviews_query_rewriter.js

@@ -0,0 +1,194 @@
+var TableNameParser = require('./table_name_parser');
+
+function OverviewsQueryRewriter(options) {
+
+    this.options = options;
+}
+
+module.exports = OverviewsQueryRewriter;
+
+// TODO: some names are introudced in the queries, and the
+// '_vovw_' (for vector overviews) is used in them, but no check
+// is performed for conflicts with existing identifiers in the query.
+
+// Build UNION expression to replace table, using overviews metadata
+// overviews metadata: { 1: 'table_ov1', ... }
+// assume table and overview names include schema if necessary and are quoted as needed
+function overviews_view_for_table(table, overviews_metadata, indent) {
+    var condition, i, len, ov_table, overview_layers, selects, z_hi, z_lo;
+    var parsed_table = TableNameParser.parse(table);
+
+    var sorted_overviews = []; // [[1, 'table_ov1'], ...]
+
+    indent = indent || '    ';
+    for (var z in overviews_metadata) {
+        if (overviews_metadata.hasOwnProperty(z) && z !== 'schema') {
+            sorted_overviews.push([z, overviews_metadata[z].table]);
+        }
+    }
+    sorted_overviews.sort(function(a, b){ return a[0]-b[0]; });
+
+    overview_layers = [];
+    z_lo = null;
+    for (i = 0, len = sorted_overviews.length; i < len; i++) {
+        z_hi = parseInt(sorted_overviews[i][0]);
+        ov_table = sorted_overviews[i][1];
+        overview_layers.push([overview_z_condition(z_lo, z_hi), ov_table]);
+        z_lo = z_hi;
+    }
+    overview_layers.push(["_vovw_z > " + z_lo, table]);
+
+    selects = overview_layers.map(function(condition_table) {
+        condition = condition_table[0];
+        ov_table = TableNameParser.parse(condition_table[1]);
+        ov_table.schema = ov_table.schema || parsed_table.schema;
+        var ov_identifier = TableNameParser.table_identifier(ov_table);
+        return indent + "SELECT * FROM " + ov_identifier + ", _vovw_scale WHERE " + condition;
+    });
+
+    return selects.join("\n"+indent+"UNION ALL\n");
+}
+
+function overview_z_condition(z_lo, z_hi) {
+    if (z_lo !== null) {
+        if (z_lo === z_hi - 1) {
+            return "_vovw_z = " + z_hi;
+        } else {
+            return "_vovw_z > " + z_lo + " AND _vovw_z <= " + z_hi;
+        }
+    } else {
+        if (z_hi === 0) {
+            return "_vovw_z = " + z_hi;
+        } else {
+            return "_vovw_z <= " + z_hi;
+        }
+    }
+}
+
+// name to be used for the view of the table using overviews
+function overviews_view_name(table) {
+    var parsed_table = TableNameParser.parse(table);
+    parsed_table.table = '_vovw_' + parsed_table.table;
+    parsed_table.schema = null;
+    return TableNameParser.table_identifier(parsed_table);
+}
+
+// replace a table name in a query by anoter name
+function replace_table_in_query(sql, old_table_name, replacement) {
+    var old_table = TableNameParser.parse(old_table_name);
+    var old_table_ident = TableNameParser.table_identifier(old_table);
+
+    // regular expression prefix (beginning) to match a table name
+    function pattern_prefix(schema, identifier) {
+        if ( schema ) {
+            // to match a table name including schema prefix
+            // name should not be part of another name, so we require
+            // to start a at a word boundary
+            if ( identifier[0] !== '"' ) {
+                return '\\b';
+            } else {
+                return '';
+            }
+        } else {
+            // to match a table name without schema
+            // name should not begin right after a dot (i.e. have a explicit schema)
+            // nor be part of another name
+            // since the pattern matches the first character of the table
+            // it must be put back in the replacement text
+            replacement = '$01'+replacement;
+            return '([^\.a-z0-9_]|^)';
+        }
+    }
+
+    // regular expression suffix (ending) to match a table name
+    function pattern_suffix(identifier) {
+        // name shouldn't be the prefix of a longer name
+        if ( identifier[identifier.length-1] !== '"' ) {
+            return '\\b';
+        } else {
+            return '';
+        }
+    }
+
+    // regular expression to match a table name
+    var regexp = pattern_prefix(old_table.schema, old_table_ident) +
+                 old_table_ident +
+                 pattern_suffix(old_table_ident);
+
+    // replace all occurrences of the table pattern
+    return sql.replace(new RegExp(regexp, 'g'), replacement);
+}
+
+function overviews_query(query, overviews, zoom_level_expression) {
+    var replaced_query = query;
+    var sql = "WITH\n  _vovw_scale AS ( SELECT " + zoom_level_expression + " AS _vovw_z )";
+    var replacement;
+    for ( var table in overviews ) {
+        if (overviews.hasOwnProperty(table)) {
+            var table_overviews = overviews[table];
+            var table_view = overviews_view_name(table);
+            var schema = table_overviews.schema;
+            replacement = "(\n" + overviews_view_for_table(table, table_overviews) + "\n  ) AS " + table_view;
+            replaced_query = replace_table_in_query(replaced_query, table, replacement);
+            var parsed_table = TableNameParser.parse(table);
+            if (!parsed_table.schema && schema) {
+                // replace also the qualified table name, if the table wasn't qualified
+                parsed_table.schema = schema;
+                table = TableNameParser.table_identifier(parsed_table);
+                replaced_query = replace_table_in_query(replaced_query, table, replacement);
+            }
+        }
+    }
+    if ( replaced_query !== query ) {
+        sql += "\n";
+        sql += replaced_query;
+    } else {
+        sql = query;
+    }
+    return sql;
+}
+
+// Transform an SQL query so that it uses overviews.
+// overviews contains metadata about the overviews to be used:
+//     { 'table-name': {1: { table: 'overview-table-1' }, ... }, ... }
+//
+// For a given query `SELECT * FROM table`,  if any of tables in it
+// has overviews as defined by the provided metadat, the query will
+// be transform into something similar to this:
+//
+//     WITH _vovw_scale AS ( ... ), -- define scale level
+//     WITH _vovw_table AS ( ... ), -- define union of overviews and base table
+//     SELECT * FROM _vovw_table -- query with table replaced by _vovw_table
+//
+// This transformation can in principle be applied to arbitrary queries
+// (except for the case of queries that include the name of tables with
+// overviews inside text literals: at the current table name substitution
+// doesnn't prevent substitution inside literals).
+// But the transformation will currently only be applied to simple queries
+// of the form detected by the overviews_supported_query function.
+OverviewsQueryRewriter.prototype.query = function(query, data) {
+    var overviews = this.overviews_metadata(data);
+    if ( !overviews || !this.is_supported_query(query)) {
+        return query;
+    }
+    var zoom_level_expression = this.options.zoom_level || '0';
+    return overviews_query(query, overviews, zoom_level_expression);
+};
+
+OverviewsQueryRewriter.prototype.is_supported_query = function(sql) {
+    var basic_query =
+        /\s*SELECT\s+[\*a-z0-9_,\s]+?\s+FROM\s+((\"[^"]+\"|[a-z0-9_]+)\.)?(\"[^"]+\"|[a-z0-9_]+)\s*;?\s*/i;
+    var unwrapped_query = new RegExp("^"+basic_query.source+"$", 'i');
+    // queries for named maps are wrapped like this:
+    var wrapped_query = new RegExp(
+        "^\\s*SELECT\\s+\\*\\s+FROM\\s+\\(" +
+        basic_query.source +
+        "\\)\\s+AS\\s+wrapped_query\\s+WHERE\\s+\\d+=1\\s*$",
+        'i'
+    );
+    return !!(sql.match(unwrapped_query) || sql.match(wrapped_query));
+};
+
+OverviewsQueryRewriter.prototype.overviews_metadata = function(data) {
+    return data && data.overviews;
+};

lib/cartodb/utils/style/postgres-datasource.js

@@ -0,0 +1,55 @@
+'use strict';
+
+var dot = require('dot');
+dot.templateSettings.strip = false;
+
+function createTemplate(method) {
+    return dot.template([
+        'SELECT',
+        method,
+        'FROM ({{=it._sql}}) _table_sql WHERE {{=it._column}} IS NOT NULL'
+    ].join('\n'));
+}
+
+var methods = {
+    quantiles: 'CDB_QuantileBins(array_agg(distinct({{=it._column}}::numeric)), {{=it._buckets}}) as quantiles',
+    equal: 'CDB_EqualIntervalBins(array_agg({{=it._column}}::numeric), {{=it._buckets}}) as equal',
+    jenks: 'CDB_JenksBins(array_agg(distinct({{=it._column}}::numeric)), {{=it._buckets}}) as jenks',
+    headtails: 'CDB_HeadsTailsBins(array_agg(distinct({{=it._column}}::numeric)), {{=it._buckets}}) as headtails'
+};
+
+var methodTemplates = Object.keys(methods).reduce(function(methodTemplates, methodName) {
+    methodTemplates[methodName] = createTemplate(methods[methodName]);
+    return methodTemplates;
+}, {});
+
+function PostgresDatasource (pgQueryRunner, username, query) {
+    this.pgQueryRunner = pgQueryRunner;
+    this.username = username;
+    this.query = query;
+}
+
+PostgresDatasource.prototype.getName = function () {
+    return 'PostgresDatasource';
+};
+
+PostgresDatasource.prototype.getRamp = function (column, buckets, method, callback) {
+    var methodName = methods.hasOwnProperty(method) ? method : 'quantiles';
+    var template = methodTemplates[methodName];
+
+    var query = template({ _column: column, _sql: this.query, _buckets: buckets });
+
+    this.pgQueryRunner.run(this.username, query, function (err, result) {
+        if (err) {
+            return callback(err);
+        }
+
+        var ramp = result[0][methodName].sort(function(a, b) {
+            return a - b;
+        });
+
+        return callback(null, ramp);
+    });
+};
+
+module.exports = PostgresDatasource;

lib/cartodb/utils/style/turbo-carto-adapter.js

@@ -0,0 +1,56 @@
+'use strict';
+
+var queue = require('queue-async');
+
+function TurboCartoAdapter(turboCartoParser) {
+    this.turboCartoParser = turboCartoParser;
+}
+
+module.exports = TurboCartoAdapter;
+
+TurboCartoAdapter.prototype.getLayers = function (username, layers, callback) {
+    var self = this;
+
+    if (!layers || layers.length === 0) {
+        return callback(null, layers);
+    }
+
+    var parseCartoQueue = queue(layers.length);
+
+    layers.forEach(function(layer) {
+        parseCartoQueue.defer(self._parseCartoCss.bind(self), username, layer);
+    });
+
+    parseCartoQueue.awaitAll(function (err, layers) {
+        if (err) {
+            return callback(err);
+        }
+
+        return callback(null, layers);
+    });
+};
+
+TurboCartoAdapter.prototype._parseCartoCss = function (username, layer, callback) {
+    if (isNotLayerToParseCartocss(layer)) {
+        return process.nextTick(function () {
+            callback(null, layer);
+        });
+    }
+
+    this.turboCartoParser.process(username, layer.options.cartocss, layer.options.sql, function (err, cartocss) {
+        // Ignore turbo-carto errors and continue
+        if (!err && cartocss) {
+            layer.options.cartocss = cartocss;
+        }
+
+        callback(null, layer);
+    });
+};
+
+function isNotLayerToParseCartocss(layer) {
+    if (!layer || !layer.options || !layer.options.cartocss || !layer.options.sql) {
+        return true;
+    }
+
+    return false;
+}

lib/cartodb/utils/style/turbo-carto-parser.js

@@ -0,0 +1,15 @@
+'use strict';
+
+var turboCarto = require('turbo-carto');
+var PostgresDatasource = require('./postgres-datasource');
+
+function TurboCartoParser (pgQueryRunner) {
+    this.pgQueryRunner = pgQueryRunner;
+}
+
+module.exports = TurboCartoParser;
+
+TurboCartoParser.prototype.process = function (username, cartocss, sql, callback) {
+    var datasource = new PostgresDatasource(this.pgQueryRunner, username, sql);
+    turboCarto(cartocss, datasource, callback);
+};

lib/cartodb/utils/table_name_parser.js

@@ -0,0 +1,106 @@
+// Quote an PostgreSQL identifier if ncecessary
+function quote_identifier_if_needed(txt) {
+    if ( txt && !txt.match(/^[a-z_][a-z_0-9]*$/)) {
+        return '"' + txt.replace(/\"/g, '""') + '"';
+    } else {
+        return txt;
+    }
+}
+
+// Parse PostgreSQL table name (possibly quoted and with optional schema).+
+// Returns { schema: 'schema_name', table: 'table_name' }
+function parse_table_name(table) {
+
+    function split_as_quoted_parts(table_name) {
+      // parse table into 'parts' that may be quoted, each part
+      // in the parts array being an object { part: 'text', quoted: false/true }
+     var parts = [];
+     var splitted = table_name.split(/\"/);
+     for (var i=0; i<splitted.length; i++ ) {
+       if ( splitted[i] === '' ) {
+         if ( parts.length > 0 && i < splitted.length-1 ) {
+             i++;
+             parts[parts.length - 1].part += '"' + splitted[i];
+         }
+       }
+       else {
+         var is_quoted = (i > 0 && splitted[i-1] === '') ||
+                         (i < splitted.length - 1 && splitted[i+1] === '');
+             parts.push({ part: splitted[i], quoted: is_quoted });
+       }
+     }
+     return parts;
+    }
+
+    var parts = split_as_quoted_parts(table);
+
+    function split_single_part(part) {
+        var schema_part = null;
+        var table_part = null;
+        if ( part.quoted ) {
+            table_part = part.part;
+        } else {
+            var parts = part.part.split('.');
+            if ( parts.length === 1 ) {
+                schema_part = null;
+                table_part = parts[0];
+            } else if ( parts.length === 2 ) {
+                schema_part = parts[0];
+                table_part = parts[1];
+            } // else invalid table name
+        }
+        return {
+            schema: schema_part,
+            table:  table_part
+        };
+    }
+
+    function split_two_parts(part1, part2) {
+        var schema_part = null;
+        var table_part = null;
+        if ( part1.quoted && !part2.quoted ) {
+            if ( part2.part[0] === '.' ) {
+                schema_part = part1.part;
+                table_part = part2.part.slice(1);
+            } // else invalid table name (missing dot)
+        } else if ( !part1.quoted && part2.quoted ) {
+            if ( part1.part[part1.part.length - 1] === '.' ) {
+                schema_part = part1.part.slice(0, -1);
+                table_part = part2.part;
+            } // else invalid table name (missing dot)
+        } // else invalid table name (missing dot)
+        return {
+            schema: schema_part,
+            table:  table_part
+        };
+    }
+
+    if ( parts.length === 1 ) {
+        return split_single_part(parts[0]);
+    } else if ( parts.length === 2 ) {
+        return split_two_parts(parts[0], parts[1]);
+    } else if ( parts.length === 3 && parts[1].part === '.' ) {
+        return {
+            schema: parts[0].part,
+            table:  parts[2].part
+        };
+    } // else invalid table name
+}
+
+function table_identifier(parsed_name) {
+    if ( parsed_name && parsed_name.table ) {
+        if ( parsed_name.schema ) {
+            return quote_identifier_if_needed(parsed_name.schema) + '.' + quote_identifier_if_needed(parsed_name.table);
+        } else {
+            return quote_identifier_if_needed(parsed_name.table);
+        }
+    } else {
+        return null;
+    }
+}
+
+module.exports = {
+  parse: parse_table_name,
+  quote: quote_identifier_if_needed,
+  table_identifier: table_identifier
+};

package.json

@@ -1,51 +1,61 @@
 {
-    "private": true,
-    "name": "windshaft-cartodb",
-    "version": "1.26.2",
-    "description": "A map tile server for CartoDB",
-    "keywords": [
-      "cartodb"
-    ],
-    "url": "https://github.com/CartoDB/Windshaft-cartodb",
-    "licenses": [{
-        "type": "BSD",
-        "url": "https://github.com/CartoDB/Windshaft-cartodb/blob/master/LICENCE"
-    }],
-    "repository": {
-        "type": "git",
-        "url": "git://github.com/CartoDB/Windshaft-cartodb.git"
-    },
-    "author": "Vizzuality <contact@vizzuality.com> (http://vizzuality.com)",
-    "contributors": [
-      "Simon Tokumine <simon@vizzuality.com>",
-      "Javi Santana <jsantana@vizzuality.com>",
-      "Sandro Santilli <strk@vizzuality.com>"
-    ],
-    "dependencies": {
-        "node-varnish": "https://github.com/Vizzuality/node-varnish/tarball/0.3.0",
-        "underscore"  : "~1.6.0",
-        "dot": "~1.0.2",
-        "windshaft": "https://github.com/CartoDB/Windshaft/tarball/0.35.1",
-        "step": "~0.0.5",
-        "request": "~2.9.203",
-        "cartodb-redis": "https://github.com/CartoDB/node-cartodb-redis/tarball/0.11.0",
-        "cartodb-psql": "https://github.com/CartoDB/node-cartodb-psql/tarball/0.4.0",
-        "redis-mpool": "https://github.com/CartoDB/node-redis-mpool/tarball/0.3.0",
-        "lzma": "~1.3.7",
-        "log4js": "https://github.com/CartoDB/log4js-node/tarball/cdb",
-        "rollbar": "~0.3.13"
-    },
-    "devDependencies": {
-        "mocha": "~1.21.4",
-        "redis": "~0.8.6",
-        "strftime": "~0.8.2",
-        "semver": "~1.1.4"
-    },
-    "scripts": {
-        "test": "make check"
-    },
-    "engines": {
-        "node": ">=0.8 <0.11",
-        "npm": ">=1.2.1"
-    }
+  "private": true,
+  "name": "windshaft-cartodb",
+  "version": "2.39.0",
+  "description": "A map tile server for CartoDB",
+  "keywords": [
+    "cartodb"
+  ],
+  "url": "https://github.com/CartoDB/Windshaft-cartodb",
+  "license": "BSD-3-Clause",
+  "repository": {
+    "type": "git",
+    "url": "git://github.com/CartoDB/Windshaft-cartodb.git"
+  },
+  "author": "Vizzuality <contact@vizzuality.com> (http://vizzuality.com)",
+  "contributors": [
+    "Simon Tokumine <simon@vizzuality.com>",
+    "Javi Santana <jsantana@vizzuality.com>",
+    "Sandro Santilli <strk@vizzuality.com>"
+  ],
+  "dependencies": {
+    "body-parser": "~1.14.0",
+    "camshaft": "0.6.0",
+    "cartodb-psql": "~0.6.1",
+    "cartodb-query-tables": "~0.1.0",
+    "cartodb-redis": "~0.13.0",
+    "debug": "~2.2.0",
+    "dot": "~1.0.2",
+    "express": "~4.13.3",
+    "fastly-purge": "~1.0.1",
+    "log4js": "https://github.com/CartoDB/log4js-node/tarball/cdb",
+    "lru-cache": "2.6.5",
+    "lzma": "~1.3.7",
+    "node-statsd": "~0.0.7",
+    "queue-async": "~1.0.7",
+    "redis-mpool": "~0.4.0",
+    "request": "~2.62.0",
+    "step": "~0.0.6",
+    "step-profiler": "~0.3.0",
+    "turbo-carto": "0.7.1",
+    "underscore": "~1.6.0",
+    "windshaft": "1.19.0"
+  },
+  "devDependencies": {
+    "istanbul": "~0.4.3",
+    "jshint": "~2.6.0",
+    "mocha": "~1.21.4",
+    "nock": "~2.11.0",
+    "redis": "~0.8.6",
+    "semver": "~1.1.4",
+    "strftime": "~0.8.2"
+  },
+  "scripts": {
+    "preinstall": "make pre-install",
+    "test": "make test-all"
+  },
+  "engines": {
+    "node": ">=0.8 <0.11",
+    "npm": ">=2.14.16"
+  }
 }

run_tests.sh

@@ -4,6 +4,7 @@ OPT_CREATE_REDIS=yes # create the redis test environment
 OPT_CREATE_PGSQL=yes # create the PostgreSQL test environment
 OPT_DROP_REDIS=yes   # drop the redis test environment
 OPT_DROP_PGSQL=yes   # drop the PostgreSQL test environment
+OPT_COVERAGE=no      # run tests with coverage
 
 export PGAPPNAME=cartodb_tiler_tester
 
@@ -23,7 +24,10 @@ cleanup() {
         return;
       fi
     fi
+    redis-cli -p ${REDIS_PORT} info stats
+    redis-cli -p ${REDIS_PORT} info keyspace
     echo "Killing test redis pid ${PID_REDIS}"
+    #kill ${PID_REDIS_MONITOR}
     kill ${PID_REDIS}
   fi
   if test x"$OPT_DROP_PGSQL" = xyes; then
@@ -69,6 +73,10 @@ while [ -n "$1" ]; do
                 OPT_CREATE_REDIS=no
                 shift
                 continue
+        elif test "$1" = "--with-coverage"; then
+                OPT_COVERAGE=yes
+                shift
+                continue
         # This is kept for backward compatibility
         elif test "$1" = "--nocreate"; then
                 OPT_CREATE_REDIS=no
@@ -85,6 +93,7 @@ if [ -z "$1" ]; then
         echo "Options:" >&2
         echo " --nocreate   do not create the test environment on start" >&2
         echo " --nodrop     do not drop the test environment on exit" >&2
+        echo " --with-coverage   use istanbul to determine code coverage" >&2
         exit 1
 fi
 
@@ -112,8 +121,16 @@ cd -
 
 PATH=node_modules/.bin/:$PATH
 
-echo "Running tests"
-mocha -t 10000 -u tdd ${MOCHA_OPTS} ${TESTS}
+#redis-cli -p ${REDIS_PORT} monitor > /tmp/windshaft-cartodb.redis.log &
+#PID_REDIS_MONITOR=$!
+
+if test x"$OPT_COVERAGE" = xyes; then
+  echo "Running tests with coverage"
+  ./node_modules/.bin/istanbul cover node_modules/.bin/_mocha -- -u tdd -t 5000 ${TESTS}
+else
+  echo "Running tests"
+  mocha -u tdd -t 5000 ${TESTS}
+fi
 ret=$?
 
 cleanup

scripts/check-node-canvas.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+if [[ "$OSTYPE" == "darwin"* ]]; then
+    CAIRO_PKG_CONFIG=`pkg-config cairo --cflags-only-I 2> /dev/null`
+    RESULT=$?
+
+    if [[ ${RESULT} -ne 0 ]]; then
+        echo "###################################################################################"
+        echo "#                            PREINSTALL HOOK ERROR                                #"
+        echo "#---------------------------------------------------------------------------------#"
+        echo "#                                                                                 #"
+        echo "# node-canvas install error: some packages required by 'cairo' are not found      #"
+        echo "#                                                                                 #"
+        echo -e "# Use '\033[1mmake all\033[0m', it will take care of common/known issues                        #"
+        echo "#                                                                                 #"
+        echo "# As an alternative try:                                                          #"
+        echo "# Try to 'export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:/opt/X11/lib/pkgconfig' #"
+        echo "#                                                                                 #"
+        echo "# If problems persist visit: https://github.com/Automattic/node-canvas/wiki       #"
+        echo "#                                                                                 #"
+        echo "###################################################################################"
+        exit 1
+    fi
+fi

scripts/install.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+if [[ "$OSTYPE" == "darwin"* ]]; then
+    export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:/opt/X11/lib/pkgconfig
+fi
+
+npm install

scripts/lzma2config.js

@@ -0,0 +1,18 @@
+if (process.argv.length !== 3) {
+    console.error('Usage: node %s lzma_string', __filename);
+    process.exit(1);
+}
+
+var LZMA = require('lzma').LZMA;
+var lzmaWorker = new LZMA();
+var lzmaInput = decodeURIComponent(process.argv[2]);
+var lzmaBuffer = new Buffer(lzmaInput, 'base64')
+    .toString('binary')
+    .split('')
+    .map(function(c) {
+        return c.charCodeAt(0) - 128
+    });
+
+lzmaWorker.decompress(lzmaBuffer, function(result) {
+    console.log(JSON.stringify(JSON.parse(JSON.parse(result).config), null, 4));
+});

test/acceptance/analysis/analysis-layers-dataviews.js

@@ -0,0 +1,148 @@
+require('../../support/test_helper');
+
+var assert = require('../../support/assert');
+var TestClient = require('../../support/test-client');
+var dot = require('dot');
+
+describe('analysis-layers-dataviews', function() {
+
+    var multitypeStyleTemplate = dot.template([
+        "#points['mapnik::geometry_type'=1] {",
+        "  marker-fill-opacity: {{=it._opacity}};",
+        "  marker-line-color: #FFF;",
+        "  marker-line-width: 0.5;",
+        "  marker-line-opacity: {{=it._opacity}};",
+        "  marker-placement: point;",
+        "  marker-type: ellipse;",
+        "  marker-width: 8;",
+        "  marker-fill: {{=it._color}};",
+        "  marker-allow-overlap: true;",
+        "}",
+        "#lines['mapnik::geometry_type'=2] {",
+        "  line-color: {{=it._color}};",
+        "  line-width: 2;",
+        "  line-opacity: {{=it._opacity}};",
+        "}",
+        "#polygons['mapnik::geometry_type'=3] {",
+        "  polygon-fill: {{=it._color}};",
+        "  polygon-opacity: {{=it._opacity}};",
+        "  line-color: #FFF;",
+        "  line-width: 0.5;",
+        "  line-opacity: {{=it._opacity}};",
+        "}"
+    ].join('\n'));
+
+
+    function cartocss(color, opacity) {
+        return multitypeStyleTemplate({
+            _color: color || '#F11810',
+            _opacity: Number.isFinite(opacity) ? opacity : 1
+        });
+    }
+
+    function createMapConfig(layers, dataviews, analysis) {
+        return {
+            version: '1.5.0',
+            layers: layers,
+            dataviews: dataviews || {},
+            analyses: analysis || []
+        };
+    }
+
+    var DEFAULT_MULTITYPE_STYLE = cartocss();
+
+    var mapConfig = createMapConfig(
+        [
+            {
+                "type": "cartodb",
+                "options": {
+                    "source": {
+                        "id": "2570e105-7b37-40d2-bdf4-1af889598745"
+                    },
+                    "cartocss": DEFAULT_MULTITYPE_STYLE,
+                    "cartocss_version": "2.3.0"
+                }
+            }
+        ],
+        {
+            pop_max_histogram: {
+                source: {
+                    id: '2570e105-7b37-40d2-bdf4-1af889598745'
+                },
+                type: 'histogram',
+                options: {
+                    column: 'pop_max'
+                }
+            }
+        },
+        [
+            {
+                "id": "2570e105-7b37-40d2-bdf4-1af889598745",
+                "type": "source",
+                "params": {
+                    "query": "select * from populated_places_simple_reduced"
+                }
+            }
+        ]
+    );
+
+    it('should get histogram dataview', function(done) {
+        var testClient = new TestClient(mapConfig, 1234);
+
+        testClient.getDataview('pop_max_histogram', function(err, dataview) {
+            assert.ok(!err, err);
+
+            assert.equal(dataview.type, 'histogram');
+            assert.equal(dataview.bins_start, 0);
+
+            testClient.drain(done);
+        });
+    });
+
+    it('should get a filtered histogram dataview', function(done) {
+        var testClient = new TestClient(mapConfig, 1234);
+
+        var params = {
+            filters: {
+                dataviews: {
+                    pop_max_histogram: {
+                        min: 2e6
+                    }
+                }
+            }
+        };
+
+        testClient.getDataview('pop_max_histogram', params, function(err, dataview) {
+            assert.ok(!err, err);
+
+            assert.equal(dataview.type, 'histogram');
+            assert.equal(dataview.bins_start, 2008000);
+
+            testClient.drain(done);
+        });
+    });
+
+    it('should skip the filter when sending own_filter=0 for histogram dataview', function(done) {
+        var testClient = new TestClient(mapConfig, 1234);
+
+        var params = {
+            filters: {
+                dataviews: {
+                    pop_max_histogram: {
+                        min: 2e6
+                    }
+                }
+            },
+            own_filter: 0
+        };
+
+        testClient.getDataview('pop_max_histogram', params, function(err, dataview) {
+            assert.ok(!err, err);
+
+            assert.equal(dataview.type, 'histogram');
+            assert.equal(dataview.bins_start, 0);
+
+            testClient.drain(done);
+        });
+    });
+});

test/acceptance/analysis/analysis-layers-geojson.js

@@ -0,0 +1,81 @@
+require('../../support/test_helper');
+
+var assert = require('../../support/assert');
+var TestClient = require('../../support/test-client');
+
+describe('analysis-layers-dataviews-geojson', function() {
+
+    function createMapConfig(layers, dataviews, analysis) {
+        return {
+            version: '1.5.0',
+            layers: layers,
+            dataviews: dataviews || {},
+            analyses: analysis || []
+        };
+    }
+
+    var CARTOCSS = [
+        "#points {",
+        "  marker-fill-opacity: 1.0;",
+        "  marker-line-color: #FFF;",
+        "  marker-line-width: 0.5;",
+        "  marker-line-opacity: 1.0;",
+        "  marker-placement: point;",
+        "  marker-type: ellipse;",
+        "  marker-width: 8;",
+        "  marker-fill: red;",
+        "  marker-allow-overlap: true;",
+        "}"
+    ].join('\n');
+
+    var mapConfig = createMapConfig(
+        [
+            {
+                "type": "cartodb",
+                "options": {
+                    "source": {
+                        "id": "2570e105-7b37-40d2-bdf4-1af889598745"
+                    },
+                    "cartocss": CARTOCSS,
+                    "cartocss_version": "2.3.0"
+                }
+            }
+        ],
+        {
+            pop_max_histogram: {
+                source: {
+                    id: '2570e105-7b37-40d2-bdf4-1af889598745'
+                },
+                type: 'histogram',
+                options: {
+                    column: 'pop_max'
+                }
+            }
+        },
+        [
+            {
+                "id": "2570e105-7b37-40d2-bdf4-1af889598745",
+                "type": "source",
+                "params": {
+                    "query": "select * from populated_places_simple_reduced"
+                }
+            }
+        ]
+    );
+
+    it('should get pop_max column from dataview', function(done) {
+        var testClient = new TestClient(mapConfig, 1234);
+
+        testClient.getTile(0, 0, 0, {format: 'geojson', layers: 0}, function(err, res, geojson) {
+            assert.ok(!err, err);
+
+            assert.ok(Array.isArray(geojson.features));
+            assert.ok(geojson.features.length > 0);
+            var feature = geojson.features[0];
+            assert.ok(feature.properties.hasOwnProperty('pop_max'), 'Missing pop_max property');
+
+            testClient.drain(done);
+        });
+    });
+
+});