Release 5.2.0

Add Mapnik 'cache-features' option

.gitignore

@@ -2,10 +2,12 @@ node_modules*
 config.status*
 config/environments/*.js
 .idea
+.vscode
+.nvmrc
 tools/munin/windshaft.conf
 logs/
 pids/
 redis.pid
-test.log
-npm-debug.log
+*.log
 coverage/
+.DS_Store

.jshintignore

@@ -1,4 +1,3 @@
 test/results/
 test/monkey/
 test/benchmark.js
-test/support/

.jshintrc

@@ -40,7 +40,7 @@
     "debug"         : false,     // true: Allow debugger statements e.g. browser breakpoints.
 //    "eqnull"        : false,     // true: Tolerate use of `== null`
 //    "es5"           : false,     // true: Allow ES5 syntax (ex: getters and setters)
-//    "esnext"        : false,     // true: Allow ES.next (ES6) syntax (ex: `const`)
+    "esnext"        : true,     // true: Allow ES.next (ES6) syntax (ex: `const`)
 //    "moz"           : false,     // true: Allow Mozilla specific syntax (extends and overrides esnext features)
 //    // (ex: `for each`, multiple try/catch, function expression…)
 //    "evil"          : false,     // true: Tolerate use of `eval` and `new Function()`

.travis.yml

@@ -1,29 +1,14 @@
-sudo: false
+sudo: required
+dist: trusty
 
-addons:
-  postgresql: "9.3"
-  apt:
-    packages:
-      - pkg-config
-      - libcairo2-dev
-      - libjpeg8-dev
-      - libgif-dev
+services:
+  - docker
 
 before_install:
-  - npm install -g npm@2
-  - createdb template_postgis
-  - createuser publicuser
-  - psql -c "CREATE EXTENSION postgis" template_postgis
+  - docker pull cartoimages/windshaft-testing
 
-env:
-  - NPROCS=1 JOBS=1 PGUSER=postgres
+script:
+  - docker run -e POSTGIS_VERSION=2.4 -v `pwd`:/srv cartoimages/windshaft-testing bash docker-test.sh
 
-language: node_js
-node_js:
-  - "0.10"
+language: generic
 
-notifications:
-  irc:
-    channels:
-      - "irc.freenode.org#cartodb"
-    use_notice: true

CONTRIBUTING.md

@@ -8,4 +8,4 @@ We love pull requests from everyone, see [Contributing to Open Source on GitHub]
 
 ## Submitting Contributions
 
-* You will need to sign a Contributor License Agreement (CLA) before making a submission. [Learn more here](https://cartodb.com/contributing).
+* You will need to sign a Contributor License Agreement (CLA) before making a submission. [Learn more here](https://carto.com/contributions).

HOWTO_RELEASE

@@ -1,11 +1,10 @@
 1. Test (make clean all check), fix if broken before proceeding
-2. Ensure proper version in package.json 
+2. Ensure proper version in package.json
 3. Ensure NEWS section exists for the new version, review it, add release date
-4. Recreate npm-shrinkwrap.json with: `npm install --no-shrinkwrap && npm shrinkwrap`
-5. Commit package.json, npm-shrinwrap.json, NEWS
+4. If there are modified dependencies in package.json, update them with `yarn upgrade {{package_name}}@{{version}}`
+5. Commit package.json, yarn.lock, NEWS
 6. git tag -a Major.Minor.Patch # use NEWS section as content
-7. Announce on cartodb@googlegroups.com
-8. Stub NEWS/package for next version
+7. Stub NEWS/package for next version
 
 Versions:
 

INSTALL.md

@@ -4,11 +4,11 @@
 Make sure that you have the requirements needed. These are
 
 - Core
-  - Node.js >=0.8
-  - npm >=1.2.1 <2.0.0
+  - Node.js >=6.9.x
+  - yarn >=0.27.5 <1.0.0
   - PostgreSQL >8.3.x, PostGIS >1.5.x
   - Redis >2.4.0 (http://www.redis.io)
-  - Mapnik 2.0.1, 2.0.2, 2.1.0, 2.2.0, 2.3.0. See [Installing Mapnik](https://github.com/CartoDB/Windshaft#installing-mapnik).
+  - Mapnik >3.x. See [Installing Mapnik](https://github.com/CartoDB/Windshaft#installing-mapnik).
   - Windshaft: check [Windshaft dependencies and installation notes](https://github.com/CartoDB/Windshaft#dependencies)
   - libcairo2-dev, libpango1.0-dev, libjpeg8-dev and libgif-dev for server side canvas support
 
@@ -43,11 +43,11 @@ psql -d template_postgis -c 'CREATE EXTENSION postgis;'
 To fetch and build all node-based dependencies, run:
 
 ```
-npm install
+yarn
 ```
 
-Note that the ```npm install``` step will populate the node_modules/
+Note that the ```yarn``` step will populate the node_modules/
 directory with modules, some of which being compiled on demand. If you
 happen to have startup errors you may need to force rebuilding those
 modules. At any time just wipe out the node_modules/ directory and run
-```npm install``` again.
+```yarn``` again.

Makefile

@@ -7,7 +7,7 @@ all:
 	@$(SHELL) ./scripts/install.sh
 
 clean:
-	rm -rf node_modules/*
+	rm -rf node_modules/
 
 distclean: clean
 	rm config.status*
@@ -16,7 +16,7 @@ config.status--test:
 	./configure --environment=test
 
 config/environments/test.js: config.status--test
-	./config.status--test 
+	./config.status--test
 
 TEST_SUITE := $(shell find test/{acceptance,integration,unit} -name "*.js")
 TEST_SUITE_UNIT := $(shell find test/unit -name "*.js")
@@ -43,7 +43,7 @@ jshint:
 	@echo "***jshint***"
 	@./node_modules/.bin/jshint lib/ test/ app.js
 
-test-all: jshint test
+test-all: test jshint
 
 coverage:
 	@RUNTESTFLAGS=--with-coverage make test

README.md

@@ -32,14 +32,14 @@ Upgrading
 Checkout your commit/branch. If you need to reinstall dependencies (you can check [NEWS](NEWS.md)) do the following:
 
 ```
-rm -rf node_modules; npm install
+rm -rf node_modules; yarn
 ```
 
 Run
 ---
 
 ```
-node app.js <env> 
+node app.js <env>
 ```
 
 Where <env> is the name of a configuration file under config/environments/.
@@ -71,12 +71,12 @@ See [CONTRIBUTING.md](CONTRIBUTING.md).
 ### Developing with a custom windshaft version
 
 If you plan or want to use a custom / not released yet version of windshaft (or any other dependency) the best option is
-to use `npm link`. You can read more about it at [npm-link: Symlink a package folder](https://docs.npmjs.com/cli/link).
+to use `yarn link`. You can read more about it at [yarn-link: Symlink a package folder](https://yarnpkg.com/en/docs/cli/link).
 
 **Quick start**:
 
 ```shell
-~/windshaft-directory $ npm install
-~/windshaft-directory $ npm link
-~/windshaft-cartodb-directory $ npm link windshaft
+~/windshaft-directory $ yarn
+~/windshaft-directory $ yarn link
+~/windshaft-cartodb-directory $ yarn link windshaft
 ```

app.js

@@ -2,23 +2,48 @@ var http = require('http');
 var https = require('https');
 var path = require('path');
 var fs = require('fs');
-
 var _ = require('underscore');
-
-var ENVIRONMENT;
-if ( process.argv[2] ) {
-    ENVIRONMENT = process.argv[2];
-} else if ( process.env.NODE_ENV ) {
-    ENVIRONMENT = process.env.NODE_ENV;
-} else {
-    ENVIRONMENT = 'development';
-}
+var semver = require('semver');
+const setICUEnvVariable = require('./lib/cartodb/utils/icu_data_env_setter');
 
 // jshint undef:false
 var log = console.log.bind(console);
 var logError = console.error.bind(console);
 // jshint undef:true
 
+var nodejsVersion = process.versions.node;
+if (!semver.satisfies(nodejsVersion, '>=6.9.0')) {
+    logError(`Node version ${nodejsVersion} is not supported, please use Node.js 6.9 or higher.`);
+    process.exit(1);
+}
+
+// This function should be called before the require('yargs').
+setICUEnvVariable();
+
+var argv = require('yargs')
+    .usage('Usage: $0 <environment> [options]')
+    .help('h')
+    .example(
+    '$0 production -c /etc/sql-api/config.js',
+    'start server in production environment with /etc/sql-api/config.js as config file'
+)
+    .alias('h', 'help')
+    .alias('c', 'config')
+    .nargs('c', 1)
+    .describe('c', 'Load configuration from path')
+    .argv;
+
+var environmentArg = argv._[0] || process.env.NODE_ENV || 'development';
+var configurationFile = path.resolve(argv.config || './config/environments/' + environmentArg + '.js');
+if (!fs.existsSync(configurationFile)) {
+    logError('Configuration file "%s" does not exist', configurationFile);
+    process.exit(1);
+}
+
+global.environment = require(configurationFile);
+var ENVIRONMENT = argv._[0] || process.env.NODE_ENV || global.environment.environment;
+process.env.NODE_ENV = ENVIRONMENT;
+
 var availableEnvironments = {
     production: true,
     staging: true,
@@ -33,16 +58,6 @@ if (!availableEnvironments[ENVIRONMENT]){
 }
 
 process.env.NODE_ENV = ENVIRONMENT;
-
-// set environment specific variables
-global.environment = require('./config/environments/' + ENVIRONMENT);
-
-global.log4js = require('log4js');
-var log4js_config = {
-  appenders: [],
-  replaceConsole: true
-};
-
 if (global.environment.uv_threadpool_size) {
     process.env.UV_THREADPOOL_SIZE = global.environment.uv_threadpool_size;
 }
@@ -58,25 +73,31 @@ var agentOptions = _.defaults(global.environment.httpAgent || {}, {
 http.globalAgent = new http.Agent(agentOptions);
 https.globalAgent = new https.Agent(agentOptions);
 
+
+global.log4js = require('log4js');
+var log4jsConfig = {
+    appenders: [],
+    replaceConsole: true
+};
+
 if ( global.environment.log_filename ) {
-  var logdir = path.dirname(global.environment.log_filename);
-  // See cwd inlog4js.configure call below
-  logdir = path.resolve(__dirname, logdir);
-  if ( ! fs.existsSync(logdir) ) {
-    logError("Log filename directory does not exist: " + logdir);
-    process.exit(1);
-  }
-  log("Logs will be written to " + global.environment.log_filename);
-  log4js_config.appenders.push(
-    { type: "file", filename: global.environment.log_filename }
-  );
+    var logFilename = path.resolve(global.environment.log_filename);
+    var logDirectory = path.dirname(logFilename);
+    if (!fs.existsSync(logDirectory)) {
+        logError("Log filename directory does not exist: " + logDirectory);
+        process.exit(1);
+    }
+    log("Logs will be written to " + logFilename);
+    log4jsConfig.appenders.push(
+        { type: "file", absolute: true, filename: logFilename }
+    );
 } else {
-  log4js_config.appenders.push(
-    { type: "console", layout: { type:'basic' } }
-  );
+    log4jsConfig.appenders.push(
+        { type: "console", layout: { type:'basic' } }
+    );
 }
 
-global.log4js.configure(log4js_config, { cwd: __dirname });
+global.log4js.configure(log4jsConfig);
 global.logger = global.log4js.getLogger();
 
 global.environment.api_hostname = require('os').hostname().split('.')[0];
@@ -99,6 +120,8 @@ var listener = server.listen(serverOptions.bind.port, serverOptions.bind.host, b
 var version = require("./package").version;
 
 listener.on('listening', function() {
+    log("Using Node.js %s", process.version);
+    log('Using configuration file "%s"', configurationFile);
     log(
         "Windshaft tileserver %s started on %s:%s PID=%d (%s)",
         version, serverOptions.bind.host, serverOptions.bind.port, process.pid, ENVIRONMENT
@@ -114,7 +137,7 @@ setInterval(function() {
 
 process.on('SIGHUP', function() {
     global.log4js.clearAndShutdownAppenders(function() {
-        global.log4js.configure(log4js_config);
+        global.log4js.configure(log4jsConfig);
         global.logger = global.log4js.getLogger();
         log('Log files reloaded');
     });
@@ -123,3 +146,17 @@ process.on('SIGHUP', function() {
 process.on('uncaughtException', function(err) {
     global.logger.error('Uncaught exception: ' + err.stack);
 });
+
+if (global.gc) {
+    var gcInterval = Number.isFinite(global.environment.gc_interval) ?
+        global.environment.gc_interval :
+        10000;
+
+    if (gcInterval > 0) {
+        setInterval(function gcForcedCycle() {
+            var start = Date.now();
+            global.gc();
+            global.statsClient.timing('windshaft.gc', Date.now() - start);
+        }, gcInterval);
+    }
+}

config/environments/development.js.example

@@ -6,6 +6,9 @@ var config = {
     // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
     // See http://docs.libuv.org/en/latest/threadpool.html
     ,uv_threadpool_size: undefined
+    // Time in milliseconds to force GC cycle.
+    // Disable by using <=0 value.
+    ,gc_interval: 10000
     // Regular expression pattern to extract username
     // from hostname. Must have a single grabbing block.
     ,user_from_host: '^(.*)\\.localhost'
@@ -23,6 +26,21 @@ var config = {
     // "tiles/layergroup" is for compatibility with versions up to 1.6.x
     ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)'
 
+    // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status.
+    //
+    // This URLs depend on how `base_url_detached` and `user_from_host` are configured: the application can be
+    // configured to accept request with the {user} in the header host or in the request path.
+    // It also might depend on the configured cdn_url via `serverMetadata.cdn_url`.
+    //
+    // This template allows to make the endpoints generation more flexible, the template exposes the following params:
+    //  1. {{=it.cdn_url}}: will be used when `serverMetadata.cdn_url` exists.
+    //  2. {{=it.user}}: will use the username as extraced from `user_from_host` or `base_url_detached`.
+    //  3. {{=it.port}}: will use the `port` from this very same configuration file.
+    ,resources_url_templates: {
+        http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map',
+        https: 'http://localhost.lan:{{=it.port}}/user/{{=it.user}}/api/v1/map'
+    }
+
     // Maximum number of connections for one process
     // 128 is a good value with a limit of 1024 open file descriptors
     ,maxConnections:128
@@ -37,7 +55,7 @@ var config = {
     ,socket_timeout: 600000
     ,enable_cors: true
     ,cache_enabled: false
-    ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
+    ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler]) (:res[X-Tiler-Errors])'
     // If log_filename is given logs will be written
     // there, in append mode. Otherwise stdout is used (default).
     // Log file will be re-opened on receiving the HUP signal
@@ -62,6 +80,7 @@ var config = {
         extent: "-180,-90,180,90",
         srid: 4326,
         */
+        // max number of rows to return when querying data, 0 means no limit
         row_limit: 65535,
         simplify_geometries: true,
         use_overviews: true, // use overviews to retrieve raster
@@ -88,6 +107,20 @@ var config = {
       // Milliseconds since last access before renderer cache item expires
       cache_ttl: 60000,
       statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
+      mvt: {
+        //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik,
+        //PostGIS 2.4 is required for this to work
+        //If disabled it will use Mapnik MVT generation
+        usePostGIS: false,
+        dbPoolParams: {
+              // maximum number of resources to create at any given time
+              size: 16,
+              // max milliseconds a resource can go unused before it should be destroyed
+              idleTimeout: 3000,
+              // frequency to check for idle resources
+              reapInterval: 1000
+          }
+      },
       mapnik: {
           // The size of the pool of internal mapnik backend
           // This pool size is per mapnik renderer created in Windshaft's RendererFactory
@@ -95,6 +128,10 @@ var config = {
           // Important: check the configuration of uv_threadpool_size to use suitable value
           poolSize: 8,
 
+          // Whether grainstore will use a child process or not to transform CartoCSS into Mapnik XML.
+          // This will prevent blocking the main thread.
+          useCartocssWorkers: false,
+
           // Metatile is the number of tiles-per-side that are going
           // to be rendered at once. If all of them will be requested
           // we'd have saved time. If only one will be used, we'd have
@@ -167,7 +204,11 @@ var config = {
               // which cost is no more expensive than snapping and results are
               // much closer to the original geometry
               removeRepeatedPoints: false // this requires postgis >=2.2
-          }
+          },
+
+          // If enabled Mapnik will reuse the features retrieved from the database
+          // instead of requesting them once per style inside a layer
+          'cache-features': true
 
       },
       http: {
@@ -209,6 +250,18 @@ var config = {
             endpoint: 'http://127.0.0.1:8080/api/v2/sql/job',
             // the template to use for adding the host header in the batch api requests
             hostHeaderTemplate: '{{=it.username}}.localhost.lan'
+        },
+        logger: {
+            // If filename is given logs comming from analysis client  will be written
+            // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
+            // Log file will be re-opened on receiving the HUP signal
+            filename: '/tmp/analysis.log'
+        },
+        // Define max execution time in ms for analyses or tags
+        // If analysis or tag are not found in redis this values will be used as default.
+        limits: {
+            moran: { timeout: 120000, maxNumberOfRows: 1e5 },
+            cpu2x: { timeout: 60000 }
         }
     }
     ,millstone: {
@@ -289,8 +342,7 @@ var config = {
         // whether the affected tables for a given SQL must query directly postgresql or use the SQL API
         cdbQueryTablesFromPostgres: true,
         // whether in mapconfig is available stats & metadata for each layer
-        layerMetadata: true
-
+        layerStats: true
     }
 };
 

config/environments/production.js.example

@@ -6,6 +6,9 @@ var config = {
     // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
     // See http://docs.libuv.org/en/latest/threadpool.html
     ,uv_threadpool_size: undefined
+    // Time in milliseconds to force GC cycle.
+    // Disable by using <=0 value.
+    ,gc_interval: 10000
     // Regular expression pattern to extract username
     // from hostname. Must have a single grabbing block.
     ,user_from_host: '^(.*)\\.cartodb\\.com$'
@@ -23,6 +26,21 @@ var config = {
     // "tiles/layergroup" is for compatibility with versions up to 1.6.x
     ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)'
 
+    // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status.
+    //
+    // This URLs depend on how `base_url_detached` and `user_from_host` are configured: the application can be
+    // configured to accept request with the {user} in the header host or in the request path.
+    // It also might depend on the configured cdn_url via `serverMetadata.cdn_url`.
+    //
+    // This template allows to make the endpoints generation more flexible, the template exposes the following params:
+    //  1. {{=it.cdn_url}}: will be used when `serverMetadata.cdn_url` exists.
+    //  2. {{=it.user}}: will use the username as extraced from `user_from_host` or `base_url_detached`.
+    //  3. {{=it.port}}: will use the `port` from this very same configuration file.
+    ,resources_url_templates: {
+        http: 'http://{{=it.cdn_url}}/{{=it.user}}/api/v1/map',
+        https: 'https://{{=it.cdn_url}}/{{=it.user}}/api/v1/map'
+    }
+
     // Maximum number of connections for one process
     // 128 is a good value with a limit of 1024 open file descriptors
     ,maxConnections:128
@@ -37,7 +55,7 @@ var config = {
     ,socket_timeout: 600000
     ,enable_cors: true
     ,cache_enabled: true
-    ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
+    ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler]) (:res[X-Tiler-Errors])'
     // If log_filename is given logs will be written
     // there, in append mode. Otherwise stdout is used (default).
     // Log file will be re-opened on receiving the HUP signal
@@ -56,6 +74,7 @@ var config = {
         host: '127.0.0.1',
         port: 6432,
         extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
+        // max number of rows to return when querying data, 0 means no limit
         row_limit: 65535,
         /*
          * Set persist_connection to false if you want
@@ -82,6 +101,20 @@ var config = {
       // Milliseconds since last access before renderer cache item expires
       cache_ttl: 60000,
       statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
+      mvt: {
+        //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik,
+        //PostGIS 2.4 is required for this to work
+        //If disabled it will use Mapnik MVT generation
+        usePostGIS: false,
+        dbPoolParams: {
+              // maximum number of resources to create at any given time
+              size: 16,
+              // max milliseconds a resource can go unused before it should be destroyed
+              idleTimeout: 3000,
+              // frequency to check for idle resources
+              reapInterval: 1000
+          }
+      },
       mapnik: {
           // The size of the pool of internal mapnik backend
           // This pool size is per mapnik renderer created in Windshaft's RendererFactory
@@ -89,6 +122,10 @@ var config = {
           // Important: check the configuration of uv_threadpool_size to use suitable value
           poolSize: 8,
 
+          // Whether grainstore will use a child process or not to transform CartoCSS into Mapnik XML.
+          // This will prevent blocking the main thread.
+          useCartocssWorkers: false,
+
           // Metatile is the number of tiles-per-side that are going
           // to be rendered at once. If all of them will be requested
           // we'd have saved time. If only one will be used, we'd have
@@ -161,7 +198,11 @@ var config = {
               // which cost is no more expensive than snapping and results are
               // much closer to the original geometry
               removeRepeatedPoints: false // this requires postgis >=2.2
-          }
+          },
+
+          // If enabled Mapnik will reuse the features retrieved from the database
+          // instead of requesting them once per style inside a layer
+          'cache-features': true
 
       },
       http: {
@@ -203,6 +244,18 @@ var config = {
             endpoint: 'http://127.0.0.1:8080/api/v2/sql/job',
             // the template to use for adding the host header in the batch api requests
             hostHeaderTemplate: '{{=it.username}}.localhost.lan'
+        },
+        logger: {
+            // If filename is given logs comming from analysis client  will be written
+            // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
+            // Log file will be re-opened on receiving the HUP signal
+            filename: 'logs/analysis.log'
+        },
+        // Define max execution time in ms for analyses or tags
+        // If analysis or tag are not found in redis this values will be used as default.
+        limits: {
+            moran: { timeout: 120000, maxNumberOfRows: 1e5 },
+            cpu2x: { timeout: 60000 }
         }
     }
     ,millstone: {
@@ -289,7 +342,7 @@ var config = {
         // whether the affected tables for a given SQL must query directly postgresql or use the SQL API
         cdbQueryTablesFromPostgres: true,
         // whether in mapconfig is available stats & metadata for each layer
-        layerMetadata: false
+        layerStats: false
     }
 };
 

config/environments/staging.js.example

@@ -6,6 +6,9 @@ var config = {
     // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
     // See http://docs.libuv.org/en/latest/threadpool.html
     ,uv_threadpool_size: undefined
+    // Time in milliseconds to force GC cycle.
+    // Disable by using <=0 value.
+    ,gc_interval: 10000
     // Regular expression pattern to extract username
     // from hostname. Must have a single grabbing block.
     ,user_from_host: '^(.*)\\.cartodb\\.com$'
@@ -23,6 +26,21 @@ var config = {
     // "/tiles/layergroup" is for compatibility with versions up to 1.6.x
     ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)'
 
+    // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status.
+    //
+    // This URLs depend on how `base_url_detached` and `user_from_host` are configured: the application can be
+    // configured to accept request with the {user} in the header host or in the request path.
+    // It also might depend on the configured cdn_url via `serverMetadata.cdn_url`.
+    //
+    // This template allows to make the endpoints generation more flexible, the template exposes the following params:
+    //  1. {{=it.cdn_url}}: will be used when `serverMetadata.cdn_url` exists.
+    //  2. {{=it.user}}: will use the username as extraced from `user_from_host` or `base_url_detached`.
+    //  3. {{=it.port}}: will use the `port` from this very same configuration file.
+    ,resources_url_templates: {
+        http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map',
+        https: 'https://{{=it.cdn_url}}/{{=it.user}}/api/v1/map'
+    }
+
     // Maximum number of connections for one process
     // 128 is a good value with a limit of 1024 open file descriptors
     ,maxConnections:128
@@ -37,7 +55,7 @@ var config = {
     ,socket_timeout: 600000
     ,enable_cors: true
     ,cache_enabled: true
-    ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms (:res[X-Tiler-Profiler]) -> :res[Content-Type]'
+    ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms (:res[X-Tiler-Profiler]) -> :res[Content-Type] (:res[X-Tiler-Errors])'
     // If log_filename is given logs will be written
     // there, in append mode. Otherwise stdout is used (default).
     // Log file will be re-opened on receiving the HUP signal
@@ -56,6 +74,7 @@ var config = {
         host: '127.0.0.1',
         port: 6432,
         extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
+        // max number of rows to return when querying data, 0 means no limit
         row_limit: 65535,
         simplify_geometries: true,
         use_overviews: true, // use overviews to retrieve raster
@@ -82,6 +101,20 @@ var config = {
       // Milliseconds since last access before renderer cache item expires
       cache_ttl: 60000,
       statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
+      mvt: {
+        //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik,
+        //PostGIS 2.4 is required for this to work
+        //If disabled it will use Mapnik MVT generation
+        usePostGIS: false,
+        dbPoolParams: {
+              // maximum number of resources to create at any given time
+              size: 16,
+              // max milliseconds a resource can go unused before it should be destroyed
+              idleTimeout: 3000,
+              // frequency to check for idle resources
+              reapInterval: 1000
+          }
+      },
       mapnik: {
           // The size of the pool of internal mapnik backend
           // This pool size is per mapnik renderer created in Windshaft's RendererFactory
@@ -89,6 +122,10 @@ var config = {
           // Important: check the configuration of uv_threadpool_size to use suitable value
           poolSize: 8,
 
+          // Whether grainstore will use a child process or not to transform CartoCSS into Mapnik XML.
+          // This will prevent blocking the main thread.
+          useCartocssWorkers: false,
+
           // Metatile is the number of tiles-per-side that are going
           // to be rendered at once. If all of them will be requested
           // we'd have saved time. If only one will be used, we'd have
@@ -161,7 +198,11 @@ var config = {
               // which cost is no more expensive than snapping and results are
               // much closer to the original geometry
               removeRepeatedPoints: false // this requires postgis >=2.2
-          }
+          },
+
+          // If enabled Mapnik will reuse the features retrieved from the database
+          // instead of requesting them once per style inside a layer
+          'cache-features': true
 
       },
       http: {
@@ -203,6 +244,18 @@ var config = {
             endpoint: 'http://127.0.0.1:8080/api/v2/sql/job',
             // the template to use for adding the host header in the batch api requests
             hostHeaderTemplate: '{{=it.username}}.localhost.lan'
+        },
+        logger: {
+            // If filename is given logs comming from analysis client  will be written
+            // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
+            // Log file will be re-opened on receiving the HUP signal
+            filename: 'logs/analysis.log'
+        },
+        // Define max execution time in ms for analyses or tags
+        // If analysis or tag are not found in redis this values will be used as default.
+        limits: {
+            moran: { timeout: 120000, maxNumberOfRows: 1e5 },
+            cpu2x: { timeout: 60000 }
         }
     }
     ,millstone: {
@@ -289,7 +342,7 @@ var config = {
         // whether the affected tables for a given SQL must query directly postgresql or use the SQL API
         cdbQueryTablesFromPostgres: true,
         // whether in mapconfig is available stats & metadata for each layer
-        layerMetadata: true
+        layerStats: true
     }
 };
 

config/environments/test.js.example

@@ -6,6 +6,9 @@ var config = {
     // Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
     // See http://docs.libuv.org/en/latest/threadpool.html
     ,uv_threadpool_size: undefined
+    // Time in milliseconds to force GC cycle.
+    // Disable by using <=0 value.
+    ,gc_interval: 10000
     // Regular expression pattern to extract username
     // from hostname. Must have a single grabbing block.
     ,user_from_host: '(.*)'
@@ -23,6 +26,20 @@ var config = {
     // "tiles/layergroup" is for compatibility with versions up to 1.6.x
     ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)'
 
+    // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status.
+    //
+    // This URLs depend on how `base_url_detached` and `user_from_host` are configured: the application can be
+    // configured to accept request with the {user} in the header host or in the request path.
+    // It also might depend on the configured cdn_url via `serverMetadata.cdn_url`.
+    //
+    // This template allows to make the endpoints generation more flexible, the template exposes the following params:
+    //  1. {{=it.cdn_url}}: will be used when `serverMetadata.cdn_url` exists.
+    //  2. {{=it.user}}: will use the username as extraced from `user_from_host` or `base_url_detached`.
+    //  3. {{=it.port}}: will use the `port` from this very same configuration file.
+    ,resources_url_templates: {
+        http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map'
+    }
+
     // Maximum number of connections for one process
     // 128 is a good value with a limit of 1024 open file descriptors
     ,maxConnections:128
@@ -37,7 +54,7 @@ var config = {
     ,socket_timeout: 600000
     ,enable_cors: true
     ,cache_enabled: false
-    ,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
+    ,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler]) (:res[X-Tiler-Errors])'
     // If log_filename is given logs will be written
     // there, in append mode. Otherwise stdout is used (default).
     // Log file will be re-opened on receiving the HUP signal
@@ -56,6 +73,7 @@ var config = {
         host: '127.0.0.1',
         port: 5432,
         extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
+        // max number of rows to return when querying data, 0 means no limit
         row_limit: 65535,
         simplify_geometries: true,
         use_overviews: true, // use overviews to retrieve raster
@@ -82,6 +100,20 @@ var config = {
       // Milliseconds since last access before renderer cache item expires
       cache_ttl: 60000,
       statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
+      mvt: {
+        //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik,
+        //PostGIS 2.4 is required for this to work
+        //If disabled it will use Mapnik MVT generation
+        usePostGIS: false,
+        dbPoolParams: {
+              // maximum number of resources to create at any given time
+              size: 16,
+              // max milliseconds a resource can go unused before it should be destroyed
+              idleTimeout: 3000,
+              // frequency to check for idle resources
+              reapInterval: 1000
+          }
+      },
       mapnik: {
           // The size of the pool of internal mapnik backend
           // This pool size is per mapnik renderer created in Windshaft's RendererFactory
@@ -89,6 +121,10 @@ var config = {
           // Important: check the configuration of uv_threadpool_size to use suitable value
           poolSize: 8,
 
+          // Whether grainstore will use a child process or not to transform CartoCSS into Mapnik XML.
+          // This will prevent blocking the main thread.
+          useCartocssWorkers: false,
+
           // Metatile is the number of tiles-per-side that are going
           // to be rendered at once. If all of them will be requested
           // we'd have saved time. If only one will be used, we'd have
@@ -161,7 +197,12 @@ var config = {
               // which cost is no more expensive than snapping and results are
               // much closer to the original geometry
               removeRepeatedPoints: false // this requires postgis >=2.2
-          }
+          },
+
+          // If enabled Mapnik will reuse the features retrieved from the database
+          // instead of requesting them once per style inside a layer
+          'cache-features': true
+
       },
       http: {
           timeout: 2000, // the timeout in ms for a http tile request
@@ -204,6 +245,18 @@ var config = {
             endpoint: 'http://127.0.0.1:8080/api/v2/sql/job',
             // the template to use for adding the host header in the batch api requests
             hostHeaderTemplate: '{{=it.username}}.localhost.lan'
+        },
+        logger: {
+            // If filename is given logs comming from analysis client  will be written
+            // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
+            // Log file will be re-opened on receiving the HUP signal
+            filename: 'node-windshaft.log'
+        },
+        // Define max execution time in ms for analyses or tags
+        // If analysis or tag are not found in redis this values will be used as default.
+        limits: {
+            moran: { timeout: 120000, maxNumberOfRows: 1e5 },
+            cpu2x: { timeout: 60000 }
         }
     }
     ,millstone: {
@@ -284,7 +337,7 @@ var config = {
         // whether the affected tables for a given SQL must query directly postgresql or use the SQL API
         cdbQueryTablesFromPostgres: true,
         // whether in mapconfig is available stats & metadata for each layer
-        layerMetadata: true
+        layerStats: true
     }
 };
 

docker-test.sh

@@ -0,0 +1,11 @@
+export NPROCS=1 && export JOBS=1 && export CXX=g++-4.9 && export PGUSER=postgres
+
+npm install -g yarn@0.27.5
+yarn
+
+ /etc/init.d/postgresql start
+
+createdb template_postgis && createuser publicuser
+psql -c "CREATE EXTENSION postgis" template_postgis
+
+POSTGIS_VERSION=2.4 npm test

docs/Map-API.md

@@ -1,11 +1,11 @@
 # Maps API
 
-The CartoDB Maps API allows you to generate maps based on data hosted in your CartoDB account and apply custom SQL and CartoCSS to the data. The API generates a XYZ-based URL to fetch Web Mercator projected tiles, using web clients such as [Leaflet](http://leafletjs.com), [Google Maps](https://developers.google.com/maps/), or [OpenLayers](http://openlayers.org/).
+The CARTO Maps API allows you to generate maps based on data hosted in your CARTO account and apply custom SQL and CartoCSS to the data. The API generates a XYZ-based URL to fetch Web Mercator projected tiles, using web clients such as [Leaflet](http://leafletjs.com), [Google Maps](https://developers.google.com/maps/), or [OpenLayers](http://openlayers.org/).
 
 You can create two types of maps with the Maps API:
 
 - **Anonymous Maps**  
-  You can create maps using your CartoDB public data. Any client can change the read-only SQL and CartoCSS parameters that generate the map tiles. These maps can be created from a JavaScript application alone and no authenticated calls are needed. See [this CartoDB.js example](/cartodb-platform/cartodb-js/getting-started/).
+  You can create maps using your CARTO public data. Any client can change the read-only SQL and CartoCSS parameters that generate the map tiles. These maps can be created from a JavaScript application alone and no authenticated calls are needed. See [this CARTO.js example](/carto-engine/carto-js/getting-started/).
 
 - **Named Maps**  
   There are also maps that have access to your private data. These maps require an owner to setup and modify any SQL and CartoCSS parameters and are not modifiable without new setup calls.
@@ -17,3 +17,4 @@ You can create two types of maps with the Maps API:
 * [Anonymous Maps](anonymous_maps.md)
 * [Named Maps](named_maps.md)
 * [Static Maps API](static_maps_api.md)
+* [MapConfig File Format]([local file in the docs repo](https://github.com/CartoDB/docs/blob/master/_app/_mapsapi/06-mapconfig.md))

docs/MapConfig-Aggregation-extension.md

@@ -0,0 +1,62 @@
+# 1. Purpose
+
+This specification describes an extension for
+[MapConfig 1.7.0](https://github.com/CartoDB/Windshaft/blob/master/doc/MapConfig-1.7.0.md) version.
+
+
+# 2. Changes over specification
+
+This extension introduces a new layer options for aggregated data tile generation.
+
+## 2.1 Aggregation options
+
+The layer options attribute is extended with a new optional `aggregation` attribute.
+The value of this attribute can be `false` to explicitly disable aggregation for the layer.
+
+```javascript
+{
+    aggregation: {
+
+        // OPTIONAL
+        // string, defines the placement of aggregated geometries. Can be one of:
+        // * "point-sample", the default places geometries at a sample point (one of the aggregated geometries)
+        // * "point-grid" places geometries at the center of the aggregation grid cells
+        // * "centroid" places geometriea at the average position of the aggregated points
+        // See https://github.com/CartoDB/Windshaft-cartodb/blob/master/docs/aggregation.md#placement for more details
+        placement: "point-sample",
+
+        // OPTIONAL
+        // object, defines the columns of the aggregated datasets. Each property corresponds to a columns name and
+        // should contain an object with two properties: "aggregate_function" (one of "sum", "max", "min", "avg", "mode" or "count"),
+        // and "aggregated_column" (the name of a column of the original layer query or "*")
+        // A column defined as `"_cdb_features_count": {"aggregate_function": "count", aggregated_column: "*"}`
+        // is always generated in addition to the defined columns.
+        // The column names `cartodb_id`, `the_geom`, `the_geom_webmercator` and `_cdb_feature_count` cannot be used
+        // for aggregated columns, as they correspond to columns always present in the result.
+        // See https://github.com/CartoDB/Windshaft-cartodb/blob/master/docs/aggregation.md#columns for more details
+        columns: {
+            "aggregated_column_1": {
+                "aggregate_function": "sum",
+                "aggregated_column": "original_column_1"
+            }
+        },
+
+        // OPTIONAL
+        // Number, defines the cell-size of the spatial aggregation grid as a pixel resolution power of two (1/4, 1/2,... 2, 4, 16)
+        // to scale from 256x256 pixels; the default is 1 corresponding to 256x256 cells per tile.
+        // See https://github.com/CartoDB/Windshaft-cartodb/blob/master/docs/aggregation.md#resolution for more details
+        resolution: 1,
+
+        // OPTIONAL
+        // Number, the minimum number of (estimated) rows in the dataset (query results) for aggregation to be applied.
+        // See https://github.com/CartoDB/Windshaft-cartodb/blob/master/docs/aggregation.md#threshold for more details
+        threshold: 500000
+    }
+}
+```
+
+# History
+
+## 1.0.0
+
+ - Initial version

docs/MapConfig-Analyses-extension.md

@@ -0,0 +1,93 @@
+# 1. Purpose
+
+This specification describes an extension for
+[MapConfig 1.4.0](https://github.com/CartoDB/Windshaft/blob/master/doc/MapConfig-1.4.0.md) version.
+
+
+# 2. Changes over specification
+
+This extension targets layers with `sql` option, including layer types: `cartodb`, `mapnik`, and `torque`.
+
+It extends MapConfig with a new attribute: `analyses`.
+
+## 2.1 Analyses attribute
+
+The new analyses attribute must be an array of analyses as per [camshaft](https://github.com/CartoDB/camshaft). Each
+analysis must adhere to the [camshaft-reference](https://github.com/CartoDB/camshaft/blob/0.8.0/reference/versions/0.7.0/reference.json) specification.
+
+Each node can have an id that can be later references to consume the query from MapConfig's layers.
+
+Basic analyses example:
+
+```javascript
+[
+    {
+        // REQUIRED
+        // string, `id` free identifier that can be reference from any layer
+        "id": "HEAD",
+        // REQUIRED
+        // string, `type` camshaft's analysis type
+        "type": "source",
+        // REQUIRED
+        // object, `params` will depend on `type`, check camshaft-reference for more information
+        "params": {
+            "query": "select * from your_table"
+        }
+    }
+]
+```
+
+# 2.2. Integration with layers
+
+As pointed before an analysis node id can be referenced from layers to consume its output query.
+
+The layer consuming the output must reference it with the following option:
+
+```
+{
+    "options": {
+        // REQUIRED
+        // object, `source` as in the future we might want to have other source options
+        "source": {
+            // REQUIRED
+            // string, `id` the analysis node identifier
+            "id": "HEAD"
+        }
+    }
+}
+```
+
+## 2.3. Complete example
+
+```
+{
+    "version": "1.4.0",
+    "layers": [
+        {
+            "type": "cartodb",
+            "options": {
+                "source": {
+                    "id": "HEAD"
+                },
+                "cartocss": "...",
+                "cartocss_version": "2.3.0"
+            }
+        }
+    ],
+    "analyses": [
+        {
+            "id": "HEAD",
+            "type": "source",
+            "params": {
+                "query": "select * from your_table"
+            }
+        }
+    ]
+}
+```
+
+# History
+
+## 1.0.0
+
+ - Initial version

docs/MapConfig-Dataviews-extension.md

@@ -0,0 +1,277 @@
+# 1. Purpose
+
+This specification describes an extension for
+[MapConfig 1.4.0](https://github.com/CartoDB/Windshaft/blob/master/doc/MapConfig-1.4.0.md) version.
+
+
+# 2. Changes over specification
+
+This extension depends on Analyses extension. It extends MapConfig with a new attribute: `dataviews`.
+
+It makes possible to get tabular data from analysis nodes: aggregated lists, aggregations, and histograms.
+
+## 2.1. Dataview types
+
+### Aggregation
+
+An aggregation is a list with aggregated results by a column and a given aggregation function.
+
+Definition
+```
+{
+    // REQUIRED
+    // string, `type` the aggregation type
+    “type”: “aggregation”,
+    // REQUIRED
+    // object, `options` dataview params
+    “options”: {
+        // REQUIRED
+        // string, `column` column name to aggregate by
+        “column”: “country”,
+        // REQUIRED
+        // string, `aggregation` operation to perform
+        “aggregation”: “count”
+        // OPTIONAL
+        // string, `aggregationColumn` column value to aggregate
+        // This param is required when `aggregation` is different than "count"
+        “aggregationColumn”: “population”
+    }
+}
+```
+
+Expected output
+```
+{
+  "type": "aggregation",
+  "categories": [
+    {
+      "category": "foo",
+      "value": 100
+    },
+    {
+      "category": "bar",
+      "value": 200
+    }
+  ]
+}
+```
+
+### Histograms
+
+Histograms represent the data distribution for a column.
+
+Definition
+```
+{
+    // REQUIRED
+    // string, `type` the histogram type
+    “type”: “histogram”,
+    // REQUIRED
+    // object, `options` dataview params
+    “options”: {
+        // REQUIRED
+        // string, `column` column name to aggregate by
+        “column”: “name”,
+        // OPTIONAL
+        // number, `bins` how many buckets the histogram should use
+        “bins”: 10
+    }
+}
+```
+
+Expected output
+```
+{
+  "type": "histogram",
+  "bins": [{"bin": 0, "start": 2, "end": 2, "min": 2, "max": 2, "freq": 1}, null, null, {"bin": 3, "min": 40, "max": 44, "freq": 2}, null],
+  "width": 10
+}
+```
+
+### Formula
+
+Formulas given a final value representing the whole dataset.
+
+Definition
+```
+{
+    // REQUIRED
+    // string, `type` the formula type
+    “type”: “formula”,
+    // REQUIRED
+    // object, `options` dataview params
+    “options”: {
+        // REQUIRED
+        // string, `column` column name to aggregate by
+        “column”: “name”,
+        // REQUIRED
+        // string, `aggregation` operation to perform
+        “operation”: “count”
+    }
+}
+```
+
+Operation must be: “min”, “max”, “count”, “avg”, or “sum”.
+
+Result
+```
+{
+  "type": "formula",
+  "operation": "count",
+  "result": 1000,
+  "nulls": 0
+}
+```
+
+
+## 2.2 Dataviews attribute
+
+The new dataviews attribute must be a dictionary of dataviews.
+
+An analysis node id can be referenced from dataviews to consume its output query.
+
+
+The layer consuming the output must reference it with the following option:
+
+```
+{
+    // REQUIRED
+    // object, `source` as in the future we might want to have other source options
+    "source": {
+        // REQUIRED
+        // string, `id` the analysis node identifier
+        "id": "HEAD"
+    }
+}
+```
+
+## 2.3. Complete example
+
+```
+{
+    "version": "1.4.0",
+    "layers": [
+        {
+            "type": "cartodb",
+            "options": {
+                "source": {
+                    "id": "HEAD"
+                },
+                "cartocss": "...",
+                "cartocss_version": "2.3.0"
+            }
+        }
+    ],
+    "dataviews" {
+        "basic_histogram": {
+            "source": {
+                "id": "HEAD"
+            },
+            "type": "histogram",
+            "options": {
+                "column": "pop_max"
+            }
+        }
+    },
+    "analyses": [
+        {
+            "id": "HEAD",
+            "type": "source",
+            "params": {
+                "query": "select * from your_table"
+            }
+        }
+    ]
+}
+```
+
+## 3. Filters
+
+Camshaft's analyses expose a filtering capability and `aggregation` and `histogram` dataviews get them for free with
+ this extension. Filters are available with the very dataview id, so if you have a "basic_histogram" histogram dataview
+ you can filter with a range filter with "basic_histogram" name.
+
+
+## 3.1 Filter types
+
+### Category
+
+Allows to remove results that are not contained within a set of elements.
+Initially this filter can be applied to a `numeric` or `text` columns.
+
+Params
+
+```
+{
+    “accept”: [“Spain”, “Germany”]
+    “reject”: [“Japan”]
+}
+```
+
+### Range filter
+
+Allows to remove results that don’t satisfy numeric min and max values.
+Filter is applied to a numeric column.
+
+Params
+
+```
+{
+    “min”: 0,
+    “max”: 1000
+}
+```
+
+## 3.2. How to apply filters
+
+Filters must be applied at map instantiation time.
+
+With :mapconfig as a valid MapConfig and with :filters (a valid JSON) as:
+
+### Anonymous map
+
+`GET /api/v1/map?config=:mapconfig&filters=:filters`
+
+`POST /api/v1/map?filters=:filters`
+with `BODY=:mapconfig`
+
+If in the future we need to support a bigger filters param and it doesn’t fit in the query string,
+ we might solve it by accepting:
+
+`POST /api/v1/map`
+with `BODY={“config”: :mapconfig, “filters”: :filters}`
+
+### Named map
+
+Assume :params (a valid JSON) as named maps params, like in: `{“color”: “red”}`
+
+`GET /api/v1/named/:name/jsonp?config=:params&filters=:filters&callback=cb`
+
+`POST /api/v1/named/:name?filters=:filters`
+with `BODY=:params`
+
+If, again, in the future we need to support a bigger filters param that doesn’t fit in the query string,
+ we might solve it by accepting:
+
+`POST /api/v1/named/:name`
+with `BODY={“config”: :params, “filters”: :filters}`
+
+
+## 3.3 Bounding box special filter
+
+A bounding box filter allows to remove results that don’t satisfy a geospatial range.
+
+The bounding box special filter is available per dataview and there is no need to create a bounding box definition as
+it’s always possible to apply a bbox filter per dataview.
+
+A dataview can get its result filtered by bounding box by sending a bbox param in the query string,
+param must be in the form `west,south,east,north`.
+
+So applying a bbox filter to a dataview looks like:
+GET /api/v1/map/:layergroupid/dataview/:dataview_name?bbox=-90,-45,90,45
+
+# History
+
+## 1.0.0-alpha
+
+ - WIP document

docs/aggregation.md

@@ -0,0 +1,187 @@
+# Tile Aggregation
+
+To be able to represent a large amount of data (say, hundred of thousands to millions of points) in a tile. This can be useful both for raster tiles (where the aggregation reduces the number of features to be rendered) and vector tiles (the tile contais less features).
+
+Aggregation is available only for point geometries. During aggregation the points are grouped using a grid; all the points laying in the same cell of the grid are summarized in a single aggregated result point.
+ - The position of the aggregated point is controlled by the `placement` parameter.
+ - The aggregated rows always contain at least a column, named `_cdb_feature_count`, which contains the number of the original points that the aggregated point represents.
+
+### Special default aggregation
+
+When no placement or columns are specified a special default aggregation is performed.
+
+This special mode performs only spatial aggregation (using a grid defined by the requested tile and the resolution, parameter, as all the other cases), and returns a _random_ record from each group (grid cell) with all its columns and an additional `_cdb_features_count` with the number of features in the group.
+
+Regarding the randomness of the sample: currently we use the row with the minimum `cartodb_id` value in each group.
+
+The rationale behind having this special aggregation with all the original columns is to provide a mostly transparent way to handle large datasets without having to provide special map configurations for those cases (i.e. preserving the logic used to produce the maps with smaller datasets). [Overviews have been used so far with this intent](https://carto.com/docs/tips-and-tricks/back-end-data-performance/), but they are inflexible.
+
+### User defined aggregations
+
+When either a explicit placement or columns are requested we no longer use the special, query; we use one determined by the placement (which will default to "centroid"), and it will have as columns only the aggregated columns specified, in addition to `_cdb_features_count`, which is always present.
+
+We might decide in the future to allow sampling column values for any of the different placement modes.
+
+### Behaviour for raster and vector tiles
+
+The vector tiles from a vector-only map will be aggregated by default.
+However, Raster tiles (or vector tiles from a map which defines CartoCSS styles) will be aggregated only upon request.
+
+Aggregation that would otherwise occur can be disabled by passing an `aggregation=false` parameter to the map instantiation HTTP call.
+
+To control how aggregation is performed, an aggregation option can be added to the layer:
+
+```json
+{
+    "layers": [
+        {
+            "options": {
+                "sql": "SELECT * FROM data",
+                "aggregation": {
+                    "placement": "centroid",
+                    "columns": {
+                        "value": {
+                            "aggregate_function": "sum",
+                            "aggregated_column": "value"
+                        }
+                    }
+                }
+            }
+        }
+    ]
+}
+```
+
+Even if aggregation is explicitly requested it may not be activated, e.g., if the geometries are not points
+or the whole dataset is too small. The map instantiation response contains metadata that informs if any particular
+layer will be aggregated when tiles are requested, both for vector (mvt) and raster (png) tiles.
+
+```json
+{
+  "layergroupid": "7b97b6e76590fef889b63edd2efb1c79:1513608333045",
+  "metadata": {
+    "layers": [
+      {
+        "type": "mapnik",
+        "id": "layer0",
+        "meta": {
+          "stats": {
+            "estimatedFeatureCount": 6232136
+          },
+          "aggregation": {
+            "png": true,
+            "mvt": true
+          }
+        }
+      }
+    ]
+  }
+}
+```
+
+## Aggregation parameters
+
+The aggregation parameters for a layer are defined inside an `aggregation` option of the layer:
+
+```json
+{
+    "layers": [
+        {
+            "options": {
+                "sql": "SELECT * FROM data",
+                "aggregation": {"...": "..."}
+            }
+        }
+    ]
+}
+```
+
+### `placement`
+
+Determines the kind of aggregated geometry generated:
+
+#### `point-sample`
+
+This is the default placement. It will place the aggregated point at a random sample of the grouped points,
+like the default aggregation does. No other attribute is sampled, though, the point will contain the aggregated attributes determined by the `columns` parameter.
+
+#### `point-grid`
+
+Generates points at the center of the aggregation grid cells (squares).
+
+#### `centroid`
+
+Generates points with the averaged coordinated of the grouped points (i.e. the points inside each grid cell).
+
+### `columns`
+
+The aggregated attributes defined by `columns` are computed by a applying an _aggregate function_ to all the points in each group.
+Valid aggregate functions are `sum`, `avg` (average), `min` (minimum), `max` (maximum) and `mode` (the most frequent value in the group).
+The values to be aggregated are defined by the _aggregated column_ of the source data. The column keys define the name of the resulting column in the aggregated dataset.
+
+For example here we define three aggregate attributes named `total`, `max_price` and `price` which are all computed with the same column, `price`,
+of the original dataset applying three different aggregate functions.
+
+```json
+{
+    "columns": {
+        "total": { "aggregate_function": "sum", "aggregated_column": "price" },
+        "max_price": { "aggregate_function": "max", "aggregated_column": "price" },
+        "price": { "aggregate_function": "avg", "aggregated_column": "price" }
+    }
+}
+```
+
+> Note that you can use the original column names as names of the result, but all the result column names must be unique.  In particular, the names `cartodb_id`, `the_geom`, `the_geom_webmercator` and `_cdb_feature_count` cannot be used for aggregated columns, as they correspond to columns always present in the result.
+
+### `resolution`
+
+Defines the cell-size of the spatial aggregation grid. This is equivalent to the [CartoCSS `-torque-resolution`](https://carto.com/docs/carto-engine/cartocss/properties-for-torque/#-torque-resolution-float) property of Torque maps.
+
+The aggregation cells are `resolution`×`resolution` pixels in size, where pixels here are defined to be 1/256 of the (linear) size of a tile.
+The default value is 1, so that aggregation coincides with raster pixels. A value of 2 would make each cell to be 4 (2×2) pixels, and a value of
+0.5 would yield 4 cells per pixel. In teneral values less than 1 produce sub-pixel precision.
+
+> Note that is independent of the number of pixels for raster tile or the coordinate resolution (mvt_extent) of vector tiles.
+
+
+### `threshold`
+
+This is the minimum number of (estimated) rows in the dataset (query results) for aggregation to be applied. If the number of rows estimate is less than the threshold aggregation will be disabled for the layer; the instantiation response will reflect that and tiles will be generated without aggregation.
+
+### Example
+
+```json
+{
+    "version": "1.7.0",
+    "extent": [-20037508.5, -20037508.5, 20037508.5, 20037508.5],
+    "srid": 3857,
+    "maxzoom": 18,
+    "minzoom": 3,
+    "layers": [
+        {
+            "type": "mapnik",
+            "options": {
+                "sql": "select * from table",
+                "cartocss": "#table { marker-width: [total]; marker-fill: ramp(value, (red, green, blue), jenks); }",
+                "cartocss_version": "2.3.0",
+                "aggregation": {
+                    "placement": "centroid",
+                    "columns": {
+                        "value": {
+                            "aggregate_function": "avg",
+                            "aggregated_column": "value"
+                        },
+                        "total": {
+                            "aggregate_function": "sum",
+                            "aggregated_column": "value"
+                        }
+                    },
+                    "resolution": 2,
+                    "threshold": 500000
+                }
+            }
+        }
+    ]
+}
+```

docs/anonymous_maps.md

@@ -1,6 +1,7 @@
 # Anonymous Maps
 
-Anonymous Maps allows you to instantiate a map given SQL and CartoCSS. It also allows you to add interaction capabilities using [UTF Grid.](https://github.com/mapbox/utfgrid-spec)
+Anonymous Maps allows you to instantiate a map given SQL and CartoCSS. It also allows you to add interaction capabilities using [UTF Grid.](https://github.com/mapbox/utfgrid-spec).
+Alternatively, you can get the data for the map (geometry and attributes for each layer) using vector tiles (in which case CartoCSS is not required).
 
 
 ## Instantiate
@@ -28,7 +29,7 @@ POST /api/v1/map
 }
 ```
 
-See [MapConfig File Formats](http://docs.cartodb.com/cartodb-platform/maps-api/mapconfig/) for details.
+See [MapConfig File Formats](http://docs.carto.com/carto-engine/maps-api/mapconfig/) for details.
 
 #### Response
 
@@ -36,17 +37,24 @@ The response includes:
 
 Attributes | Description
 --- | ---
-layergroupid | The ID for that map, used to compose the URL for the tiles. The final URL is: `https://{username}.cartodb.com/api/v1/map/{layergroupid}/{z}/{x}/{y}.png`
+layergroupid | The ID for that map, used to compose the URL for the tiles. The final URL is: `https://{username}.carto.com/api/v1/map/{layergroupid}/{z}/{x}/{y}.png`
 updated_at | The ISO date of the last time the data involved in the query was updated.
 metadata | Includes information about the layers.
 cdn_url | URLs to fetch the data using the best CDN for your zone.
 
+**Improved response metadata**
+
+Originally, you needed to concantenate the `layergroupid` with the correct domain and the path for the tiles.
+Now, for convenience, the layergroup includes the final URLs in two formats:
+1. Leaflet's urlTemplate alike: useful when working with raster tiles or with libraries with an API similar to Leaflet's one.
+1. [TileJSON spec](https://github.com/mapbox/tilejson-spec): useful when working with Mapbox GL or any other library that supports TileJSON.
+
 ### Example
 
 #### Call
 
 ```bash
-curl 'https://{username}.cartodb.com/api/v1/map' -H 'Content-Type: application/json' -d @mapconfig.json
+curl 'https://{username}.carto.com/api/v1/map' -H 'Content-Type: application/json' -d @mapconfig.json
 ```
 
 #### Response
@@ -61,35 +69,236 @@ curl 'https://{username}.cartodb.com/api/v1/map' -H 'Content-Type: application/j
         "type": "mapnik",
         "meta": {}
       }
-    ]
+    ],
+    "tilejson": {
+        "raster": {
+            "tilejson": "2.2.0",
+            "tiles": [
+                "http://a.cdb.com/c01a54877c62831bb51720263f91fb33/{z}/{x}/{y}.png",
+                "http://b.cdb.com/c01a54877c62831bb51720263f91fb33/{z}/{x}/{y}.png"
+            ]
+        }
+    },
+    "url": {
+        "raster": {
+            "urlTemplate": "http://{s}.cdb.com/c01a54877c62831bb51720263f91fb33/{z}/{x}/{y}.png",
+            "subdomains": ["a", "b"]
+        }
+    }
   },
   "cdn_url": {
     "http": "http://cdb.com",
-    "https": "https://cdb.com"
+    "https": "https://cdb.com",
+    "templates": {
+         "http": { "subdomains": ["a","b"], "url": "http://{s}.cdb.com" },
+         "https": { "subdomains": ["a","b"], "url": "https://{s}.example.com" },
+    }
   }
 }
 ```
 
-### Retrieve resources from the layergroup
+## Map Tile Rendering
+
+Map tiles are used to create the graphic representation of your map in a web browser. Tiles can be requested either as pre-rendered *raster* tiles (images) or as *vector* map data to be rendered by the client (browser).
+
+- **Raster**: If a tile is requested as a raster image format, like PNG, the map will be rendered on the server, using the CartoCSS styles defined in the layers of the map. It is necessary that all the layers of a map define CartoCSS styles in order to obtain raster tiles. Raster tiles are made up of 256x256 pixels; to avoid graphic quality issues tiles should be used unscaled to represent the zoom level (Z) for which they are requested. In order to render tiles, data will be retrieved from the database (in vector format) on the server-side.
+
+- **Vector**: Tiles can also be requested as MVT (Mapbox Vector Tiles). In this case, only the geospatial vector data, without any styling, is returned. These tiles should be processed in the client-side to render the map.  In this case layers do not need to define CartoCSS, as any rendering and styling will be performed on the client side. The vector data of a tile represents real-world geometries by defining the vertices of points, lines or polygons in a tile-specific coordinate system.
+
+## Retrieve resources from the layergroup
 
 When you have a layergroup, there are several resources for retrieving layergoup details such as, accessing Mapnik tiles, getting individual layers, accessing defined Attributes, and blending and layer selection.
 
-#### Mapnik tiles
+### Raster tiles
 
-These tiles will get just the Mapnik layers. To get individual layers, see the following section.
+These raster tiles are PNG images that represent only the Mapnik layers of a map. See [individual layers](#individual-layers) for details about how to retrieve other layers.
 
 ```bash
-https://{username}.cartodb.com/api/v1/map/{layergroupid}/{z}/{x}/{y}.png
+https://{username}.carto.com/api/v1/map/{layergroupid}/{z}/{x}/{y}.png
 ```
 
-#### Individual layers
+### Mapbox Vector Tiles (MVT)
 
-The MapConfig specification holds the layers definition in a 0-based index. Layers can be requested individually in different formats depending on the layer type.
+[Mapbox Vector Tiles (MVT)](https://www.mapbox.com/vector-tiles/specification/) are map tiles that transfer geographic vector data to the client-side. Browser performance is fast since you can pan and zoom without having to query the server.
+
+CARTO uses Web Graphics Library (WebGL) to process MVT files on the browser. This is useful since WebGL is compatible with most web browsers, include support for multiple client-side mapping engines, and do not require additional information from the server; which makes it more efficient for rendering map tiles. However, you can use any implementation tool for processing MVT files.
+
+The following examples describe how to fetch MVT tiles with a cURL request.
+
+#### MVT and Windshaft
+
+CARTO uses Windshaft as the map tiler library to render multilayer maps with the Maps API. You can use Windshaft to request MVT using the same layer type that is used for requesting raster tiles (Mapnik layer). Simply change the file format `.mvt` in the URL.
+
+
+```bash
+https://{username}.cartodb.com/api/v1/map/HASH/:layer/{z}/{x}/{y}.mvt
+```
+
+The following example instantiates an anonymous map with layer options:
+
+```bash
+{
+  user_name: 'mycartodbuser',
+  sublayers: [{
+    sql: "SELECT * FROM table_name";
+    cartocss: '#layer { marker-fill: #F0F0F0; }'
+  }],
+  maps_api_template: 'https://{user}.cartodb.com' // Optional
+}
+```
+
+**Note**: If no layer type is specified, Mapnik tiles are used by default. To access MVT tiles, specify `https://{username}.cartodb.com/api/v1/map/HASH/{z}/{x}/{y}.mvt` as the `maps_api_template` variable.
+
+**Tip:** If you are using [Named Maps](https://carto.com/docs/carto-engine/maps-api/named-maps/) to instantiate a layer, indicate the MVT file format and layer in the response:
+
+```bash
+https://{username}.cartodb.com/api/v1/map/named/:templateId/:layer/{z}/{x}/{y}.mvt
+```
+
+For all layers in a Named Map, you must indicate Mapnik as the layer filter:
+
+```bash
+https://{username}.cartodb.com/api/v1/map/named/:templateId/mapnik/{z}/{x}/{y}.mvt
+```
+
+#### Layergroup Filter for MVT Tiles
+
+To filter layers using Windshaft, use the following request where layers are numbered:
+
+```bash
+https://{username}.cartodb.com/api/v1/map/HASH/0,1,2/{z}/{x}/{y}.mvt
+```
+
+To request all layers, remove the layergroup filter parameter:
+
+```bash
+https://{username}.cartodb.com/api/v1/map/HASH/{z}/{x}/{y}.mvt
+```
+
+To filter a specific layer:
+
+```bash
+https://{username}.cartodb.com/api/v1/map/HASH/2/{z}/{x}/{y}.mvt
+```
+
+#### Example 1: MVT Tiles with Windshaft, CARTO.js, and MapboxGL
+
+1) Import the required libraries:
+
+```bash
+<script src='https://api.tiles.mapbox.com/mapbox-gl-js/v0.9.0/mapbox-gl.js'></script>
+<link href='https://api.tiles.mapbox.com/mapbox-gl-js/v0.9.0/mapbox-gl.css' rel='stylesheet' />
+<script src="http://libs.cartocdn.com/cartodb.js/v3/3.15/cartodb.core.js"></script>
+```
+
+2) Configure Map Client:
+
+```bash
+mapboxgl.accessToken = '{yourMapboxToken}';
+```
+
+3) Create Map Object (Mapbox):
+
+```bash
+var map = new mapboxgl.Map({
+container: 'map',
+zoom: 1,
+minZoom: 0,
+maxZoom: 18,
+center: [30, 0]
+});
+```
+
+4) Define Layer Options (CARTO):
+
+```bash
+var layerOptions = {
+user_name: "{username}",
+sublayers: [{
+sql: "SELECT * FROM {table_name}",
+cartocss: "...",
+     }]
+};
+```
+
+5) Request Tiles (from CARTO) and Set to Map Object (Mapbox):
+
+**Note:** By default, [CARTO core functions](https://carto.com/docs/carto-engine/carto-js/core-api/) retrieve URLs for fully rendered tiles. You must replace the default format (.png) with the MVT format (.mvt).
+
+
+```bash
+cartodb.Tiles.getTiles(layerOptions, function(result, err) {
+var tiles = result.tiles.map(function(tileUrl) {
+return tileUrl
+.replace('{s}', 'a')
+.replace(/\.png/, '.mvt');
+});
+map.setStyle(simpleStyle(tiles));
+});
+```
+
+#### Example 2: MVT Libraries with Windshaft and MapboxGL
+
+When you are not including CARTO.js to implement MVT tiles, you must use the `map.setStyle` parameter to specify vector map rendering.
+
+1) Import the required libraries:
+
+```bash
+<script src='https://api.tiles.mapbox.com/mapbox-gl-js/v0.9.0/mapbox-gl.js'></script>
+<link href='https://api.tiles.mapbox.com/mapbox-gl-js/v0.9.0/mapbox-gl.css' rel='stylesheet'/>
+```
+
+2) Configure Map Client:
+
+```bash
+mapboxgl.accessToken = '{yourMapboxToken}';
+```
+
+3) Create Map Object (Mapbox):
+
+```bash
+var map = new mapboxgl.Map({
+container: 'map',
+zoom: 1,
+minZoom: 0,
+maxZoom: 18,
+center: [30, 0]
+});
+```
+
+4) Set the Style
+
+```bash
+map.setStyle({
+    "version": 7,
+    "glyphs": "...",
+    "constants": {...},
+    "sources": {
+      "cartodb": {
+        "type": "vector",
+        "tiles": [ "http://{username}.cartodb.com/api/v1/map/named/templateId/mapnik/{z}/{x}/{y}.mvt"
+        ],
+        "maxzoom": 18
+      }
+    },
+    "layers": [{...}]
+});
+```
+
+**Tip:** If you are using MapboxGL, see the following resource for additional information.
+
+- [MapboxGL API Reference](https://www.mapbox.com/mapbox-gl-js/api/)
+- [MapboxGL Style Specifications](https://www.mapbox.com/mapbox-gl-js/style-spec/)
+- [Example of MapboxGL Implementation](https://www.mapbox.com/mapbox-gl-js/examples/)
+
+### Individual layers
+
+The MapConfig specification holds the layers definition in a 0-based index. Layers can be requested individually, in different formats, depending on the layer type.
 
 Individual layers can be accessed using that 0-based index. For UTF grid tiles:
 
 ```bash
-https://{username}.cartodb.com/api/v1/map/{layergroupid}/{layer}/{z}/{x}/{y}.grid.json
+https://{username}.carto.com/api/v1/map/{layergroupid}/{layer}/{z}/{x}/{y}.grid.json
 ```
 
 In this case, `layer` as 0 returns the UTF grid tiles/attributes for layer 0, the only layer in the example MapConfig.
@@ -97,25 +306,25 @@ In this case, `layer` as 0 returns the UTF grid tiles/attributes for layer 0, th
 If the MapConfig had a Torque layer at index 1 it could be possible to request it with:
 
 ```bash
-https://{username}.cartodb.com/api/v1/map/{layergroupid}/1/{z}/{x}/{y}.torque.json
+https://{username}.carto.com/api/v1/map/{layergroupid}/1/{z}/{x}/{y}.torque.json
 ```
 
-#### Attributes defined in `attributes` section
+### Attributes defined in `attributes` section
 
 ```bash
-https://{username}.cartodb.com/api/v1/map/{layergroupid}/{layer}/attributes/{feature_id}
+https://{username}.carto.com/api/v1/map/{layergroupid}/{layer}/attributes/{feature_id}
 ```
 
-Which returns JSON with the attributes defined, like:
+Which returns JSON with the attributes defined, such as:
 
 ```javascript
 { "c": 1, "d": 2 }
 ```
 
-#### Blending and layer selection
+### Blending and layer selection
 
 ```bash
-https://{username}.cartodb.com/api/v1/map/{layergroupid}/{layer_filter}/{z}/{x}/{y}.png
+https://{username}.carto.com/api/v1/map/{layergroupid}/{layer_filter}/{z}/{x}/{y}.png
 ```
 
 Note: currently format is limited to `png`.
@@ -127,7 +336,7 @@ Note: currently format is limited to `png`.
 Using `all` as `layer_filter` will blend all layers in the layergroup
 
 ```bash
-https://{username}.cartodb.com/api/v1/map/{layergroupid}/all/{z}/{x}/{y}.png
+https://{username}.carto.com/api/v1/map/{layergroupid}/all/{z}/{x}/{y}.png
 ```
 
 - Filter by layer index
@@ -135,19 +344,13 @@ https://{username}.cartodb.com/api/v1/map/{layergroupid}/all/{z}/{x}/{y}.png
 A list of comma separated layer indexes can be used to just render a subset of layers. For example `0,3,4` will filter and blend layers with indexes 0, 3, and 4.
 
 ```bash
-https://{username}.cartodb.com/api/v1/map/{layergroupid}/0,3,4/{z}/{x}/{y}.png
+https://{username}.carto.com/api/v1/map/{layergroupid}/0,3,4/{z}/{x}/{y}.png
 ```
 
 Some notes about filtering:
 
   - Invalid index values or out of bounds indexes will end in `Invalid layer filtering` errors.
-  - Once a Mapnik layer is selected, all Mapnik layers will get blended. As this may change in the future **it is
-  recommended** to always select all Mapnik layers if you want to select at least one so you will get a consistent
-  behavior in the future.
-  - Ordering is not considered. So right now filtering layers 0,3,4 is the very same thing as filtering 3,4,0. As this
-  may change in the future **it is recommended** to always select the layers in ascending order so you will get a
-  consistent behavior in the future.
-
+  - Ordering is not considered. So right now filtering layers 0,3,4 is the very same thing as filtering 3,4,0. As this may change in the future, **it is recommended** to always select the layers in ascending order so that you will always get consistent behavior.
 
 ## Create JSONP
 
@@ -172,7 +375,7 @@ callback | JSON callback name.
 #### Call
 
 ```bash
-curl "https://{username}.cartodb.com/api/v1/map?callback=callback&config=%7B%22version%22%3A%221.0.1%22%2C%22layers%22%3A%5B%7B%22type%22%3A%22cartodb%22%2C%22options%22%3A%7B%22sql%22%3A%22select+%2A+from+european_countries_e%22%2C%22cartocss%22%3A%22%23european_countries_e%7B+polygon-fill%3A+%23FF6600%3B+%7D%22%2C%22cartocss_version%22%3A%222.3.0%22%2C%22interactivity%22%3A%5B%22cartodb_id%22%5D%7D%7D%5D%7D"
+curl "https://{username}.carto.com/api/v1/map?callback=callback&config=%7B%22version%22%3A%221.0.1%22%2C%22layers%22%3A%5B%7B%22type%22%3A%22cartodb%22%2C%22options%22%3A%7B%22sql%22%3A%22select+%2A+from+european_countries_e%22%2C%22cartocss%22%3A%22%23european_countries_e%7B+polygon-fill%3A+%23FF6600%3B+%7D%22%2C%22cartocss_version%22%3A%222.3.0%22%2C%22interactivity%22%3A%5B%22cartodb_id%22%5D%7D%7D%5D%7D"
 ```
 
 #### Response
@@ -188,7 +391,6 @@ callback({
 })
 ```
 
-
 ## Remove
 
 Anonymous Maps cannot be removed by an API call. They will expire after about five minutes, or sometimes longer. If an Anonymous Map expires and tiles are requested from it, an error will be raised. This could happen if a user leaves a map open and after time, returns to the map and attempts to interact with it in a way that requires new tiles (e.g. zoom). The client will need to go through the steps of creating the map again to fix the problem.

docs/general_concepts.md

@@ -4,7 +4,7 @@ The following concepts are the same for every endpoint in the API except when it
 
 ## Auth
 
-By default, users do not have access to private tables in CartoDB. In order to instantiate a map from private table data an API Key is required. Additionally, to include some endpoints, an API Key must be included (e.g. creating a Named Map).
+By default, users do not have access to private tables in CARTO. In order to instantiate a map from private table data an API Key is required. Additionally, to include some endpoints, an API Key must be included (e.g. creating a Named Map).
 
 To execute an authorized request, `api_key=YOURAPIKEY` should be added to the request URL. The param can be also passed as POST param. Using HTTPS is mandatory when you are performing requests that include your `api_key`.
 
@@ -22,6 +22,6 @@ Errors are reported using standard HTTP codes and extended information encoded i
 
 If you use JSONP, the 200 HTTP code is always returned so the JavaScript client can receive errors from the JSON object.
 
-## CORS support
+## CORS Support
 
 All the endpoints, which might be accessed using a web browser, add CORS headers and allow OPTIONS method.

docs/named_maps.md

@@ -1,12 +1,12 @@
 # Named Maps
 
-Named Maps are essentially the same as Anonymous Maps except the MapConfig is stored on the server, and the map is given a unique name. You can create Named Maps from private data, and users without an API Key can view your Named Map (while keeping your data private). 
+Named Maps are essentially the same as Anonymous Maps except the MapConfig is stored on the server, and the map is given a unique name. You can create Named Maps from private data, and users without an API Key can view your Named Map (while keeping your data private).
 
-The Named Map workflow consists of uploading a MapConfig file to CartoDB servers, to select data from your CartoDB user database by using SQL, and specifying the CartoCSS for your map. 
+The Named Map workflow consists of uploading a MapConfig file to CARTO servers, to select data from your CARTO user database by using SQL, and specifying the CartoCSS for your map.
 
-The response back from the API provides the template_id of your Named Map as the `name` (the identifier of your Named Map), which is the name that you specified in the MapConfig. You can  which you can then use to create your Named Map details, or [fetch XYZ tiles](#fetching-xyz-tiles-for-named-maps) directly for Named Maps. 
+The response back from the API provides the template_id of your Named Map as the `name` (the identifier of your Named Map), which is the name that you specified in the MapConfig. You can  which you can then use to create your Named Map details, or [fetch XYZ tiles](#fetching-xyz-tiles-for-named-maps) directly for Named Maps.
 
-**Tip:** You can also use a Named Map that you created (which is defined by its `name`), to create a map using CartoDB.js. This is achieved by adding the [`namedmap` type](http://docs.cartodb.com/cartodb-platform/cartodb-js/layer-source-object/#named-maps-layer-source-object-type-namedmap) layer source object to draw the Named Map.
+**Tip:** You can also use a Named Map that you created (which is defined by its `name`), to create a map using CARTO.js. This is achieved by adding the [`namedmap` type](http://docs.carto.com/carto-engine/carto-js/layer-source-object/#named-maps-layer-source-object-type-namedmap) layer source object to draw the Named Map.
 
 The main differences, compared to Anonymous Maps, is that Named Maps include:
 
@@ -14,11 +14,11 @@ The main differences, compared to Anonymous Maps, is that Named Maps include:
   This allows you to control who is able to see the map based on an auth token, and create a secure Named Map with password-protection.
 
 - **template map**  
-  The template map is static and may contain placeholders, enabling you to modify your maps appearance by using variables. Templates maps are persistent with no preset expiration. They can only be created, or deleted, by a CartoDB user with a valid API KEY (See [auth argument](#arguments)).
+  The template map is static and may contain placeholders, enabling you to modify your maps appearance by using variables. Templates maps are persistent with no preset expiration. They can only be created, or deleted, by a CARTO user with a valid API KEY (See [auth argument](#arguments)).
 
-  Uploading a MapConfig creates a Named Map. MapConfigs are uploaded to the server by sending the server a "template".json file, which contain the [MapConfig specifications](http://docs.cartodb.com/cartodb-platform/maps-api/mapconfig/).
+  Uploading a MapConfig creates a Named Map. MapConfigs are uploaded to the server by sending the server a "template".json file, which contain the [MapConfig specifications](http://docs.carto.com/carto-engine/maps-api/mapconfig/).
 
-**Note:** There is a limit of 4,096 Named Maps allowed per account. If you need to create more Named Maps, it is recommended to use a single Named Map and change the variables using [placeholders](#placeholder-format), instead of uploading multiple [Named Map MapConfigs](http://docs.cartodb.com/cartodb-platform/maps-api/mapconfig/#named-map-layer-options).
+**Note:** There is a limit of 4,096 Named Maps allowed per account. If you need to create more Named Maps, it is recommended to use a single Named Map and change the variables using [placeholders](#placeholder-format), instead of uploading multiple [Named Map MapConfigs](http://docs.carto.com/carto-engine/maps-api/mapconfig/#named-map-layer-options).
 
 ## Create
 
@@ -33,7 +33,7 @@ POST /api/v1/map/named
 Params | Description
 --- | ---
 api_key | is required
-MapConfig | a [Named Map MapConfig](http://docs.cartodb.com/cartodb-platform/maps-api/mapconfig/#named-map-layer-options) is required to create a Named Map
+MapConfig | a [Named Map MapConfig](http://docs.carto.com/carto-engine/maps-api/mapconfig/#named-map-layer-options) is required to create a Named Map
 
 #### template.json
 
@@ -84,6 +84,10 @@ The `name` argument defines how to name this "template_name".json. Note that the
       "south": -45,
       "east": 45,
       "north": 45
+    },
+    "preview_layers": {
+      "0": true,
+      "layer1": false
     }
   }
 }
@@ -95,22 +99,22 @@ Params | Description
 --- | ---
 name | There can only be _one_ template with the same name for any user. Valid names start with a letter or a number, and only contain letters, numbers, dashes (-), or underscores (_). _This is specific to the name of your Named Map that is specified in the `name` property of the template file_.
 
-auth | 
+auth |
 --- | ---
 |_ method | `"token"` or `"open"` (`"open"` is the default if no method is specified. Use `"token"` to password-protect your map)
-|_ valid_tokens | when `"method"` is set to `"token"`, the values listed here allow you to instantiate the Named Map. See this [example](http://docs.cartodb.com/faqs/manipulating-your-data/#how-to-create-a-password-protected-named-map) for how to create a password-protected map.
+|_ valid_tokens | when `"method"` is set to `"token"`, the values listed here allow you to instantiate the Named Map. See this [example](http://docs.carto.com/faqs/manipulating-your-data/#how-to-create-a-password-protected-named-map) for how to create a password-protected map.
 placeholders | Placeholders are variables that can be placed in your template.json file's SQL or CartoCSS.
-layergroup | the layergroup configurations, as specified in the template. See [MapConfig File Format](http://docs.cartodb.com/cartodb-platform/maps-api/mapconfig/) for more information.
-view (optional) | extra keys to specify the view area for the map. It can be used to have a static preview of a Named Map without having to instantiate it. It is possible to specify it with `center` + `zoom` or with a bounding box `bbox`. Center+zoom takes precedence over bounding box.
+layergroup | the layergroup configurations, as specified in the template. See [MapConfig File Format](http://docs.carto.com/carto-engine/maps-api/mapconfig/) for more information.
+view (optional) | extra keys to specify the view area for the map. It can be used to have a static preview of a Named Map without having to instantiate it. It is possible to specify it with `center` + `zoom` or with a bounding box `bbox`. Center+zoom takes precedence over bounding box. Also it is possible to choose which layers are visible or not with `preview_layers` indicating its visibility by layer index or id (visible by default).
 --- | ---
 |_ zoom | The zoom level to use
 
-|_ center | 
+|_ center |
 --- | ---
 |_ |_ lng | The longitude to use for the center
 |_ |_ lat | The latitude to use for the center
 
-|_ bounds | 
+|_ bounds |
 --- | ---
 |_ |_ west | LowerCorner longitude for the bounding box, in decimal degrees (aka most western)
 |_ |_ south | LowerCorner latitude for the bounding box, in decimal degrees (aka most southern)
@@ -120,7 +124,7 @@ view (optional) | extra keys to specify the view area for the map. It can be use
 
 ### Placeholder Format
 
-Placeholders are variables that can be placed in your template.json file. Placeholders need to be defined with a `type` and a default value for MapConfigs. See details about defining a MapConfig `type` for [Layergoup configurations](http://docs.cartodb.com/cartodb-platform/maps-api/mapconfig/#layergroup-configurations).
+Placeholders are variables that can be placed in your template.json file. Placeholders need to be defined with a `type` and a default value for MapConfigs. See details about defining a MapConfig `type` for [Layergroup configurations](http://docs.carto.com/carto-engine/maps-api/mapconfig/#layergroup-configurations).
 
 Valid placeholder names start with a letter and can only contain letters, numbers, or underscores. They have to be written between the `<%=` and `%>` strings in order to be replaced inside the Named Maps API.
 
@@ -155,12 +159,12 @@ This is the call for creating the Named Map. It is sending the template.json fil
 curl -X POST \
    -H 'Content-Type: application/json' \
    -d @template.json \
-   'https://{username}.cartodb.com/api/v1/map/named?api_key={api_key}'
+   'https://{username}.carto.com/api/v1/map/named?api_key={api_key}'
 ```
 
 #### Response
 
-The response back from the API provides the name of your MapConfig as a template, enabling you to edit the Named Map details by inserting your variables into the template where placeholders are defined, and create custom queries using SQL. 
+The response back from the API provides the name of your MapConfig as a template, enabling you to edit the Named Map details by inserting your variables into the template where placeholders are defined, and create custom queries using SQL.
 
 ```javascript
 {
@@ -170,7 +174,7 @@ The response back from the API provides the name of your MapConfig as a template
 
 ## Instantiate
 
-Instantiating a Named Map allows you to fetch the map tiles. You can use the Maps API to instantiate, or use the CartoDB.js `createLayer()` function. The result is an Anonymous Map.
+Instantiating a Named Map allows you to fetch the map tiles. You can use the Maps API to instantiate, or use the CARTO.js `createLayer()` function. The result is an Anonymous Map.
 
 #### Definition
 
@@ -209,7 +213,7 @@ Valid auth token will be needed, if required by the template.
 curl -X POST \
   -H 'Content-Type: application/json' \
   -d @params.json \
-  'https://{username}.cartodb.com/api/v1/map/named/{template_name}?auth_token={auth_token}'
+  'https://{username}.carto.com/api/v1/map/named/{template_name}?auth_token={auth_token}'
 ```
 
 #### Response
@@ -229,7 +233,7 @@ curl -X POST \
 }
 ```
 
-You can then use the `layergroupid` for fetching tiles and grids as you would normally (see [Anonymous Maps](http://docs.cartodb.com/cartodb-platform/maps-api/anonymous-maps/)).
+You can then use the `layergroupid` for fetching tiles and grids as you would normally (see [Anonymous Maps](http://docs.carto.com/carto-engine/maps-api/anonymous-maps/)).
 
 ## Update
 
@@ -261,7 +265,7 @@ Updating a Named Map removes all the Named Map instances, so they need to be ini
 curl -X PUT \
   -H 'Content-Type: application/json' \
   -d @template.json \
-  'https://{username}.cartodb.com/api/v1/map/named/{template_name}?api_key={api_key}'
+  'https://{username}.carto.com/api/v1/map/named/{template_name}?api_key={api_key}'
 ```
 
 #### Response
@@ -303,7 +307,7 @@ api_key | is required
 #### Call
 
 ```bash
-curl -X DELETE 'https://{username}.cartodb.com/api/v1/map/named/{template_name}?api_key={api_key}'
+curl -X DELETE 'https://{username}.carto.com/api/v1/map/named/{template_name}?api_key={api_key}'
 ```
 
 #### Response
@@ -337,7 +341,7 @@ api_key | is required
 #### Call
 
 ```bash
-curl -X GET 'https://{username}.cartodb.com/api/v1/map/named?api_key={api_key}'
+curl -X GET 'https://{username}.carto.com/api/v1/map/named?api_key={api_key}'
 ```
 
 #### Response
@@ -377,7 +381,7 @@ api_key | is required
 #### Call
 
 ```bash
-curl -X GET 'https://{username}.cartodb.com/api/v1/map/named/{template_name}?api_key={api_key}'
+curl -X GET 'https://{username}.carto.com/api/v1/map/named/{template_name}?api_key={api_key}'
 ```
 
 #### Response
@@ -418,7 +422,7 @@ callback | JSON callback name
 #### Call
 
 ```bash
-curl 'https://{username}.cartodb.com/api/v1/map/named/{template_name}/jsonp?auth_token={auth_token}&callback=callback&config=template_params_json'
+curl 'https://{username}.carto.com/api/v1/map/named/{template_name}/jsonp?auth_token={auth_token}&callback=callback&config=template_params_json'
 ```
 
 #### Response
@@ -450,9 +454,9 @@ callback({
 })
 ```
 
-## CartoDB.js for Named Maps
+## CARTO.js for Named Maps
 
-You can use a Named Map that you created (which is defined by its `name`), to create a map using CartoDB.js. This is achieved by adding the [`namedmap` type](http://docs.cartodb.com/cartodb-platform/cartodb-js/layer-source-object/#named-maps-layer-source-object-type-namedmap) layer source object to draw the Named Map.
+You can use a Named Map that you created (which is defined by its `name`), to create a map using CARTO.js. This is achieved by adding the [`namedmap` type](http://docs.carto.com/carto-engine/carto-js/layer-source-object/#named-maps-layer-source-object-type-namedmap) layer source object to draw the Named Map.
 
 ```javascript
 {
@@ -482,17 +486,17 @@ You can use a Named Map that you created (which is defined by its `name`), to cr
 
 **Note:** Instantiating a Named Map over a `createLayer` does not require an API Key and by default, does not include auth tokens. _If_ you defined auth tokens for the Named Map configuration, then you will have to include them.
 
-[CartoDB.js](http://docs.cartodb.com/cartodb-platform/cartodb-js/) has methods for accessing your Named Maps.
+[CARTO.js](http://docs.carto.com/carto-engine/carto-js/) has methods for accessing your Named Maps.
 
-1. [layer.setParams()](http://docs.cartodb.com/cartodb-platform/cartodb-js/api-methods/#layersetparamskey-value) allows you to change the template variables (in the placeholders object) via JavaScript
+1. [layer.setParams()](http://docs.carto.com/carto-engine/carto-js/api-methods/#layersetparamskey-value) allows you to change the template variables (in the placeholders object) via JavaScript
 
-    **Note:** The CartoDB.js `layer.setParams()` function is not supported when using Named Maps for Torque. Alternatively, you can create a [Torque layer in a Named Map](http://bl.ocks.org/iriberri/de37be6406f9cc7cfe5a)
+    **Note:** The CARTO.js `layer.setParams()` function is not supported when using Named Maps for Torque. Alternatively, you can create a [Torque layer in a Named Map](http://bl.ocks.org/iriberri/de37be6406f9cc7cfe5a)
 
-2. [layer.setAuthToken()](http://docs.cartodb.com/cartodb-platform/cartodb-js/api-methods/#layersetauthtokenauthtoken) allows you to set the auth tokens to create the layer
+2. [layer.setAuthToken()](http://docs.carto.com/carto-engine/carto-js/api-methods/#layersetauthtokenauthtoken) allows you to set the auth tokens to create the layer
 
 ### Torque Layer in a Named Map
 
-If you are creating a Torque layer in a Named Map without using the Torque.js library, you can apply the Torque layer by applying the following code with CartoDBjs:
+If you are creating a Torque layer in a Named Map without using the Torque.js library, you can apply the Torque layer by applying the following code with CARTO.js:
 
 ```javascript
  // add cartodb layer with one sublayer
@@ -516,18 +520,18 @@ If you are creating a Torque layer in a Named Map without using the Torque.js li
     })
     .addTo(map)
     .done(function(layer) {
-  
+
     });
 }
 ```
 
-#### Examples of Named Maps created with CartoDB.js
+#### Examples of Named Maps created with CARTO.js
 
-- [Named Map selectors with interaction](http://bl.ocks.org/ohasselblad/515a8af1f99d5e690484)
+- [Named Map selectors with interaction](http://bl.ocks.org/andy-esch/515a8af1f99d5e690484)
 
-- [Named Map with interactivity](http://bl.ocks.org/ohasselblad/d1a45b8ff5e7bd90cd68)
+- [Named Map with interactivity](http://bl.ocks.org/andy-esch/d1a45b8ff5e7bd90cd68)
 
-- [Toggling sublayers in a Named Map](http://bl.ocks.org/ohasselblad/c1a0f4913610eec53cd3)
+- [Toggling sublayers in a Named Map](http://bl.ocks.org/andy-esch/c1a0f4913610eec53cd3)
 
 ## Fetching XYZ Tiles for Named Maps
 
@@ -535,7 +539,7 @@ Optionally, authenticated users can fetch projected tiles (XYZ tiles or Mapnik R
 
 ### Fetch XYZ Tiles Directly with a URL
 
-Authenticated users, with an auth token, can use XYZ-based URLs to fetch tiles directly, and instantiate the Named Map as part of the request to your application. You do not have to do any other steps to initialize your map. 
+Authenticated users, with an auth token, can use XYZ-based URLs to fetch tiles directly, and instantiate the Named Map as part of the request to your application. You do not have to do any other steps to initialize your map.
 
 To call a template_id in a URL:
 
@@ -543,21 +547,21 @@ To call a template_id in a URL:
 
 For example, a complete URL might appear as:
 
-"https://{username}.cartodb.com/api/v1/map/named/{template_id}/{layer}/{z}/{x}/{y}.png"
+"https://{username}.carto.com/api/v1/map/named/{template_id}/{layer}/{z}/{x}/{y}.png"
 
 The placeholders indicate the following:
 
-- [`template_id`](http://docs.cartodb.com/cartodb-platform/maps-api/named-maps/#response) is the response of your Named Map.
+- [`template_id`](http://docs.carto.com/carto-engine/maps-api/named-maps/#response) is the response of your Named Map.
 - layers can be a number (referring to the # layer of your map), all layers of your map, or a list of layers.
-  - To show just the basemap layer, enter the number value `0` in the layer placeholder "https://{username}.cartodb.com/api/v1/map/named/{template_id}/0/{z}/{x}/{y}.png"
-  - To show the first layer, enter the number value `1` in the layer placeholder "https://{username}.cartodb.com/api/v1/map/named/{template_id}/1/{z}/{x}/{y}.png"
-  - To show all layers, enter the value `all` for the layer placeholder "https://{username}.cartodb.com/api/v1/map/named/{template_id}/all/{z}/{x}/{y}.png"
-  - To show a [list of layers](http://docs.cartodb.com/cartodb-platform/maps-api/anonymous-maps/#blending-and-layer-selection), enter the comma separated layer value as 0,1,2 in the layer placeholder. For example, to show the basemap and the first layer, "https://{username}.cartodb.com/api/v1/map/named/{template_id}/0,1/{z}/{x}/{y}.png"
+  - To show just the basemap layer, enter the number value `0` in the layer placeholder "https://{username}.carto.com/api/v1/map/named/{template_id}/0/{z}/{x}/{y}.png"
+  - To show the first layer, enter the number value `1` in the layer placeholder "https://{username}.carto.com/api/v1/map/named/{template_id}/1/{z}/{x}/{y}.png"
+  - To show all layers, enter the value `all` for the layer placeholder "https://{username}.carto.com/api/v1/map/named/{template_id}/all/{z}/{x}/{y}.png"
+  - To show a [list of layers](http://docs.carto.com/carto-engine/maps-api/anonymous-maps/#blending-and-layer-selection), enter the comma separated layer value as 0,1,2 in the layer placeholder. For example, to show the basemap and the first layer, "https://{username}.carto.com/api/v1/map/named/{template_id}/0,1/{z}/{x}/{y}.png"
 
 
 ### Get Mapnik Retina Tiles
 
-Mapnik Retina tiles are not directly supported for Named Maps, so you cannot use the Named Map template_id. To fetch Mapnik Retina tiles, get the [layergroupid](http://docs.cartodb.com/cartodb-platform/maps-api/named-maps/#response-1) to initialize the map.
+Mapnik Retina tiles are not directly supported for Named Maps, so you cannot use the Named Map template_id. To fetch Mapnik Retina tiles, get the [layergroupid](http://docs.carto.com/carto-engine/maps-api/named-maps/#response-1) to initialize the map.
 
 Instantiate the map by using your `layergroupid` in the token placeholder:
 

docs/quickstart.md

@@ -22,10 +22,10 @@ $.ajax({
   type: 'POST',
   dataType: 'json',
   contentType: 'application/json',
-  url: 'https://{username}.cartodb.com/api/v1/map',
+  url: 'https://{username}.carto.com/api/v1/map',
   data: JSON.stringify(mapconfig),
   success: function(data) {
-    var templateUrl = 'https://{username}.cartodb.com/api/v1/map/' + data.layergroupid + '/{z}/{x}/{y}.png'
+    var templateUrl = 'https://{username}.carto.com/api/v1/map/' + data.layergroupid + '/{z}/{x}/{y}.png'
     console.log(templateUrl);
   }
 })
@@ -33,7 +33,7 @@ $.ajax({
 
 ## Named Maps
 
-Let's create a Named Map using some private tables in a CartoDB account.
+Let's create a Named Map using some private tables in a CARTO account.
 The following map config sets up a map of European countries that have a white fill color:
 
 ```javascript
@@ -56,12 +56,12 @@ The following map config sets up a map of European countries that have a white f
 }
 ```
 
-The MapConfig needs to be sent to CartoDB's Map API using an authenticated call. Here we will use a command line tool called `curl`. For more info about this tool, see [this blog post](http://quickleft.com/blog/command-line-tutorials-curl), or type `man curl` in bash. Using `curl`, and storing the config from above in a file `MapConfig.json`, the call would look like:
+The MapConfig needs to be sent to CARTO's Map API using an authenticated call. Here we will use a command line tool called `curl`. For more info about this tool, see [this blog post](http://quickleft.com/blog/command-line-tutorials-curl), or type `man curl` in bash. Using `curl`, and storing the config from above in a file `MapConfig.json`, the call would look like:
 
 #### Call
 
 ```bash
-curl 'https://{username}.cartodb.com/api/v1/map/named?api_key={api_key}' -H 'Content-Type: application/json' -d @mapconfig.json
+curl 'https://{username}.carto.com/api/v1/map/named?api_key={api_key}' -H 'Content-Type: application/json' -d @mapconfig.json
 ```
 
 To get the `URL` to fetch the tiles you need to instantiate the map, where `template_id` is the template name from the previous response.
@@ -69,7 +69,7 @@ To get the `URL` to fetch the tiles you need to instantiate the map, where `temp
 #### Call
 
 ```bash
-curl -X POST 'https://{username}.cartodb.com/api/v1/map/named/{template_id}' -H 'Content-Type: application/json'
+curl -X POST 'https://{username}.carto.com/api/v1/map/named/{template_id}' -H 'Content-Type: application/json'
 ```
 
 The response will return JSON with properties for the `layergroupid`, the timestamp (`last_updated`) of the last data modification and some key/value pairs with `metadata` for the `layers`.
@@ -96,5 +96,5 @@ Note: all `layers` in `metadata` will always have a `type` string and a `meta` d
 You can use the `layergroupid` to instantiate a URL template for accessing tiles on the client. Here we use the `layergroupid` from the example response above in this URL template:
 
 ```bash
-https://{username}.cartodb.com/api/v1/map/{layergroupid}/{z}/{x}/{y}.png
+https://{username}.carto.com/api/v1/map/{layergroupid}/{z}/{x}/{y}.png
 ```

docs/static_maps_api.md

@@ -78,7 +78,7 @@ format | the format for the image, supported types: `png`, `jpg`
 --- | ---
 |_ jpg | will have a default quality of 85.
 
-A Named Maps static image will get its constraints from the [`view` argument of the Create Named Map function](http://docs.cartodb.com/cartodb-platform/maps-api/named-maps/#arguments). If `view` is not defined, it will estimate the extent based on the involved tables, otherwise it fallbacks to `"zoom": 1`, `"lng": 0` and `"lat": 0`.
+A Named Maps static image will get its constraints from the [`view` argument of the Create Named Map function](http://docs.carto.com/carto-engine/maps-api/named-maps/#arguments). If `view` is not defined, it will estimate the extent based on the involved tables, otherwise it fallbacks to `"zoom": 1`, `"lng": 0` and `"lat": 0`.
 
 #### Layers
 
@@ -122,9 +122,9 @@ By manipulating the `"urlTemplate"` custom basemaps can be used in generating st
 },
 ```
 
-**CartoDB**
+**CARTO**
 
-As described in the [MapConfig File Format](http://docs.cartodb.com/cartodb-platform/maps-api/mapconfig/), a "cartodb" type layer is now just an alias to a "mapnik" type layer as above, intended for backwards compatibility.
+As described in the [MapConfig File Format](http://docs.carto.com/carto-engine/maps-api/mapconfig/), a "cartodb" type layer is now just an alias to a "mapnik" type layer as above, intended for backwards compatibility.
 
 ```javascript
 {
@@ -142,18 +142,23 @@ Additionally, static images from Torque maps and other map layers can be used to
 
 ### Caching
 
-It is important to note that generated images are cached from the live data referenced with the `layergroupid token` on the specified CartoDB account. This means that if the data changes, the cached image will also change. When linking dynamically, it is important to take into consideration the state of the data and longevity of the static image to avoid broken images or changes in how the image is displayed. To obtain a static snapshot of the map as it is today and preserve the image long-term regardless of changes in data, the image must be saved and stored locally.
+It is important to note that generated images are cached from the live data referenced with the `layergroupid token` on the specified CARTO account. This means that if the data changes, the cached image will also change. When linking dynamically, it is important to take into consideration the state of the data and longevity of the static image to avoid broken images or changes in how the image is displayed. To obtain a static snapshot of the map as it is today and preserve the image long-term regardless of changes in data, the image must be saved and stored locally.
 
 ### Limits
 
-* While images can encompass an entirety of a map, the default limit for pixel range is 8192 x 8192.
-* Image resolution by default is set to 72 DPI
-* JPEG quality by default is 85%
-* Timeout limits for generating static maps are the same across the CartoDB Editor and Platform. It is important to ensure timely processing of queries.
+* While images can encompass an entirety of a map, the limit for pixel range is 8192 x 8192.
+* Image resolution is set to 72 DPI
+* JPEG quality is 85%
+* Timeout limits for generating static maps are the same across CARTO Builder and CARTO Engine. It is important to ensure timely processing of queries.
+* If you are publishing your map as a static image with the API, you must manually add [attributions](https://carto.com/attribution) for your static map image. For example, add the following attribution code:
+
+{% highlight javascript %}
+attribution: '&copy; <a href="http://www.openstreetmap.org/copyright">OpenStreetMap</a> contributors, &copy; <a href="https://carto.com/attributions">CARTO</a>
+{% endhighlight %}
 
 ## Examples
 
-After instantiating a map from a CartoDB account:
+After instantiating a map from a CARTO account:
 
 #### Call
 

lib/cartodb/api/auth_api.js

@@ -19,22 +19,22 @@ function AuthApi(pgConnection, metadataBackend, mapStore, templateMaps) {
 
 module.exports = AuthApi;
 
-// Check if a request is authorized by a signer
+// Check if the user is authorized by a signer
 //
-// @param req express request object
+// @param res express response object
 // @param callback function(err, signed_by) signed_by will be
 //                 null if the request is not signed by anyone
 //                 or will be a string cartodb username otherwise.
 //
-AuthApi.prototype.authorizedBySigner = function(req, callback) {
-    if ( ! req.params.token || ! req.params.signer ) {
+AuthApi.prototype.authorizedBySigner = function(res, callback) {
+    if ( ! res.locals.token || ! res.locals.signer ) {
         return callback(null, false); // no signer requested
     }
 
     var self = this;
 
-    var layergroup_id = req.params.token;
-    var auth_token = req.params.auth_token;
+    var layergroup_id = res.locals.token;
+    var auth_token = res.locals.auth_token;
 
     this.mapStore.load(layergroup_id, function(err, mapConfig) {
         if (err) {
@@ -84,30 +84,29 @@ AuthApi.prototype.authorizedByAPIKey = function(user, req, callback) {
  * Check access authorization
  *
  * @param req - standard req object. Importantly contains table and host information
+ * @param res - standard res object. Contains the auth parameters in locals
  * @param callback function(err, allowed) is access allowed not?
  */
-AuthApi.prototype.authorize = function(req, callback) {
+AuthApi.prototype.authorize = function(req, res, callback) {
     var self = this;
-    var user = req.context.user;
+    var user = res.locals.user;
 
     step(
         function () {
             self.authorizedByAPIKey(user, req, this);
         },
         function checkApiKey(err, authorized){
-            if (req.profiler) {
-                req.profiler.done('authorizedByAPIKey');
-            }
+            req.profiler.done('authorizedByAPIKey');
             assert.ifError(err);
 
             // if not authorized by api_key, continue
             if (!authorized)  {
                 // not authorized by api_key, check if authorized by signer
-                return self.authorizedBySigner(req, this);
+                return self.authorizedBySigner(res, this);
             }
 
             // authorized by api key, login as the given username and stop
-            self.pgConnection.setDBAuth(user, req.params, function(err) {
+            self.pgConnection.setDBAuth(user, res.locals, function(err) {
                 callback(err, true); // authorized (or error)
             });
         },
@@ -122,7 +121,7 @@ AuthApi.prototype.authorize = function(req, callback) {
                 // if no signer name was given, let dbparams and
                 // PostgreSQL do the rest.
                 //
-                if ( ! req.params.signer ) {
+                if ( ! res.locals.signer ) {
                     return callback(null, true); // authorized so far
                 }
 
@@ -130,10 +129,8 @@ AuthApi.prototype.authorize = function(req, callback) {
                 return callback(null, false);
             }
 
-            self.pgConnection.setDBAuth(user, req.params, function(err) {
-                if (req.profiler) {
-                    req.profiler.done('setDBAuth');
-                }
+            self.pgConnection.setDBAuth(user, res.locals, function(err) {
+                req.profiler.done('setDBAuth');
                 callback(err, true); // authorized (or error)
             });
         }

lib/cartodb/api/filter_stats_api.js

@@ -1,6 +1,6 @@
 var _ = require('underscore');
 var step = require('step');
-var CamshaftFilter = require('../models/filter/camshaft');
+var AnalysisFilter = require('../models/filter/analysis');
 
 function FilterStatsApi(pgQueryRunner) {
     this.pgQueryRunner = pgQueryRunner;
@@ -40,8 +40,8 @@ FilterStatsApi.prototype.getFilterStats = function (username, unfiltered_query,
       },
       function getFilteredRows() {
           if ( filters && !_.isEmpty(filters)) {
-              var camshaftFilter = new CamshaftFilter(filters);
-              var query = camshaftFilter.sql(unfiltered_query);
+              var analysisFilter = new AnalysisFilter(filters);
+              var query = analysisFilter.sql(unfiltered_query);
               getEstimatedRows(self.pgQueryRunner, username, query, this);
           } else {
               this(null, null);

lib/cartodb/api/user_limits_api.js

@@ -1,3 +1,5 @@
+var step = require('step');
+
 /**
  *
  * @param metadataBackend
@@ -13,16 +15,65 @@ function UserLimitsApi(metadataBackend, options) {
 
 module.exports = UserLimitsApi;
 
-UserLimitsApi.prototype.getRenderLimits = function (username, callback) {
+UserLimitsApi.prototype.getRenderLimits = function (username, apiKey, callback) {
     var self = this;
-    this.metadataBackend.getTilerRenderLimit(username, function handleTilerLimits(err, renderLimit) {
+
+    var limits = {
+        cacheOnTimeout: self.options.limits.cacheOnTimeout || false,
+        render: self.options.limits.render || 0
+    };
+
+    self.getTimeoutRenderLimit(username, apiKey, function (err, timeoutRenderLimit) {
         if (err) {
             return callback(err);
         }
 
-        return callback(null, {
-            cacheOnTimeout: self.options.limits.cacheOnTimeout || false,
-            render: renderLimit || self.options.limits.render || 0
-        });
+        if (timeoutRenderLimit && timeoutRenderLimit.render) {
+            if (Number.isFinite(timeoutRenderLimit.render)) {
+                limits.render = timeoutRenderLimit.render;
+            }
+        }
+
+        return callback(null, limits);
     });
 };
+
+UserLimitsApi.prototype.getTimeoutRenderLimit = function (username, apiKey, callback) {
+    var self = this;
+
+    step(
+        function isAuthorized() {
+            var next = this;
+
+            if (!apiKey) {
+                return next(null, false);
+            }
+
+            self.metadataBackend.getUserMapKey(username, function (err, userApiKey) {
+                if (err) {
+                    return next(err);
+                }
+
+                return next(null, userApiKey === apiKey);
+            });
+        },
+        function getUserTimeoutRenderLimits(err, authorized) {
+            var next = this;
+
+            if (err) {
+                return next(err);
+            }
+
+            self.metadataBackend.getUserTimeoutRenderLimits(username, function (err, timeoutRenderLimit) {
+                if (err) {
+                    return next(err);
+                }
+
+                next(null, {
+                    render: authorized ? timeoutRenderLimit.render : timeoutRenderLimit.renderPublic
+                });
+            });
+        },
+        callback
+    );
+};

lib/cartodb/backends/analysis-status.js

@@ -9,7 +9,10 @@ module.exports = AnalysisStatusBackend;
 AnalysisStatusBackend.prototype.getNodeStatus = function (params, callback) {
     var nodeId = params.nodeId;
 
-    var statusQuery = 'SELECT node_id, status, updated_at FROM cdb_analysis_catalog where node_id = \'' + nodeId + '\'';
+    var statusQuery = [
+        'SELECT node_id, status, updated_at, last_error_message as error_message',
+        'FROM cdb_analysis_catalog where node_id = \'' + nodeId + '\''
+    ].join(' ');
 
     var pg = new PSQL(dbParamsFromReqParams(params));
     pg.query(statusQuery, function(err, result) {
@@ -21,10 +24,16 @@ AnalysisStatusBackend.prototype.getNodeStatus = function (params, callback) {
 
         var rows = result.rows || [];
 
-        return callback(null, rows[0] || {
+        var statusResponse = rows[0] || {
             node_id: nodeId,
             status: 'unknown'
-        });
+        };
+
+        if (statusResponse.status !== 'failed') {
+            delete statusResponse.error_message;
+        }
+
+        return callback(null, statusResponse);
     }, true); // use read-only transaction
 };
 

lib/cartodb/backends/analysis.js

@@ -1,19 +1,93 @@
-var camshaft = require('camshaft');
+'use strict';
 
-function AnalysisBackend(options) {
-    var batchConfig = options.batch || {};
+var _ = require('underscore');
+var camshaft = require('camshaft');
+var fs = require('fs');
+
+var REDIS_LIMITS = {
+    DB: 5,
+    PREFIX: 'limits:analyses:' // + username
+};
+
+function AnalysisBackend (metadataBackend, options) {
+    this.metadataBackend = metadataBackend;
+    this.options = options || {};
+    this.options.limits = this.options.limits || {};
+    this.setBatchConfig(this.options.batch);
+    this.setLoggerConfig(this.options.logger);
+}
+
+module.exports = AnalysisBackend;
+
+AnalysisBackend.prototype.setBatchConfig = function (options) {
+    var batchConfig = options || {};
     batchConfig.endpoint = batchConfig.endpoint || 'http://127.0.0.1:8080/api/v1/sql/job';
     batchConfig.inlineExecution = batchConfig.inlineExecution || false;
     batchConfig.hostHeaderTemplate = batchConfig.hostHeaderTemplate || '{{=it.username}}.localhost.lan';
     this.batchConfig = batchConfig;
-}
+};
 
-module.exports = AnalysisBackend;
+AnalysisBackend.prototype.setLoggerConfig = function (options) {
+    this.loggerConfig = options || {};
+
+    if (this.loggerConfig.filename) {
+        this.stream = fs.createWriteStream(this.loggerConfig.filename, { flags: 'a', encoding: 'utf8' });
+
+        process.on('SIGHUP', function () {
+            if (this.stream) {
+                this.stream.destroy();
+            }
+
+            this.stream = fs.createWriteStream(this.loggerConfig.filename, { flags: 'a', encoding: 'utf8' });
+        }.bind(this));
+    }
+};
 
 AnalysisBackend.prototype.create = function(analysisConfiguration, analysisDefinition, callback) {
     analysisConfiguration.batch.endpoint = this.batchConfig.endpoint;
     analysisConfiguration.batch.inlineExecution = this.batchConfig.inlineExecution;
     analysisConfiguration.batch.hostHeaderTemplate = this.batchConfig.hostHeaderTemplate;
 
-    camshaft.create(analysisConfiguration, analysisDefinition, callback);
+    analysisConfiguration.logger = {
+        stream: this.stream ? this.stream : process.stdout
+    };
+
+    this.getAnalysesLimits(analysisConfiguration.user, function(err, limits) {
+        analysisConfiguration.limits = limits || {};
+        camshaft.create(analysisConfiguration, analysisDefinition, callback);
+    });
+};
+
+AnalysisBackend.prototype.getAnalysesLimits = function(username, callback) {
+    var self = this;
+
+    var analysesLimits = {
+        analyses: {
+            // buffer: {
+            //     timeout: 1000,
+            //     maxNumberOfRows: 1e6
+            // }
+        }
+    };
+
+    Object.keys(self.options.limits).forEach(function(analysisTypeOrTag) {
+        analysesLimits.analyses[analysisTypeOrTag] = _.extend({}, self.options.limits[analysisTypeOrTag]);
+    });
+
+    var analysesLimitsKey = REDIS_LIMITS.PREFIX + username;
+    this.metadataBackend.redisCmd(REDIS_LIMITS.DB, 'HGETALL', [analysesLimitsKey], function(err, analysesTimeouts) {
+        // analysesTimeouts wil be something like: { moran: 3000, intersection: 5000 }
+        analysesTimeouts = analysesTimeouts || {};
+
+        Object.keys(analysesTimeouts).forEach(function(analysisType) {
+            analysesLimits.analyses[analysisType] = _.defaults(
+                {
+                    timeout: Number.isFinite(+analysesTimeouts[analysisType]) ? +analysesTimeouts[analysisType] : 0
+                },
+                analysesLimits.analyses[analysisType]
+            );
+        });
+
+        return callback(null, analysesLimits);
+    });
 };

lib/cartodb/backends/dataview.js

@@ -2,11 +2,8 @@ var assert = require('assert');
 
 var _ = require('underscore');
 var PSQL = require('cartodb-psql');
-var camshaft = require('camshaft');
 var step = require('step');
 
-var Timer = require('../stats/timer');
-
 var BBoxFilter = require('../models/filter/bbox');
 
 var DataviewFactory = require('../models/dataview/factory');
@@ -16,6 +13,9 @@ var overviewsQueryRewriter = new OverviewsQueryRewriter({
     zoom_level: 'CDB_ZoomFromScale(!scale_denominator!)'
 });
 
+var dot = require('dot');
+dot.templateSettings.strip = false;
+
 function DataviewBackend(analysisBackend) {
     this.analysisBackend = analysisBackend;
 }
@@ -23,251 +23,132 @@ function DataviewBackend(analysisBackend) {
 module.exports = DataviewBackend;
 
 DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, params, callback) {
-    var self = this;
-
-    var timer = new Timer();
 
     var dataviewName = params.dataviewName;
-
-    var mapConfig;
-    var dataviewDefinition;
     step(
         function getMapConfig() {
             mapConfigProvider.getMapConfig(this);
         },
-        function _getDataviewDefinition(err, _mapConfig) {
+        function runDataviewQuery(err, mapConfig) {
             assert.ifError(err);
 
-            mapConfig = _mapConfig;
-
-            var _dataviewDefinition = getDataviewDefinition(mapConfig.obj(), dataviewName);
-            if (!_dataviewDefinition) {
+            var dataviewDefinition = getDataviewDefinition(mapConfig.obj(), dataviewName);
+            if (!dataviewDefinition) {
                 throw new Error("Dataview '" + dataviewName + "' does not exists");
             }
 
-            dataviewDefinition = _dataviewDefinition;
-
-            return dataviewDefinition;
-        },
-        function loadAnalysis(err) {
-            assert.ifError(err);
-
-            var analysisConfiguration = {
-                db: {
-                    host: params.dbhost,
-                    port: params.dbport,
-                    dbname: params.dbname,
-                    user: params.dbuser,
-                    pass: params.dbpassword
-                },
-                batch: {
-                    username: user,
-                    apiKey: params.api_key
-                }
-            };
-
-            var sourceId = dataviewDefinition.source.id;
-            var analysisDefinition = getAnalysisDefinition(mapConfig.obj().analyses, sourceId);
-
-            var next = this;
-
-            self.analysisBackend.create(analysisConfiguration, analysisDefinition, function(err, analysis) {
-                if (err) {
-                    return next(err);
-                }
-
-                var sourceId2Node = {};
-                var rootNode = analysis.getRoot();
-                if (rootNode.params && rootNode.params.id) {
-                    sourceId2Node[rootNode.params.id] = rootNode;
-                }
-
-                analysis.getSortedNodes().forEach(function(node) {
-                    if (node.params && node.params.id) {
-                        sourceId2Node[node.params.id] = node;
-                    }
-                });
-
-                var node = sourceId2Node[sourceId];
-
-                if (!node) {
-                    return next(new Error('Analysis node not found for dataview'));
-                }
-
-                return next(null, node);
-            });
-        },
-        function runDataviewQuery(err, node) {
-            assert.ifError(err);
+            var ownFilter = +params.own_filter;
+            var noFilters = +params.no_filters;
+            if (Number.isFinite(ownFilter) && Number.isFinite(noFilters)) {
+                err = new Error();
+                err.message = 'Both own_filter and no_filters cannot be sent in the same request';
+                err.type = 'dataview';
+                err.http_status = 400;
+                return callback(err);
+            }
 
             var pg = new PSQL(dbParamsFromReqParams(params));
-
-            var ownFilter = +params.own_filter;
-            ownFilter = !!ownFilter;
-
-            var query;
-
-            if (ownFilter) {
-                query = node.getQuery();
-            } else {
-                var applyFilters = {};
-                applyFilters[dataviewName] = false;
-                query = node.getQuery(applyFilters);
-            }
-
-            var sourceId =  dataviewDefinition.source.id; // node.id
-            var layer = _.find(
-                mapConfig.obj().layers,
-                function(l){ return l.options.source && (l.options.source.id === sourceId); }
-            );
-            var queryRewriteData = layer && layer.options.query_rewrite_data;
-            if ( queryRewriteData ) {
-                if ( node.type === 'source' ) {
-                    var filters = node.filters; // TODO: node.getFilters() when available in camshaft
-                    var filters_disabler = Object.keys(filters).reduce(
-                        function(disabler, filter_id){ disabler[filter_id] = false; return disabler; },
-                        {}
-                    );
-                    var unfiltered_query = node.getQuery(filters_disabler);
-                    queryRewriteData = _.extend(
-                        {},
-                        queryRewriteData, { filters: filters, unfiltered_query: unfiltered_query }
-                    );
-                }
-            }
-
+            
+            var query = getDataviewQuery(dataviewDefinition, ownFilter, noFilters);
             if (params.bbox) {
-                var bboxFilter = new BBoxFilter({column: 'the_geom', srid: 4326}, {bbox: params.bbox});
+                var bboxFilter = new BBoxFilter({column: 'the_geom_webmercator', srid: 3857}, {bbox: params.bbox});
                 query = bboxFilter.sql(query);
-                if ( queryRewriteData ) {
-                    var bbox_filter_definition = {
-                        type: 'bbox',
-                        options: {
-                            column: 'the_geom',
-                            srid: 4326,
-                        },
-                        params: {
-                            bbox: params.bbox
-                        }
-                    };
-                    queryRewriteData = _.extend(queryRewriteData, { bbox_filter: bbox_filter_definition });
-                }
             }
 
+            var queryRewriteData = getQueryRewriteData(mapConfig, dataviewDefinition, params);
+
             var dataviewFactory = DataviewFactoryWithOverviews.getFactory(
                 overviewsQueryRewriter, queryRewriteData, { bbox: params.bbox }
             );
 
-            var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins'),
-                function castNumbers(overrides, val, k) {
-                    overrides[k] = Number.isFinite(+val) ? +val : val;
-                    return overrides;
-                },
-                {ownFilter: ownFilter}
-            );
-
             var dataview = dataviewFactory.getDataview(query, dataviewDefinition);
-            dataview.getResult(pg, overrideParams, this);
+            dataview.getResult(pg, getOverrideParams(params, !!ownFilter), this);
         },
         function returnCallback(err, result) {
-            return callback(err, result, timer.getTimes());
+            return callback(err, result);
         }
     );
 };
 
-DataviewBackend.prototype.search = function (mapConfigProvider, user, params, callback) {
-    var self = this;
+function getDataviewQuery(dataviewDefinition, ownFilter, noFilters) {
+    if (noFilters) {
+        return dataviewDefinition.sql.no_filters;
+    } else if (ownFilter === 1) {
+        return dataviewDefinition.sql.own_filter_on;
+    } else {
+        return dataviewDefinition.sql.own_filter_off;
+    }
+}
 
-    var timer = new Timer();
+function getQueryRewriteData(mapConfig, dataviewDefinition, params) {
+    var sourceId = dataviewDefinition.source.id; // node.id
+    var layer = _.find(mapConfig.obj().layers, function(l) {
+        return l.options.source && (l.options.source.id === sourceId);
+    });
+    var queryRewriteData = layer && layer.options.query_rewrite_data;
+    if (queryRewriteData && dataviewDefinition.node.type === 'source') {
+        queryRewriteData = _.extend({}, queryRewriteData, {
+            filters: dataviewDefinition.node.filters,
+            unfiltered_query: dataviewDefinition.sql.own_filter_on
+        });
+    }
 
-    var dataviewName = params.dataviewName;
+    if (params.bbox && queryRewriteData) {
+        var bbox_filter_definition = {
+            type: 'bbox',
+            options: {
+                column: 'the_geom_webmercator',
+                srid: 3857
+            },
+            params: {
+                bbox: params.bbox
+            }
+        };
+        queryRewriteData = _.extend(queryRewriteData, { bbox_filter: bbox_filter_definition });
+    }
 
-    var mapConfig;
-    var dataviewDefinition;
+    return queryRewriteData;
+}
+
+function getOverrideParams(params, ownFilter) {
+    var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins', 'offset', 'categories'),
+        function castNumbers(overrides, val, k) {
+            if (!Number.isFinite(+val)) {
+                throw new Error('Invalid number format for parameter \'' + k + '\'');
+            }
+            overrides[k] = +val;
+            return overrides;
+        },
+        {ownFilter: ownFilter}
+    );
+
+    // validation will be delegated to the proper dataview
+    if (params.aggregation !== undefined) {
+        overrideParams.aggregation = params.aggregation;
+    }
+
+    return overrideParams;
+}
+
+DataviewBackend.prototype.search = function (mapConfigProvider, user, dataviewName, params, callback) {
     step(
         function getMapConfig() {
             mapConfigProvider.getMapConfig(this);
         },
-        function _getDataviewDefinition(err, _mapConfig) {
+        function runDataviewSearchQuery(err, mapConfig) {
             assert.ifError(err);
 
-            mapConfig = _mapConfig;
-
-            var _dataviewDefinition = getDataviewDefinition(mapConfig.obj(), dataviewName);
-            if (!_dataviewDefinition) {
+            var dataviewDefinition = getDataviewDefinition(mapConfig.obj(), dataviewName);
+            if (!dataviewDefinition) {
                 throw new Error("Dataview '" + dataviewName + "' does not exists");
             }
 
-            dataviewDefinition = _dataviewDefinition;
-
-            return dataviewDefinition;
-        },
-        function loadAnalysis(err) {
-            assert.ifError(err);
-
-            var analysisConfiguration = {
-                db: {
-                    host: params.dbhost,
-                    port: params.dbport,
-                    dbname: params.dbname,
-                    user: params.dbuser,
-                    pass: params.dbpassword
-                },
-                batch: {
-                    // TODO load this from configuration
-                    endpoint: 'http://127.0.0.1:8080/api/v1/sql/job',
-                    username: user,
-                    apiKey: params.api_key
-                }
-            };
-
-            var sourceId = dataviewDefinition.source.id;
-            var analysisDefinition = getAnalysisDefinition(mapConfig.obj().analyses, sourceId);
-
-            var next = this;
-
-            self.analysisBackend.create(analysisConfiguration, analysisDefinition, function(err, analysis) {
-                if (err) {
-                    return next(err);
-                }
-
-                var sourceId2Node = {};
-                var rootNode = analysis.getRoot();
-                if (rootNode.params && rootNode.params.id) {
-                    sourceId2Node[rootNode.params.id] = rootNode;
-                }
-
-                analysis.getSortedNodes().forEach(function(node) {
-                    if (node.params && node.params.id) {
-                        sourceId2Node[node.params.id] = node;
-                    }
-                });
-
-                var node = sourceId2Node[sourceId];
-
-                if (!node) {
-                    return next(new Error('Analysis node not found for dataview'));
-                }
-
-                return next(null, node);
-            });
-        },
-        function runDataviewQuery(err, node) {
-            assert.ifError(err);
-
             var pg = new PSQL(dbParamsFromReqParams(params));
 
             var ownFilter = +params.own_filter;
             ownFilter = !!ownFilter;
 
-            var query;
-            if (ownFilter) {
-                query = node.getQuery();
-            } else {
-                var applyFilters = {};
-                applyFilters[dataviewName] = false;
-                query = node.getQuery(applyFilters);
-            }
+            var query = (ownFilter) ? dataviewDefinition.sql.own_filter_on : dataviewDefinition.sql.own_filter_off;
 
             if (params.bbox) {
                 var bboxFilter = new BBoxFilter({column: 'the_geom', srid: 4326}, {bbox: params.bbox});
@@ -280,23 +161,11 @@ DataviewBackend.prototype.search = function (mapConfigProvider, user, params, ca
             dataview.search(pg, userQuery, this);
         },
         function returnCallback(err, result) {
-            return callback(err, result, timer.getTimes());
+            return callback(err, result);
         }
     );
 };
 
-function getAnalysisDefinition(mapConfigAnalyses, sourceId) {
-    mapConfigAnalyses = mapConfigAnalyses || [];
-    for (var i = 0; i < mapConfigAnalyses.length; i++) {
-        var analysisGraph = new camshaft.reference.AnalysisGraph(mapConfigAnalyses[i]);
-        var nodes = analysisGraph.getNodesWithId();
-        if (nodes.hasOwnProperty(sourceId)) {
-            return mapConfigAnalyses[i];
-        }
-    }
-    throw new Error('There is no associated analysis for the dataview source id');
-}
-
 function getDataviewDefinition(mapConfig, dataviewName) {
     var dataviews = mapConfig.dataviews || {};
     return dataviews[dataviewName];

lib/cartodb/backends/layer-stats/empty-layer-stats.js

@@ -0,0 +1,16 @@
+function EmptyLayerStats(types) {
+    this._types = types || {};
+}
+
+EmptyLayerStats.prototype.is = function (type) {
+    return this._types[type] ? this._types[type] : false;
+};
+
+EmptyLayerStats.prototype.getStats =
+function (layer, dbConnection, callback) {
+    setImmediate(function() {
+        callback(null, {});
+    });
+};
+
+module.exports = EmptyLayerStats;

lib/cartodb/backends/layer-stats/factory.js

@@ -0,0 +1,23 @@
+var LayerStats = require('./layer-stats');
+var EmptyLayerStats = require('./empty-layer-stats');
+var MapnikLayerStats = require('./mapnik-layer-stats');
+var TorqueLayerStats = require('./torque-layer-stats');
+
+module.exports = function LayerStatsFactory(type) {
+    var layerStatsIterator = [];
+    var selectedType = type || 'ALL';
+
+    if (selectedType === 'ALL') {
+        layerStatsIterator.push(new EmptyLayerStats({ http: true, plain: true }));
+        layerStatsIterator.push(new MapnikLayerStats());
+        layerStatsIterator.push(new TorqueLayerStats());
+    } else if (selectedType === 'mapnik') {
+        layerStatsIterator.push(new EmptyLayerStats({ http: true, plain: true, torque: true }));
+        layerStatsIterator.push(new MapnikLayerStats());
+    } else if (selectedType === 'torque') {
+        layerStatsIterator.push(new EmptyLayerStats({ http: true, plain: true, mapnik: true }));
+        layerStatsIterator.push(new TorqueLayerStats());
+    }
+
+    return new LayerStats(layerStatsIterator);
+};

lib/cartodb/backends/layer-stats/layer-stats.js

@@ -0,0 +1,45 @@
+var queue = require('queue-async');
+
+function LayerStats(layerStatsIterator) {
+    this.layerStatsIterator = layerStatsIterator;
+}
+
+LayerStats.prototype.getStats = function (mapConfig, dbConnection, callback) {
+    var self = this;
+    var stats = [];
+
+    if (!mapConfig.getLayers().length) {
+        return callback(null, stats);
+    }
+    var metaQueue = queue(mapConfig.getLayers().length);
+    mapConfig.getLayers().forEach(function (layer, layerId) {
+        var layerType = mapConfig.layerType(layerId);
+
+        for (var i = 0; i < self.layerStatsIterator.length; i++) {
+            if (self.layerStatsIterator[i].is(layerType)) {
+                var getStats = self.layerStatsIterator[i].getStats.bind(self.layerStatsIterator[i]);
+                metaQueue.defer(getStats, layer, dbConnection);
+                break;
+            }
+        }
+    });
+
+    metaQueue.awaitAll(function (err, results) {
+        if (err) {
+            return callback(err);
+        }
+
+        if (!results) {
+            return callback(null, null);
+        }
+
+        mapConfig.getLayers().forEach(function (layer, layerIndex) {
+            stats[layerIndex] = results[layerIndex];
+        });
+
+        return callback(err, stats);
+    });
+
+};
+
+module.exports = LayerStats;

lib/cartodb/backends/layer-stats/mapnik-layer-stats.js

@@ -0,0 +1,28 @@
+var queryUtils = require('../../utils/query-utils');
+
+function MapnikLayerStats () {
+    this._types = {
+        mapnik: true,
+        cartodb: true
+    };
+}
+
+MapnikLayerStats.prototype.is = function (type) {
+    return this._types[type] ? this._types[type] : false;
+};
+
+MapnikLayerStats.prototype.getStats =
+function (layer, dbConnection, callback) {
+    var queryRowCountSql = queryUtils.getQueryRowCount(layer.options.sql);
+    // This query would gather stats for postgresql table if not exists
+    dbConnection.query(queryRowCountSql, function (err, res) {
+        if (err) {
+            return callback(null, {estimatedFeatureCount: -1});
+        } else {
+            // We decided that the relation is 1 row == 1 feature
+            return callback(null, {estimatedFeatureCount: res.rows[0].rows});
+        }
+    });
+};
+
+module.exports = MapnikLayerStats;

lib/cartodb/backends/layer-stats/torque-layer-stats.js

@@ -0,0 +1,16 @@
+function TorqueLayerStats() {
+    this._types = {
+        torque: true
+    };
+}
+
+TorqueLayerStats.prototype.is = function (type) {
+    return this._types[type] ? this._types[type] : false;
+};
+
+TorqueLayerStats.prototype.getStats =
+function (layer, dbConnection, callback) {
+    return callback(null, {});
+};
+
+module.exports = TorqueLayerStats;

lib/cartodb/backends/stats.js

@@ -0,0 +1,16 @@
+var layerStats = require('./layer-stats/factory');
+
+function StatsBackend() {
+}
+
+module.exports = StatsBackend;
+
+StatsBackend.prototype.getStats = function(mapConfig, dbConnection, callback) {
+    var enabledFeatures = global.environment.enabledFeatures;
+    var layerStatsEnabled = enabledFeatures ? enabledFeatures.layerStats: false;
+    if (layerStatsEnabled) {
+        layerStats().getStats(mapConfig, dbConnection, callback);
+    } else {
+        return callback(null, []);
+    }
+};

lib/cartodb/backends/template_maps.js

@@ -55,11 +55,9 @@ util.inherits(TemplateMaps, EventEmitter);
 module.exports = TemplateMaps;
 
 
-var o = TemplateMaps.prototype;
-
 //--------------- PRIVATE METHODS --------------------------------
 
-o._userTemplateLimit = function() {
+TemplateMaps.prototype._userTemplateLimit = function() {
   return this.opts.max_user_templates || 0;
 };
 
@@ -70,7 +68,7 @@ o._userTemplateLimit = function() {
  * @param redisArgs - the arguments for the redis function in an array
  * @param callback - function to pass results too.
  */
-o._redisCmd = function(redisFunc, redisArgs, callback) {
+TemplateMaps.prototype._redisCmd = function(redisFunc, redisArgs, callback) {
   var redisClient;
   var that = this;
   var db = that.db_signatures;
@@ -97,7 +95,7 @@ o._redisCmd = function(redisFunc, redisArgs, callback) {
 var _reValidNameIdentifier = /^[a-z0-9][0-9a-z_\-]*$/i;
 var _reValidPlaceholderIdentifier = /^[a-z][0-9a-z_]*$/i;
 // jshint maxcomplexity:15
-o._checkInvalidTemplate = function(template) {
+TemplateMaps.prototype._checkInvalidTemplate = function(template) {
   if ( template.version !== '0.0.1' ) {
     return new Error("Unsupported template version " + template.version);
   }
@@ -200,7 +198,7 @@ function templateDefaults(template) {
 // @param callback function(err, tpl_id)
 //        Return template identifier (only valid for given user)
 //
-o.addTemplate = function(owner, template, callback) {
+TemplateMaps.prototype.addTemplate = function(owner, template, callback) {
     var self = this;
 
     template = templateDefaults(template);
@@ -258,7 +256,7 @@ o.addTemplate = function(owner, template, callback) {
 //
 // @param callback function(err)
 //
-o.delTemplate = function(owner, tpl_id, callback) {
+TemplateMaps.prototype.delTemplate = function(owner, tpl_id, callback) {
     var self = this;
     step(
         function deleteTemplate() {
@@ -297,7 +295,8 @@ o.delTemplate = function(owner, tpl_id, callback) {
 //
 // @param callback function(err)
 //
-o.updTemplate = function(owner, tpl_id, template, callback) {
+TemplateMaps.prototype.updTemplate = function(owner, tpl_id, template, callback) {
+
     var self = this;
 
     template = templateDefaults(template);
@@ -356,7 +355,7 @@ o.updTemplate = function(owner, tpl_id, template, callback) {
 // @param callback function(err, tpl_id_list)
 //        Returns a list of template identifiers
 //
-o.listTemplates = function(owner, callback) {
+TemplateMaps.prototype.listTemplates = function(owner, callback) {
     this._redisCmd('HKEYS', [ this.key_usr_tpl({owner:owner}) ], callback);
 };
 
@@ -370,7 +369,7 @@ o.listTemplates = function(owner, callback) {
 // @param callback function(err, template)
 //        Return full template definition
 //
-o.getTemplate = function(owner, tpl_id, callback) {
+TemplateMaps.prototype.getTemplate = function(owner, tpl_id, callback) {
     var self = this;
     step(
         function getTemplate() {
@@ -386,7 +385,7 @@ o.getTemplate = function(owner, tpl_id, callback) {
     );
 };
 
-o.isAuthorized = function(template, authTokens) {
+TemplateMaps.prototype.isAuthorized = function(template, authTokens) {
     if (!template) {
         return false;
     }
@@ -431,14 +430,18 @@ var _reNumber = /^([-+]?[\d\.]?\d+([eE][+-]?\d+)?)$/,
     _reCSSColorVal = /^#[0-9a-fA-F]{3,6}$/;
 
 function _replaceVars (str, params) {
-  //return _.template(str, params); // lazy way, possibly dangerous
-  // Construct regular expressions for each param
+    // Construct regular expressions for each param
     Object.keys(params).forEach(function(k) {
         str = str.replace(new RegExp("<%=\\s*" + k + "\\s*%>", "g"), params[k]);
     });
     return str;
 }
-o.instance = function(template, params) {
+
+function isObject(val) {
+    return ( _.isObject(val) && !_.isArray(val) && !_.isFunction(val));
+}
+
+TemplateMaps.prototype.instance = function(template, params) {
   var all_params = {};
   var phold = template.placeholders || {};
   Object.keys(phold).forEach(function(k) {
@@ -475,6 +478,13 @@ o.instance = function(template, params) {
 
   // NOTE: we're deep-cloning the layergroup here
   var layergroup = JSON.parse(JSON.stringify(template.layergroup));
+
+  if (layergroup.buffersize && isObject(layergroup.buffersize)) {
+      Object.keys(layergroup.buffersize).forEach(function(k) {
+        layergroup.buffersize[k] = parseInt(_replaceVars(layergroup.buffersize[k], all_params), 10);
+      });
+  }
+
   for (var i=0; i<layergroup.layers.length; ++i) {
     var lyropt = layergroup.layers[i].options;
 
@@ -500,7 +510,7 @@ o.instance = function(template, params) {
 };
 
 // Return a fingerPrint of the object
-o.fingerPrint = function(template) {
+TemplateMaps.prototype.fingerPrint = function(template) {
   return crypto.createHash('md5')
     .update(JSON.stringify(template))
     .digest('hex')

lib/cartodb/backends/turbo-carto-postgres-datasource.js

@@ -0,0 +1,116 @@
+'use strict';
+
+var dot = require('dot');
+dot.templateSettings.strip = false;
+
+function createTemplate(method) {
+    return dot.template([
+        'SELECT',
+        'min({{=it._column}}) min_val,',
+        'max({{=it._column}}) max_val,',
+        'avg({{=it._column}}) avg_val,',
+        method,
+        'FROM ({{=it._sql}}) _table_sql WHERE {{=it._column}} IS NOT NULL',
+        'AND',
+        '  {{=it._column}} != \'infinity\'::float',
+        'AND',
+        '  {{=it._column}} != \'-infinity\'::float',
+        'AND',
+        '  {{=it._column}} != \'NaN\'::float'
+    ].join('\n'));
+}
+
+var methods = {
+    quantiles: 'CDB_QuantileBins(array_agg({{=it._column}}::numeric), {{=it._buckets}}) as quantiles',
+    equal: 'CDB_EqualIntervalBins(array_agg({{=it._column}}::numeric), {{=it._buckets}}) as equal',
+    jenks: 'CDB_JenksBins(array_agg({{=it._column}}::numeric), {{=it._buckets}}) as jenks',
+    headtails: 'CDB_HeadsTailsBins(array_agg({{=it._column}}::numeric), {{=it._buckets}}) as headtails'
+};
+
+var methodTemplates = Object.keys(methods).reduce(function(methodTemplates, methodName) {
+    methodTemplates[methodName] = createTemplate(methods[methodName]);
+    return methodTemplates;
+}, {});
+
+methodTemplates.category = dot.template([
+    'WITH',
+    'categories AS (',
+    '  SELECT {{=it._column}} AS category, count(1) AS value, row_number() OVER (ORDER BY count(1) desc) as rank',
+    '  FROM ({{=it._sql}}) _cdb_aggregation_all',
+    '  GROUP BY {{=it._column}}',
+    '  ORDER BY 2 DESC, 1 ASC',
+    '),',
+    'agg_categories AS (',
+    '  SELECT category',
+    '  FROM categories',
+    '  WHERE rank <= {{=it._buckets}}',
+    ')',
+    'SELECT array_agg(category) AS category FROM agg_categories'
+].join('\n'));
+
+var STRATEGY = {
+    SPLIT: 'split',
+    EXACT: 'exact'
+};
+
+var method2strategy = {
+    headtails: STRATEGY.SPLIT,
+    category: STRATEGY.EXACT
+};
+
+function PostgresDatasource (psql, query) {
+    this.psql = psql;
+    this.query = query;
+}
+
+PostgresDatasource.prototype.getName = function () {
+    return 'PostgresDatasource';
+};
+
+PostgresDatasource.prototype.getRamp = function (column, buckets, method, callback) {
+    if (method && !methodTemplates.hasOwnProperty(method)) {
+        return callback(new Error(
+            'Invalid method "' + method + '", valid methods: [' + Object.keys(methodTemplates).join(',') + ']'
+        ));
+    }
+    var methodName = method || 'quantiles';
+    var template = methodTemplates[methodName];
+
+    var query = template({ _column: column, _sql: this.query, _buckets: buckets });
+
+    this.psql.query(query, function (err, resultSet) {
+        if (err) {
+            return callback(err);
+        }
+
+        var result = getResult(resultSet);
+        var strategy = method2strategy[methodName];
+        var ramp = result[methodName] || [];
+        var stats = {
+            min_val: result.min_val,
+            max_val: result.max_val,
+            avg_val: result.avg_val
+        };
+        // Skip null values from ramp
+        // Generated turbo-carto won't be correct, but better to keep it working than failing
+        // TODO fix cartodb-postgres extension quantification functions
+        ramp = ramp.filter(function(value) { return value !== null; });
+        if (strategy !== STRATEGY.EXACT) {
+            ramp = ramp.sort(function(a, b) {
+                return a - b;
+            });
+        }
+
+        return callback(null, { ramp: ramp, strategy: strategy, stats: stats });
+    }, true); // use read-only transaction
+};
+
+function getResult(resultSet) {
+    resultSet = resultSet || {};
+    var result = resultSet.rows || [];
+    result = result[0] || {};
+
+    return result;
+}
+
+module.exports = PostgresDatasource;

lib/cartodb/cache/named_map_provider_cache.js

@@ -1,24 +1,17 @@
 var _ = require('underscore');
 var dot = require('dot');
-var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider');
-var MapConfigNamedLayersAdapter = require('../models/mapconfig_named_layers_adapter');
-var AnalysisMapConfigAdapter = require('../models/analysis-mapconfig-adapter');
+var NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider');
 var templateName = require('../backends/template_maps').templateName;
 var queue = require('queue-async');
 
 var LruCache = require("lru-cache");
 
-function NamedMapProviderCache(templateMaps, pgConnection, metadataBackend, analysisBackend, userLimitsApi,
-                               overviewsAdapter, turboCartoAdapter) {
+function NamedMapProviderCache(templateMaps, pgConnection, metadataBackend, userLimitsApi, mapConfigAdapter) {
     this.templateMaps = templateMaps;
     this.pgConnection = pgConnection;
     this.metadataBackend = metadataBackend;
     this.userLimitsApi = userLimitsApi;
-
-    this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps);
-    this.analysisMapConfigAdapter = new AnalysisMapConfigAdapter(analysisBackend);
-    this.overviewsAdapter = overviewsAdapter;
-    this.turboCartoAdapter = turboCartoAdapter;
+    this.mapConfigAdapter = mapConfigAdapter;
 
     this.providerCache = new LruCache({ max: 2000 });
 }
@@ -36,10 +29,7 @@ NamedMapProviderCache.prototype.get = function(user, templateId, config, authTok
             this.pgConnection,
             this.metadataBackend,
             this.userLimitsApi,
-            this.namedLayersAdapter,
-            this.overviewsAdapter,
-            this.turboCartoAdapter,
-            this.analysisMapConfigAdapter,
+            this.mapConfigAdapter,
             user,
             templateId,
             config,

lib/cartodb/controllers/analyses.js

@@ -0,0 +1,167 @@
+var PSQL = require('cartodb-psql');
+var cors = require('../middleware/cors');
+var userMiddleware = require('../middleware/user');
+
+function AnalysesController(prepareContext) {
+    this.prepareContext = prepareContext;
+}
+
+module.exports = AnalysesController;
+
+AnalysesController.prototype.register = function (app) {
+    app.get(
+        `${app.base_url_mapconfig}/analyses/catalog`,
+        cors(),
+        userMiddleware,
+        this.prepareContext,
+        this.createPGClient(),
+        this.getDataFromQuery({ queryTemplate: catalogQueryTpl, key: 'catalog' }),
+        this.getDataFromQuery({ queryTemplate: tablesQueryTpl, key: 'tables' }),
+        this.prepareResponse(),
+        this.setCacheControlHeader(),
+        this.sendResponse(),
+        this.unathorizedError()
+    );
+};
+
+AnalysesController.prototype.createPGClient = function () {
+    return function createPGClientMiddleware (req, res, next) {
+        res.locals.pg = new PSQL(dbParamsFromReqParams(res.locals));
+        next();
+    };
+};
+
+AnalysesController.prototype.getDataFromQuery = function ({ queryTemplate, key }) {
+    const readOnlyTransactionOn = true;
+
+    return function getCatalogMiddleware(req, res, next) {
+        const { pg, user } = res.locals;
+        const sql = queryTemplate({ _username: user });
+
+        pg.query(sql, (err, resultSet = {}) => {
+            if (err) {
+                return next(err);
+            }
+
+            res.locals[key] = resultSet.rows || [];
+
+            next();
+        }, readOnlyTransactionOn);
+    };
+};
+
+AnalysesController.prototype.prepareResponse = function () {
+    return function prepareResponseMiddleware (req, res, next) {
+        const { catalog, tables } = res.locals;
+
+        const analysisIdToTable = tables.reduce((analysisIdToTable, table) => {
+            const analysisId = table.relname.split('_')[2];
+
+            if (analysisId && analysisId.length === 40) {
+                analysisIdToTable[analysisId] = table;
+            }
+
+            return analysisIdToTable;
+        }, {});
+
+        const analysisCatalog = catalog.map(analysis => {
+            if (analysisIdToTable.hasOwnProperty(analysis.node_id)) {
+                analysis.table = analysisIdToTable[analysis.node_id];
+            }
+
+            return analysis;
+        })
+        .sort((analysisA, analysisB) => {
+            if (!!analysisA.table && !!analysisB.table) {
+                return analysisB.table.size - analysisA.table.size;
+            }
+
+            if (!!analysisA.table) {
+                return -1;
+            }
+
+            if (!!analysisB.table) {
+                return 1;
+            }
+
+            return -1;
+        });
+
+        res.body = { catalog: analysisCatalog };
+
+        next();
+    };
+};
+
+AnalysesController.prototype.setCacheControlHeader = function () {
+    return function setCacheControlHeaderMiddleware (req, res, next) {
+        res.set('Cache-Control', 'public,max-age=10,must-revalidate');
+        next();
+    };
+};
+
+AnalysesController.prototype.sendResponse = function() {
+    return function sendResponseMiddleware (req, res) {
+        res.status(200);
+
+        if (req.query && req.query.callback) {
+            res.jsonp(res.body);
+        } else {
+            res.json(res.body);
+        }
+    };
+};
+
+AnalysesController.prototype.unathorizedError = function () {
+    return function unathorizedErrorMiddleware(err, req, res, next) {
+        if (err.message.match(/permission\sdenied/)) {
+            err = new Error('Unauthorized');
+            err.http_status = 401;
+        }
+
+        next(err);
+    };
+};
+
+const catalogQueryTpl = ctx => `
+    SELECT analysis_def->>'type' as type, * FROM cdb_analysis_catalog WHERE username = '${ctx._username}'
+`;
+
+var tablesQueryTpl = ctx => `
+    WITH analysis_tables AS (
+        SELECT
+            n.nspname AS nspname,
+            c.relname AS relname,
+            pg_total_relation_size(
+                format('%s.%s', pg_catalog.quote_ident(n.nspname), pg_catalog.quote_ident(c.relname))
+            ) AS size,
+            format('%s.%s', pg_catalog.quote_ident(nspname), pg_catalog.quote_ident(relname)) AS fully_qualified_name
+        FROM pg_catalog.pg_class c, pg_catalog.pg_namespace n
+        WHERE c.relnamespace = n.oid
+        AND pg_catalog.quote_ident(c.relname) ~ '^analysis_[a-z0-9]{10}_[a-z0-9]{40}$'
+        AND n.nspname IN ('${ctx._username}', 'public')
+    )
+    SELECT *, pg_size_pretty(size) as size_pretty
+    FROM analysis_tables
+    ORDER BY size DESC
+`;
+
+function dbParamsFromReqParams(params) {
+    var dbParams = {};
+    if ( params.dbuser ) {
+        dbParams.user = params.dbuser;
+    }
+    if ( params.dbpassword ) {
+        dbParams.pass = params.dbpassword;
+    }
+    if ( params.dbhost ) {
+        dbParams.host = params.dbhost;
+    }
+    if ( params.dbport ) {
+        dbParams.port = params.dbport;
+    }
+    if ( params.dbname ) {
+        dbParams.dbname = params.dbname;
+    }
+    return dbParams;
+}

lib/cartodb/controllers/base.js

@@ -1,269 +0,0 @@
-var assert = require('assert');
-
-var _ = require('underscore');
-var step = require('step');
-var debug = require('debug')('windshaft:cartodb');
-
-var LZMA = require('lzma').LZMA;
-var lzmaWorker = new LZMA();
-
-// Whitelist query parameters and attach format
-var REQUEST_QUERY_PARAMS_WHITELIST = [
-    'config',
-    'map_key',
-    'api_key',
-    'auth_token',
-    'callback',
-    'zoom',
-    'lon',
-    'lat',
-    // widgets & filters
-    'filters', // json
-    'own_filter', // 0, 1
-    'bbox', // w,s,e,n
-    'bins', // number
-    'start', // number
-    'end', // number
-    'column_type', // string
-    // widgets search
-    'q'
-];
-
-function BaseController(authApi, pgConnection) {
-    this.authApi = authApi;
-    this.pgConnection = pgConnection;
-}
-
-module.exports = BaseController;
-
-// jshint maxcomplexity:9
-/**
- * Whitelist input and get database name & default geometry type from
- * subdomain/user metadata held in CartoDB Redis
- * @param req - standard express request obj. Should have host & table
- * @param callback
- */
-BaseController.prototype.req2params = function(req, callback){
-    var self = this;
-
-    if ( req.query.lzma ) {
-
-        // Decode (from base64)
-        var lzma = new Buffer(req.query.lzma, 'base64')
-            .toString('binary')
-            .split('')
-            .map(function(c) {
-                return c.charCodeAt(0) - 128;
-            });
-
-
-        // Decompress
-        lzmaWorker.decompress(
-            lzma,
-            function(result) {
-                if (req.profiler) {
-                    req.profiler.done('lzma');
-                }
-                try {
-                    delete req.query.lzma;
-                    _.extend(req.query, JSON.parse(result));
-                    self.req2params(req, callback);
-                } catch (err) {
-                    req.profiler.done('req2params');
-                    callback(new Error('Error parsing lzma as JSON: ' + err));
-                }
-            }
-        );
-        return;
-    }
-
-    req.query = _.pick(req.query, REQUEST_QUERY_PARAMS_WHITELIST);
-    req.params = _.extend({}, req.params); // shuffle things as request is a strange array/object
-
-    var user = req.context.user;
-
-    if ( req.params.token ) {
-        // Token might match the following patterns:
-        // - {user}@{tpl_id}@{token}:{cache_buster}
-        var tksplit = req.params.token.split(':');
-        req.params.token = tksplit[0];
-        if ( tksplit.length > 1 ) {
-            req.params.cache_buster= tksplit[1];
-        }
-        tksplit = req.params.token.split('@');
-        if ( tksplit.length > 1 ) {
-            req.params.signer = tksplit.shift();
-            if ( ! req.params.signer ) {
-                req.params.signer = user;
-            }
-            else if ( req.params.signer !== user ) {
-                var err = new Error(
-                        'Cannot use map signature of user "' + req.params.signer + '" on db of user "' + user + '"'
-                );
-                err.http_status = 403;
-                req.profiler.done('req2params');
-                callback(err);
-                return;
-            }
-            if ( tksplit.length > 1 ) {
-                /*var template_hash = */tksplit.shift(); // unused
-            }
-            req.params.token = tksplit.shift();
-        }
-    }
-
-    // bring all query values onto req.params object
-    _.extend(req.params, req.query);
-
-    if (req.profiler) {
-        req.profiler.done('req2params.setup');
-    }
-
-    step(
-        function getPrivacy(){
-            self.authApi.authorize(req, this);
-        },
-        function validateAuthorization(err, authorized) {
-            if (req.profiler) {
-                req.profiler.done('authorize');
-            }
-            assert.ifError(err);
-            if(!authorized) {
-                err = new Error("Sorry, you are unauthorized (permission denied)");
-                err.http_status = 403;
-                throw err;
-            }
-            return null;
-        },
-        function getDatabase(err){
-            assert.ifError(err);
-            self.pgConnection.setDBConn(user, req.params, this);
-        },
-        function finishSetup(err) {
-            if ( err ) {
-                req.profiler.done('req2params');
-                return callback(err, req);
-            }
-
-            // Add default database connection parameters
-            // if none given
-            _.defaults(req.params, {
-                dbuser: global.environment.postgres.user,
-                dbpassword: global.environment.postgres.password,
-                dbhost: global.environment.postgres.host,
-                dbport: global.environment.postgres.port
-            });
-
-            req.profiler.done('req2params');
-            callback(null, req);
-        }
-    );
-};
-// jshint maxcomplexity:6
-
-// jshint maxcomplexity:9
-BaseController.prototype.send = function(req, res, body, status, headers) {
-    if (req.params.dbhost) {
-        res.set('X-Served-By-DB-Host', req.params.dbhost);
-    }
-
-    if (req.profiler) {
-        res.set('X-Tiler-Profiler', req.profiler.toJSONString());
-    }
-
-    if (headers) {
-        res.set(headers);
-    }
-
-    res.status(status);
-
-    if (!Buffer.isBuffer(body) && typeof body === 'object') {
-        if (req.query && req.query.callback) {
-            res.jsonp(body);
-        } else {
-            res.json(body);
-        }
-    } else {
-        res.send(body);
-    }
-
-    if (req.profiler) {
-        try {
-            // May throw due to dns, see
-            // See http://github.com/CartoDB/Windshaft/issues/166
-            req.profiler.sendStats();
-        } catch (err) {
-            debug("error sending profiling stats: " + err);
-        }
-    }
-};
-// jshint maxcomplexity:6
-
-BaseController.prototype.sendError = function(req, res, err, label) {
-    var allErrors = Array.isArray(err) ? err : [err];
-    label = label || 'UNKNOWN';
-    err = allErrors[0] || new Error(label);
-    allErrors[0] = err;
-
-    var statusCode = findStatusCode(err);
-
-    debug('[%s ERROR] -- %d: %s, %s', label, statusCode, err, err.stack);
-
-    // If a callback was requested, force status to 200
-    if (req.query && req.query.callback) {
-        statusCode = 200;
-    }
-
-    var errorResponseBody = { errors: allErrors.map(errorMessage) };
-
-    this.send(req, res, errorResponseBody, statusCode);
-};
-
-function errorMessage(err) {
-    // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
-    var message = (_.isString(err) ? err : err.message) || 'Unknown error';
-
-    // Strip connection info, if any
-    return message
-        // See https://github.com/CartoDB/Windshaft/issues/173
-        .replace(/Connection string: '[^']*'\n\s/im, '')
-        // See https://travis-ci.org/CartoDB/Windshaft/jobs/20703062#L1644
-        .replace(/is the server.*encountered/im, 'encountered');
-}
-module.exports.errorMessage = errorMessage;
-
-function findStatusCode(err) {
-    var statusCode;
-    if ( err.http_status ) {
-        statusCode = err.http_status;
-    } else {
-        statusCode = statusFromErrorMessage('' + err);
-    }
-    return statusCode;
-}
-module.exports.findStatusCode = findStatusCode;
-
-function statusFromErrorMessage(errMsg) {
-    // Find an appropriate statusCode based on message
-    // jshint maxcomplexity:7
-    var statusCode = 400;
-    if ( -1 !== errMsg.indexOf('permission denied') ) {
-        statusCode = 403;
-    }
-    else if ( -1 !== errMsg.indexOf('authentication failed') ) {
-        statusCode = 403;
-    }
-    else if (errMsg.match(/Postgis Plugin.*[\s|\n].*column.*does not exist/)) {
-        statusCode = 400;
-    }
-    else if ( -1 !== errMsg.indexOf('does not exist') ) {
-        if ( -1 !== errMsg.indexOf(' role ') ) {
-            statusCode = 403; // role 'xxx' does not exist
-        } else if ( errMsg.match(/function .* does not exist/) ) {
-            statusCode = 400; // invalid SQL (SQL function does not exist)
-        } else {
-            statusCode = 404;
-        }
-    }
-    return statusCode;
-}

lib/cartodb/controllers/index.js

@@ -1,4 +1,5 @@
 module.exports = {
+    Analyses: require('./analyses'),
     Layergroup: require('./layergroup'),
     Map: require('./map'),
     NamedMaps: require('./named_maps'),

lib/cartodb/controllers/layergroup.js

@@ -1,16 +1,15 @@
 var assert = require('assert');
 var step = require('step');
 
-var util = require('util');
-var BaseController = require('./base');
-
 var cors = require('../middleware/cors');
 var userMiddleware = require('../middleware/user');
+var allowQueryParams = require('../middleware/allow-query-params');
+var vectorError = require('../middleware/vector-error');
 
 var DataviewBackend = require('../backends/dataview');
 var AnalysisStatusBackend = require('../backends/analysis-status');
 
-var MapStoreMapConfigProvider = require('../models/mapconfig/map_store_provider');
+var MapStoreMapConfigProvider = require('../models/mapconfig/provider/map-store-provider');
 
 var QueryTables = require('cartodb-query-tables');
 
@@ -21,100 +20,162 @@ var QueryTables = require('cartodb-query-tables');
  * @param {TileBackend} tileBackend
  * @param {PreviewBackend} previewBackend
  * @param {AttributesBackend} attributesBackend
- * @param {WidgetBackend} widgetBackend
  * @param {SurrogateKeysCache} surrogateKeysCache
  * @param {UserLimitsApi} userLimitsApi
  * @param {LayergroupAffectedTables} layergroupAffectedTables
  * @param {AnalysisBackend} analysisBackend
  * @constructor
  */
-function LayergroupController(authApi, pgConnection, mapStore, tileBackend, previewBackend, attributesBackend,
-                          widgetBackend, surrogateKeysCache, userLimitsApi, layergroupAffectedTables, analysisBackend) {
-    BaseController.call(this, authApi, pgConnection);
-
+function LayergroupController(prepareContext, pgConnection, mapStore, tileBackend, previewBackend, attributesBackend,
+                              surrogateKeysCache, userLimitsApi, layergroupAffectedTables, analysisBackend) {
     this.pgConnection = pgConnection;
     this.mapStore = mapStore;
     this.tileBackend = tileBackend;
     this.previewBackend = previewBackend;
     this.attributesBackend = attributesBackend;
-    this.widgetBackend = widgetBackend;
     this.surrogateKeysCache = surrogateKeysCache;
     this.userLimitsApi = userLimitsApi;
     this.layergroupAffectedTables = layergroupAffectedTables;
 
     this.dataviewBackend = new DataviewBackend(analysisBackend);
     this.analysisStatusBackend = new AnalysisStatusBackend();
-}
 
-util.inherits(LayergroupController, BaseController);
+    this.prepareContext = prepareContext;
+}
 
 module.exports = LayergroupController;
 
-
 LayergroupController.prototype.register = function(app) {
-    app.get(app.base_url_mapconfig +
-        '/:token/:z/:x/:y@:scale_factor?x.:format', cors(), userMiddleware,
-        this.tile.bind(this));
+    app.get(
+        app.base_url_mapconfig + '/:token/:z/:x/:y@:scale_factor?x.:format',
+        cors(),
+        userMiddleware,
+        this.prepareContext,
+        this.tile.bind(this),
+        vectorError()
+    );
 
-    app.get(app.base_url_mapconfig +
-        '/:token/:z/:x/:y.:format', cors(), userMiddleware,
-        this.tile.bind(this));
+    app.get(
+        app.base_url_mapconfig + '/:token/:z/:x/:y.:format',
+        cors(),
+        userMiddleware,
+        this.prepareContext,
+        this.tile.bind(this),
+        vectorError()
+    );
 
-    app.get(app.base_url_mapconfig +
-        '/:token/:layer/:z/:x/:y.(:format)', cors(), userMiddleware,
-        this.layer.bind(this));
+    app.get(
+        app.base_url_mapconfig + '/:token/:layer/:z/:x/:y.(:format)',
+        cors(),
+        userMiddleware,
+        validateLayerRouteMiddleware,
+        this.prepareContext,
+        this.layer.bind(this),
+        vectorError()
+    );
 
-    app.get(app.base_url_mapconfig +
-        '/:token/:layer/attributes/:fid', cors(), userMiddleware,
-        this.attributes.bind(this));
+    app.get(
+        app.base_url_mapconfig + '/:token/:layer/attributes/:fid',
+        cors(),
+        userMiddleware,
+        this.prepareContext,
+        this.attributes.bind(this)
+    );
 
-    app.get(app.base_url_mapconfig +
-        '/static/center/:token/:z/:lat/:lng/:width/:height.:format', cors(), userMiddleware,
-        this.center.bind(this));
+    app.get(
+        app.base_url_mapconfig + '/static/center/:token/:z/:lat/:lng/:width/:height.:format',
+        cors(),
+        userMiddleware,
+        allowQueryParams(['layer']),
+        this.prepareContext,
+        this.center.bind(this)
+    );
 
-    app.get(app.base_url_mapconfig +
-        '/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format', cors(), userMiddleware,
-        this.bbox.bind(this));
+    app.get(
+        app.base_url_mapconfig + '/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format',
+        cors(),
+        userMiddleware,
+        allowQueryParams(['layer']),
+        this.prepareContext,
+        this.bbox.bind(this)
+    );
 
     // Undocumented/non-supported API endpoint methods.
     // Use at your own peril.
-    app.get(app.base_url_mapconfig +
-        '/:token/:layer/widget/:widgetName', cors(), userMiddleware,
-        this.widget.bind(this));
 
-    app.get(app.base_url_mapconfig +
-            '/:token/:layer/widget/:widgetName/search', cors(), userMiddleware,
-        this.widgetSearch.bind(this));
+    var allowedDataviewQueryParams = [
+        'filters', // json
+        'own_filter', // 0, 1
+        'no_filters', // 0, 1
+        'bbox', // w,s,e,n
+        'start', // number
+        'end', // number
+        'column_type', // string
+        'bins', // number
+        'aggregation', //string
+        'offset', // number
+        'q', // widgets search
+        'categories', // number
+    ];
 
-    app.get(app.base_url_mapconfig +
-            '/:token/dataview/:dataviewName', cors(), userMiddleware,
-        this.dataview.bind(this));
+    app.get(
+        app.base_url_mapconfig + '/:token/dataview/:dataviewName',
+        cors(),
+        userMiddleware,
+        allowQueryParams(allowedDataviewQueryParams),
+        this.prepareContext,
+        this.dataview.bind(this)
+    );
 
-    app.get(app.base_url_mapconfig +
-            '/:token/dataview/:dataviewName/search', cors(), userMiddleware,
-        this.dataviewSearch.bind(this));
+    app.get(
+        app.base_url_mapconfig + '/:token/:layer/widget/:dataviewName',
+        cors(),
+        userMiddleware,
+        allowQueryParams(allowedDataviewQueryParams),
+        this.prepareContext,
+        this.dataview.bind(this)
+    );
 
-    app.get(app.base_url_mapconfig +
-        '/:token/analysis/node/:nodeId', cors(), userMiddleware,
-        this.analysisNodeStatus.bind(this));
+    app.get(
+        app.base_url_mapconfig + '/:token/dataview/:dataviewName/search',
+        cors(),
+        userMiddleware,
+        allowQueryParams(allowedDataviewQueryParams),
+        this.prepareContext,
+        this.dataviewSearch.bind(this)
+    );
+
+    app.get(
+        app.base_url_mapconfig + '/:token/:layer/widget/:dataviewName/search',
+        cors(),
+        userMiddleware,
+        allowQueryParams(allowedDataviewQueryParams),
+        this.prepareContext,
+        this.dataviewSearch.bind(this)
+    );
+
+    app.get(
+        app.base_url_mapconfig + '/:token/analysis/node/:nodeId',
+        cors(),
+        userMiddleware,
+        this.prepareContext,
+        this.analysisNodeStatus.bind(this)
+    );
 };
 
-LayergroupController.prototype.analysisNodeStatus = function(req, res) {
+LayergroupController.prototype.analysisNodeStatus = function(req, res, next) {
     var self = this;
 
     step(
-        function setupParams() {
-            self.req2params(req, this);
-        },
-        function retrieveNodeStatus(err) {
-            assert.ifError(err);
-            self.analysisStatusBackend.getNodeStatus(req.params, this);
+        function retrieveNodeStatus() {
+            self.analysisStatusBackend.getNodeStatus(res.locals, this);
         },
         function finish(err, nodeStatus, stats) {
             req.profiler.add(stats || {});
 
             if (err) {
-                self.sendError(req, res, err, 'GET NODE STATUS');
+                err.label = 'GET NODE STATUS';
+                next(err);
             } else {
                 self.sendResponse(req, res, nodeStatus, 200, {
                     'Cache-Control': 'public,max-age=5',
@@ -125,54 +186,50 @@ LayergroupController.prototype.analysisNodeStatus = function(req, res) {
     );
 };
 
-LayergroupController.prototype.dataview = function(req, res) {
+LayergroupController.prototype.dataview = function(req, res, next) {
     var self = this;
 
     step(
-        function setupParams() {
-            self.req2params(req, this);
-        },
-        function retrieveDataview(err) {
-            assert.ifError(err);
-
+        function retrieveDataview() {
             var mapConfigProvider = new MapStoreMapConfigProvider(
-                self.mapStore, req.context.user, self.userLimitsApi, req.params
+                self.mapStore, res.locals.user, self.userLimitsApi, res.locals
+            );
+            self.dataviewBackend.getDataview(
+                mapConfigProvider,
+                res.locals.user,
+                res.locals,
+                this
             );
-            self.dataviewBackend.getDataview(mapConfigProvider, req.context.user, req.params, this);
         },
         function finish(err, dataview, stats) {
             req.profiler.add(stats || {});
 
             if (err) {
-                self.sendError(req, res, err, 'GET DATAVIEW');
+                err.label = 'GET DATAVIEW';
+                next(err);
             } else {
                 self.sendResponse(req, res, dataview, 200);
             }
         }
     );
-
 };
 
-LayergroupController.prototype.dataviewSearch = function(req, res) {
+LayergroupController.prototype.dataviewSearch = function(req, res, next) {
     var self = this;
 
     step(
-        function setupParams() {
-            self.req2params(req, this);
-        },
-        function searchDataview(err) {
-            assert.ifError(err);
-
+        function searchDataview() {
             var mapConfigProvider = new MapStoreMapConfigProvider(
-                self.mapStore, req.context.user, self.userLimitsApi, req.params
+                self.mapStore, res.locals.user, self.userLimitsApi, res.locals
             );
-            self.dataviewBackend.search(mapConfigProvider, req.context.user, req.params, this);
+            self.dataviewBackend.search(mapConfigProvider, res.locals.user, req.params.dataviewName, res.locals, this);
         },
         function finish(err, searchResult, stats) {
             req.profiler.add(stats || {});
 
             if (err) {
-                self.sendError(req, res, err, 'GET DATAVIEW SEARCH');
+                err.label = 'GET DATAVIEW SEARCH';
+                next(err);
             } else {
                 self.sendResponse(req, res, searchResult, 200);
             }
@@ -181,84 +238,24 @@ LayergroupController.prototype.dataviewSearch = function(req, res) {
 
 };
 
-LayergroupController.prototype.widget = function(req, res) {
-    var self = this;
-
-    step(
-        function setupParams() {
-            self.req2params(req, this);
-        },
-        function retrieveList(err) {
-            assert.ifError(err);
-
-            var mapConfigProvider = new MapStoreMapConfigProvider(
-                self.mapStore, req.context.user, self.userLimitsApi, req.params
-            );
-            self.widgetBackend.getWidget(mapConfigProvider, req.params, this);
-        },
-        function finish(err, widget, stats) {
-            req.profiler.add(stats || {});
-
-            if (err) {
-                self.sendError(req, res, err, 'GET WIDGET');
-            } else {
-                self.sendResponse(req, res, widget, 200);
-            }
-        }
-    );
-
-};
-
-LayergroupController.prototype.widgetSearch = function(req, res) {
-    var self = this;
-
-    step(
-        function setupParams() {
-            self.req2params(req, this);
-        },
-        function retrieveList(err) {
-            assert.ifError(err);
-
-            var mapConfigProvider = new MapStoreMapConfigProvider(
-                self.mapStore, req.context.user, self.userLimitsApi, req.params
-            );
-            self.widgetBackend.search(mapConfigProvider, req.params, this);
-        },
-        function finish(err, searchResult, stats) {
-            req.profiler.add(stats || {});
-
-            if (err) {
-                self.sendError(req, res, err, 'GET WIDGET');
-            } else {
-                self.sendResponse(req, res, searchResult, 200);
-            }
-        }
-    );
-
-};
-
-LayergroupController.prototype.attributes = function(req, res) {
+LayergroupController.prototype.attributes = function(req, res, next) {
     var self = this;
 
     req.profiler.start('windshaft.maplayer_attribute');
 
     step(
-        function setupParams() {
-            self.req2params(req, this);
-        },
-        function retrieveFeatureAttributes(err) {
-            assert.ifError(err);
-
+        function retrieveFeatureAttributes() {
             var mapConfigProvider = new MapStoreMapConfigProvider(
-                self.mapStore, req.context.user, self.userLimitsApi, req.params
+                self.mapStore, res.locals.user, self.userLimitsApi, res.locals
             );
-            self.attributesBackend.getFeatureAttributes(mapConfigProvider, req.params, false, this);
+            self.attributesBackend.getFeatureAttributes(mapConfigProvider, res.locals, false, this);
         },
         function finish(err, tile, stats) {
             req.profiler.add(stats || {});
 
             if (err) {
-                self.sendError(req, res, err, 'GET ATTRIBUTES');
+                err.label = 'GET ATTRIBUTES';
+                next(err);
             } else {
                 self.sendResponse(req, res, tile, 200);
             }
@@ -268,49 +265,48 @@ LayergroupController.prototype.attributes = function(req, res) {
 };
 
 // Gets a tile for a given token and set of tile ZXY coords. (OSM style)
-LayergroupController.prototype.tile = function(req, res) {
+LayergroupController.prototype.tile = function(req, res, next) {
     req.profiler.start('windshaft.map_tile');
-    this.tileOrLayer(req, res);
+    this.tileOrLayer(req, res, next);
 };
 
 // Gets a tile for a given token, layer set of tile ZXY coords. (OSM style)
 LayergroupController.prototype.layer = function(req, res, next) {
-    if (req.params.token === 'static') {
-        return next();
-    }
     req.profiler.start('windshaft.maplayer_tile');
-    this.tileOrLayer(req, res);
+    this.tileOrLayer(req, res, next);
 };
 
-LayergroupController.prototype.tileOrLayer = function (req, res) {
+LayergroupController.prototype.tileOrLayer = function (req, res, next) {
     var self = this;
 
     step(
-        function mapController$prepareParams() {
-            self.req2params(req, this);
-        },
-        function mapController$getTileOrGrid(err) {
-            assert.ifError(err);
+        function mapController$getTileOrGrid() {
             self.tileBackend.getTile(
-                new MapStoreMapConfigProvider(self.mapStore, req.context.user, self.userLimitsApi, req.params),
+                new MapStoreMapConfigProvider(self.mapStore, res.locals.user, self.userLimitsApi, res.locals),
                 req.params, this
             );
         },
         function mapController$finalize(err, tile, headers, stats) {
             req.profiler.add(stats);
-            self.finalizeGetTileOrGrid(err, req, res, tile, headers);
+            self.finalizeGetTileOrGrid(err, req, res, tile, headers, next);
         }
     );
 };
 
+function getStatusCode(tile, format){
+    return tile.length===0 && format==='mvt'? 204:200;
+}
+
 // This function is meant for being called as the very last
 // step by all endpoints serving tiles or grids
-LayergroupController.prototype.finalizeGetTileOrGrid = function(err, req, res, tile, headers) {
+LayergroupController.prototype.finalizeGetTileOrGrid = function(err, req, res, tile, headers, next) {
     var supportedFormats = {
         grid_json: true,
         json_torque: true,
         torque_json: true,
-        png: true
+        png: true,
+        png32: true,
+        mvt: true
     };
 
     var formatStat = 'invalid';
@@ -332,52 +328,52 @@ LayergroupController.prototype.finalizeGetTileOrGrid = function(err, req, res, t
         }
         err.message = errMsg;
 
-        this.sendError(req, res, err, 'TILE RENDER');
+        err.label = 'TILE RENDER';
+        next(err);
+
         global.statsClient.increment('windshaft.tiles.error');
         global.statsClient.increment('windshaft.tiles.' + formatStat + '.error');
     } else {
-        this.sendResponse(req, res, tile, 200, headers);
+        this.sendResponse(req, res, tile, getStatusCode(tile, formatStat), headers);
         global.statsClient.increment('windshaft.tiles.success');
         global.statsClient.increment('windshaft.tiles.' + formatStat + '.success');
     }
 };
 
-LayergroupController.prototype.bbox = function(req, res) {
+LayergroupController.prototype.bbox = function(req, res, next) {
     this.staticMap(req, res, +req.params.width, +req.params.height, {
         west: +req.params.west,
         north: +req.params.north,
         east: +req.params.east,
         south: +req.params.south
-    });
+    }, null, next);
 };
 
-LayergroupController.prototype.center = function(req, res) {
+LayergroupController.prototype.center = function(req, res, next) {
     this.staticMap(req, res, +req.params.width, +req.params.height, +req.params.z, {
         lng: +req.params.lng,
         lat: +req.params.lat
-    });
+    }, next);
 };
 
-LayergroupController.prototype.staticMap = function(req, res, width, height, zoom /* bounds */, center) {
+LayergroupController.prototype.staticMap = function(req, res, width, height, zoom /* bounds */, center, next) {
     var format = req.params.format === 'jpg' ? 'jpeg' : 'png';
-    req.params.layer = 'all';
-    req.params.format = 'png';
+    // We force always the tile to be generated using PNG because
+    // is the only format we support by now
+    res.locals.format = 'png';
+    res.locals.layer = res.locals.layer || 'all';
 
     var self = this;
 
     step(
-        function reqParams() {
-            self.req2params(req, this);
-        },
-        function getImage(err) {
-            assert.ifError(err);
+        function getImage() {
             if (center) {
                 self.previewBackend.getImage(
-                    new MapStoreMapConfigProvider(self.mapStore, req.context.user, self.userLimitsApi, req.params),
+                    new MapStoreMapConfigProvider(self.mapStore, res.locals.user, self.userLimitsApi, res.locals),
                     format, width, height, zoom, center, this);
             } else {
                 self.previewBackend.getImage(
-                    new MapStoreMapConfigProvider(self.mapStore, req.context.user, self.userLimitsApi, req.params),
+                    new MapStoreMapConfigProvider(self.mapStore, res.locals.user, self.userLimitsApi, res.locals),
                     format, width, height, zoom /* bounds */, this);
             }
         },
@@ -386,7 +382,8 @@ LayergroupController.prototype.staticMap = function(req, res, width, height, zoo
             req.profiler.add(stats || {});
 
             if (err) {
-                self.sendError(req, res, err, 'STATIC_MAP');
+                err.label = 'STATIC_MAP';
+                next(err);
             } else {
                 res.set('Content-Type', headers['Content-Type'] || 'image/' + format);
                 self.sendResponse(req, res, image, 200);
@@ -398,22 +395,24 @@ LayergroupController.prototype.staticMap = function(req, res, width, height, zoo
 LayergroupController.prototype.sendResponse = function(req, res, body, status, headers) {
     var self = this;
 
+    req.profiler.done('res');
+
     res.set('Cache-Control', 'public,max-age=31536000');
 
     // Set Last-Modified header
     var lastUpdated;
-    if (req.params.cache_buster) {
+    if (res.locals.cache_buster) {
         // Assuming cache_buster is a timestamp
-        lastUpdated = new Date(parseInt(req.params.cache_buster));
+        lastUpdated = new Date(parseInt(res.locals.cache_buster));
     } else {
         lastUpdated = new Date();
     }
     res.set('Last-Modified', lastUpdated.toUTCString());
 
-    var dbName = req.params.dbname;
+    var dbName = res.locals.dbname;
     step(
         function getAffectedTables() {
-            self.getAffectedTables(req.context.user, dbName, req.params.token, this);
+            self.getAffectedTables(res.locals.user, dbName, res.locals.token, this);
         },
         function sendResponse(err, affectedTables) {
             req.profiler.done('affectedTables');
@@ -424,10 +423,24 @@ LayergroupController.prototype.sendResponse = function(req, res, body, status, h
                 res.set('X-Cache-Channel', affectedTables.getCacheChannel());
                 self.surrogateKeysCache.tag(res, affectedTables);
             }
-            self.send(req, res, body, status, headers);
+
+            if (headers) {
+                res.set(headers);
+            }
+
+            res.status(status);
+
+            if (!Buffer.isBuffer(body) && typeof body === 'object') {
+                if (req.query && req.query.callback) {
+                    res.jsonp(body);
+                } else {
+                    res.json(body);
+                }
+            } else {
+                res.send(body);
+            }
         }
     );
-
 };
 
 LayergroupController.prototype.getAffectedTables = function(user, dbName, layergroupId, callback) {
@@ -446,13 +459,15 @@ LayergroupController.prototype.getAffectedTables = function(user, dbName, layerg
                 function getSQL(err, mapConfig) {
                     assert.ifError(err);
 
-                    var queries = mapConfig.getLayers()
-                        .map(function(lyr) {
-                            return lyr.options.sql;
-                        })
-                        .filter(function(sql) {
-                            return !!sql;
-                        });
+                    var queries = [];
+                    mapConfig.getLayers().forEach(function(layer) {
+                        queries.push(layer.options.sql);
+                        if (layer.options.affected_tables) {
+                            layer.options.affected_tables.map(function(table) {
+                                queries.push('SELECT * FROM ' + table + ' LIMIT 0');
+                            });
+                        }
+                    });
 
                     return queries.length ? queries.join(';') : null;
                 },
@@ -487,3 +502,12 @@ LayergroupController.prototype.getAffectedTables = function(user, dbName, layerg
         callback
     );
 };
+
+
+function validateLayerRouteMiddleware(req, res, next) {
+    if (req.params.token === 'static') {
+        return next('route');
+    }
+
+    next();
+}

lib/cartodb/controllers/named_maps.js

@@ -1,190 +1,10 @@
-var step = require('step');
-var assert = require('assert');
-var _ = require('underscore');
-var NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
+const NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
+const cors = require('../middleware/cors');
+const userMiddleware = require('../middleware/user');
+const allowQueryParams = require('../middleware/allow-query-params');
+const vectorError = require('../middleware/vector-error');
 
-var util = require('util');
-var BaseController = require('./base');
-
-var cors = require('../middleware/cors');
-var userMiddleware = require('../middleware/user');
-
-function NamedMapsController(authApi, pgConnection, namedMapProviderCache, tileBackend, previewBackend,
-                             surrogateKeysCache, tablesExtentApi, metadataBackend) {
-    BaseController.call(this, authApi, pgConnection);
-
-    this.namedMapProviderCache = namedMapProviderCache;
-    this.tileBackend = tileBackend;
-    this.previewBackend = previewBackend;
-    this.surrogateKeysCache = surrogateKeysCache;
-    this.tablesExtentApi = tablesExtentApi;
-    this.metadataBackend = metadataBackend;
-}
-
-util.inherits(NamedMapsController, BaseController);
-
-module.exports = NamedMapsController;
-
-NamedMapsController.prototype.register = function(app) {
-    app.get(app.base_url_templated +
-        '/:template_id/:layer/:z/:x/:y.(:format)', cors(), userMiddleware,
-        this.tile.bind(this));
-
-    app.get(app.base_url_mapconfig +
-        '/static/named/:template_id/:width/:height.:format', cors(), userMiddleware,
-        this.staticMap.bind(this));
-};
-
-NamedMapsController.prototype.sendResponse = function(req, res, resource, headers, namedMapProvider) {
-    this.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(req.context.user, namedMapProvider.getTemplateName()));
-    res.set('Content-Type', headers['content-type'] || headers['Content-Type'] || 'image/png');
-    res.set('Cache-Control', 'public,max-age=7200,must-revalidate');
-
-    var self = this;
-
-    step(
-        function getAffectedTablesAndLastUpdatedTime() {
-            namedMapProvider.getAffectedTablesAndLastUpdatedTime(this);
-        },
-        function sendResponse(err, result) {
-            req.profiler.done('affectedTables');
-            if (err) {
-                global.logger.log('ERROR generating cache channel: ' + err);
-            }
-            if (!result || !!result.tables) {
-                // we increase cache control as we can invalidate it
-                res.set('Cache-Control', 'public,max-age=31536000');
-
-                var lastModifiedDate;
-                if (Number.isFinite(result.lastUpdatedTime)) {
-                    lastModifiedDate = new Date(result.getLastUpdatedAt());
-                } else {
-                    lastModifiedDate = new Date();
-                }
-                res.set('Last-Modified', lastModifiedDate.toUTCString());
-
-                res.set('X-Cache-Channel', result.getCacheChannel());
-                if (result.tables.length > 0) {
-                    self.surrogateKeysCache.tag(res, result);
-                }
-            }
-            self.send(req, res, resource, 200);
-        }
-    );
-};
-
-NamedMapsController.prototype.tile = function(req, res) {
-    var self = this;
-
-    var cdbUser = req.context.user;
-
-    var namedMapProvider;
-    step(
-        function reqParams() {
-            self.req2params(req, this);
-        },
-        function getNamedMapProvider(err) {
-            assert.ifError(err);
-            self.namedMapProviderCache.get(
-                cdbUser,
-                req.params.template_id,
-                req.query.config,
-                req.query.auth_token,
-                req.params,
-                this
-            );
-        },
-        function getTile(err, _namedMapProvider) {
-            assert.ifError(err);
-            namedMapProvider = _namedMapProvider;
-            self.tileBackend.getTile(namedMapProvider, req.params, this);
-        },
-        function handleImage(err, tile, headers, stats) {
-            if (req.profiler) {
-                req.profiler.add(stats);
-            }
-            if (err) {
-                self.sendError(req, res, err, 'NAMED_MAP_TILE');
-            } else {
-                self.sendResponse(req, res, tile, headers, namedMapProvider);
-            }
-        }
-    );
-};
-
-NamedMapsController.prototype.staticMap = function(req, res) {
-    var self = this;
-
-    var cdbUser = req.context.user;
-
-    var format = req.params.format === 'jpg' ? 'jpeg' : 'png';
-    req.params.format = 'png';
-    req.params.layer = 'all';
-
-    var namedMapProvider;
-    step(
-        function reqParams() {
-            self.req2params(req, this);
-        },
-        function getNamedMapProvider(err) {
-            assert.ifError(err);
-            self.namedMapProviderCache.get(
-                cdbUser,
-                req.params.template_id,
-                req.query.config,
-                req.query.auth_token,
-                req.params,
-                this
-            );
-        },
-        function prepareImageOptions(err, _namedMapProvider) {
-            assert.ifError(err);
-            namedMapProvider = _namedMapProvider;
-            self.getStaticImageOptions(cdbUser, req.params, namedMapProvider, this);
-        },
-        function getImage(err, imageOpts) {
-            assert.ifError(err);
-
-            var width = +req.params.width;
-            var height = +req.params.height;
-
-            if (!_.isUndefined(imageOpts.zoom) && imageOpts.center) {
-                self.previewBackend.getImage(
-                    namedMapProvider, format, width, height, imageOpts.zoom, imageOpts.center, this);
-            } else {
-                self.previewBackend.getImage(
-                    namedMapProvider, format, width, height, imageOpts.bounds, this);
-            }
-        },
-        function incrementMapViews(err, image, headers, stats) {
-            assert.ifError(err);
-
-            var next = this;
-            namedMapProvider.getMapConfig(function(mapConfigErr, mapConfig) {
-                self.metadataBackend.incMapviewCount(cdbUser, mapConfig.obj().stat_tag, function(sErr) {
-                    if (err) {
-                        global.logger.log("ERROR: failed to increment mapview count for user '%s': %s", cdbUser, sErr);
-                    }
-                    next(err, image, headers, stats);
-                });
-            });
-        },
-        function handleImage(err, image, headers, stats) {
-            if (req.profiler) {
-                req.profiler.done('render-' + format);
-                req.profiler.add(stats || {});
-            }
-
-            if (err) {
-                self.sendError(req, res, err, 'STATIC_VIZ_MAP');
-            } else {
-                self.sendResponse(req, res, image, headers, namedMapProvider);
-            }
-        }
-    );
-};
-
-var DEFAULT_ZOOM_CENTER = {
+const DEFAULT_ZOOM_CENTER = {
     zoom: 1,
     center: {
         lng: 0,
@@ -196,88 +16,368 @@ function numMapper(n) {
     return +n;
 }
 
-NamedMapsController.prototype.getStaticImageOptions = function(cdbUser, params, namedMapProvider, callback) {
-    var self = this;
+function getRequestParams(locals) {
+    const params = Object.assign({}, locals);
 
-    if ([params.zoom, params.lon, params.lat].map(numMapper).every(Number.isFinite)) {
-        return callback(null, {
-            zoom: params.zoom,
-            center: {
-                lng: params.lon,
-                lat: params.lat
-            }
-        });
-    }
+    delete params.template;
+    delete params.affectedTablesAndLastUpdate;
+    delete params.namedMapProvider;
+    delete params.allowedQueryParams;
 
-    if (params.bbox) {
-        var bbox = params.bbox.split(',').map(numMapper);
-        if (bbox.length === 4 && bbox.every(Number.isFinite)) {
-            return callback(null, {
-                bounds: {
-                    west: bbox[0],
-                    south: bbox[1],
-                    east: bbox[2],
-                    north: bbox[3]
-                }
-            });
-        }
-    }
+    return params;
+}
 
-    step(
-        function getTemplate() {
-            namedMapProvider.getTemplate(this);
-        },
-        function handleTemplateView(err, template) {
-            assert.ifError(err);
+function NamedMapsController(prepareContext, namedMapProviderCache, tileBackend, previewBackend,
+                             surrogateKeysCache, tablesExtentApi, metadataBackend) {
+    this.namedMapProviderCache = namedMapProviderCache;
+    this.tileBackend = tileBackend;
+    this.previewBackend = previewBackend;
+    this.surrogateKeysCache = surrogateKeysCache;
+    this.tablesExtentApi = tablesExtentApi;
+    this.metadataBackend = metadataBackend;
+    this.prepareContext = prepareContext;
+}
 
-            if (template.view) {
-                var zoomCenter = templateZoomCenter(template.view);
-                if (zoomCenter) {
-                    if (Number.isFinite(+params.zoom)) {
-                        zoomCenter.zoom = +params.zoom;
-                    }
-                    return zoomCenter;
-                }
+module.exports = NamedMapsController;
 
-                var bounds = templateBounds(template.view);
-                if (bounds) {
-                    return bounds;
-                }
-            }
+NamedMapsController.prototype.register = function(app) {
+    app.get(
+        app.base_url_templated + '/:template_id/:layer/:z/:x/:y.(:format)',
+        cors(),
+        userMiddleware,
+        this.prepareContext,
+        this.getNamedMapProvider('NAMED_MAP_TILE'),
+        this.getAffectedTables(),
+        this.getTile('NAMED_MAP_TILE'),
+        this.setSurrogateKey(),
+        this.setCacheChannelHeader(),
+        this.setLastModifiedHeader(),
+        this.setCacheControlHeader(),
+        this.setContentTypeHeader(),
+        this.respond(),
+        vectorError()
+    );
 
-            return false;
-        },
-        function estimateBoundsIfNoImageOpts(err, imageOpts) {
-            if (imageOpts) {
-                return imageOpts;
-            }
-
-            var next = this;
-            namedMapProvider.getAffectedTablesAndLastUpdatedTime(function(err, affectedTablesAndLastUpdate) {
-                if (err) {
-                    return next(null);
-                }
-
-                var affectedTables = affectedTablesAndLastUpdate.tables || [];
-
-                if (affectedTables.length === 0) {
-                    return next(null);
-                }
-
-                self.tablesExtentApi.getBounds(cdbUser, affectedTables, function(err, result) {
-                    return next(null, result);
-                });
-            });
-
-        },
-        function returnCallback(err, imageOpts) {
-            return callback(err, imageOpts || DEFAULT_ZOOM_CENTER);
-        }
+    app.get(
+        app.base_url_mapconfig + '/static/named/:template_id/:width/:height.:format',
+        cors(),
+        userMiddleware,
+        allowQueryParams(['layer', 'zoom', 'lon', 'lat', 'bbox']),
+        this.prepareContext,
+        this.getNamedMapProvider('STATIC_VIZ_MAP'),
+        this.getAffectedTables(),
+        this.getTemplate('STATIC_VIZ_MAP'),
+        this.prepareLayerFilterFromPreviewLayers('STATIC_VIZ_MAP'),
+        this.getStaticImageOptions(),
+        this.getImage('STATIC_VIZ_MAP'),
+        this.incrementMapViews(),
+        this.setSurrogateKey(),
+        this.setCacheChannelHeader(),
+        this.setLastModifiedHeader(),
+        this.setCacheControlHeader(),
+        this.setContentTypeHeader(),
+        this.respond()
     );
 };
 
+NamedMapsController.prototype.getNamedMapProvider = function (label) {
+    return function getNamedMapProviderMiddleware (req, res, next) {
+        const { user } = res.locals;
+        const { config, auth_token } = req.query;
+        const { template_id } = req.params;
+
+        // We force always the tile to be generated using PNG because
+        // is the only format we support by now
+        res.locals.format = 'png';
+        res.locals.layer = res.locals.layer || 'all';
+
+        const params = getRequestParams(res.locals);
+
+        this.namedMapProviderCache.get(user, template_id, config, auth_token, params, (err, namedMapProvider) => {
+            if (err) {
+                err.label = label;
+                return next(err);
+            }
+
+            res.locals.namedMapProvider = namedMapProvider;
+
+            next();
+        });
+    }.bind(this);
+};
+
+NamedMapsController.prototype.getAffectedTables = function () {
+    return function getAffectedTables (req, res, next) {
+        const { namedMapProvider } = res.locals;
+
+        namedMapProvider.getAffectedTablesAndLastUpdatedTime((err, affectedTablesAndLastUpdate) => {
+            req.profiler.done('affectedTables');
+
+            if (err) {
+                return next(err);
+            }
+
+            res.locals.affectedTablesAndLastUpdate = affectedTablesAndLastUpdate;
+
+            next();
+        });
+    }.bind(this);
+};
+
+NamedMapsController.prototype.getTemplate = function (label) {
+    return function getTemplateMiddleware (req, res, next) {
+        const { namedMapProvider } = res.locals;
+
+        namedMapProvider.getTemplate((err, template) => {
+            if (err) {
+                err.label = label;
+                return next(err);
+            }
+
+            res.locals.template = template;
+
+            next();
+        });
+    };
+};
+
+NamedMapsController.prototype.prepareLayerFilterFromPreviewLayers = function (label) {
+    return function prepareLayerFilterFromPreviewLayersMiddleware (req, res, next) {
+        const { user, template } = res.locals;
+        const { template_id } = req.params;
+        const { config, auth_token } = req.query;
+
+        if (!template || !template.view || !template.view.preview_layers) {
+            return next();
+        }
+
+        var previewLayers = template.view.preview_layers;
+        var layerVisibilityFilter = [];
+
+        template.layergroup.layers.forEach((layer, index) => {
+            if (previewLayers[''+index] !== false && previewLayers[layer.id] !== false) {
+                layerVisibilityFilter.push(''+index);
+            }
+        });
+
+        if (!layerVisibilityFilter.length) {
+            return next();
+        }
+
+        const params = getRequestParams(res.locals);
+
+        // overwrites 'all' default filter
+        params.layer = layerVisibilityFilter.join(',');
+
+        // recreates the provider
+        this.namedMapProviderCache.get(user, template_id, config, auth_token, params, (err, provider) => {
+            if (err) {
+                err.label = label;
+                return next(err);
+            }
+
+            res.locals.namedMapProvider = provider;
+
+            next();
+        });
+    }.bind(this);
+};
+
+NamedMapsController.prototype.getTile = function (label) {
+    return function getTileMiddleware (req, res, next) {
+        const { namedMapProvider } = res.locals;
+
+        this.tileBackend.getTile(namedMapProvider, req.params, (err, tile, headers, stats) => {
+            req.profiler.add(stats);
+
+            if (err) {
+                err.label = label;
+                return next(err);
+            }
+
+            res.locals.body = tile;
+            res.locals.headers = headers;
+            res.locals.stats = stats;
+
+            next();
+        });
+    }.bind(this);
+};
+
+NamedMapsController.prototype.getStaticImageOptions = function () {
+    return function getStaticImageOptionsMiddleware(req, res, next) {
+        const { user, namedMapProvider, template } = res.locals;
+
+        const imageOpts = getImageOptions(res.locals, template);
+
+        if (imageOpts) {
+            res.locals.imageOpts = imageOpts;
+            return next();
+        }
+
+        res.locals.imageOpts = DEFAULT_ZOOM_CENTER;
+
+        namedMapProvider.getAffectedTablesAndLastUpdatedTime((err, affectedTablesAndLastUpdate) => {
+            if (err) {
+                return next();
+            }
+
+            var affectedTables = affectedTablesAndLastUpdate.tables || [];
+
+            if (affectedTables.length === 0) {
+                return next();
+            }
+
+            this.tablesExtentApi.getBounds(user, affectedTables, (err, bounds) => {
+                if (err) {
+                    return next();
+                }
+
+                res.locals.imageOpts = bounds;
+
+                return next();
+            });
+        });
+    }.bind(this);
+};
+
+function getImageOptions (params, template) {
+    const { zoom, lon, lat, bbox } = params;
+
+    let imageOpts = getImageOptionsFromCoordinates(zoom, lon, lat);
+    if (imageOpts) {
+        return imageOpts;
+    }
+
+    imageOpts = getImageOptionsFromBoundingBox(bbox);
+    if (imageOpts) {
+        return imageOpts;
+    }
+
+    imageOpts = getImageOptionsFromTemplate(template, zoom);
+    if (imageOpts) {
+        return imageOpts;
+    }
+}
+
+function getImageOptionsFromCoordinates (zoom, lon, lat) {
+    if ([zoom, lon, lat].map(numMapper).every(Number.isFinite)) {
+        return {
+            zoom: zoom,
+            center: {
+                lng: lon,
+                lat: lat
+            }
+        };
+    }
+}
+
+
+function getImageOptionsFromTemplate (template, zoom) {
+    if (template.view) {
+        var zoomCenter = templateZoomCenter(template.view);
+        if (zoomCenter) {
+            if (Number.isFinite(+zoom)) {
+                zoomCenter.zoom = +zoom;
+            }
+
+            return zoomCenter;
+        }
+
+        var bounds = templateBounds(template.view);
+        if (bounds) {
+            return bounds;
+        }
+    }
+}
+
+function getImageOptionsFromBoundingBox (bbox = '') {
+    var _bbox = bbox.split(',').map(numMapper);
+
+    if (_bbox.length === 4 && _bbox.every(Number.isFinite)) {
+        return {
+            bounds: {
+                west: _bbox[0],
+                south: _bbox[1],
+                east: _bbox[2],
+                north: _bbox[3]
+            }
+        };
+    }
+}
+
+NamedMapsController.prototype.getImage = function (label) {
+    return function getImageMiddleware (req, res, next) {
+        const { imageOpts, namedMapProvider } = res.locals;
+        const { zoom, center, bounds } = imageOpts;
+
+        let { width, height } = req.params;
+
+        width = +width;
+        height = +height;
+
+        const format = req.params.format === 'jpg' ? 'jpeg' : 'png';
+
+        if (zoom !== undefined && center) {
+            return this.previewBackend.getImage(namedMapProvider, format, width, height, zoom, center,
+                (err, image, headers, stats) => {
+                if (err) {
+                    err.label = label;
+                    return next(err);
+                }
+
+                res.locals.body = image;
+                res.locals.headers = headers;
+                res.locals.stats = stats;
+
+                next();
+            });
+        }
+
+        this.previewBackend.getImage(namedMapProvider, format, width, height, bounds, (err, image, headers, stats) => {
+            if (err) {
+                err.label = label;
+                return next(err);
+            }
+
+            res.locals.body = image;
+            res.locals.headers = headers;
+            res.locals.stats = stats;
+
+            next();
+        });
+    }.bind(this);
+};
+
+function incrementMapViewsError (ctx) {
+    return `ERROR: failed to increment mapview count for user '${ctx.user}': ${ctx.err}`;
+}
+
+NamedMapsController.prototype.incrementMapViews = function () {
+    return function incrementMapViewsMiddleware(req, res, next) {
+        const { user, namedMapProvider } = res.locals;
+
+        namedMapProvider.getMapConfig((err, mapConfig) => {
+            if (err) {
+                global.logger.log(incrementMapViewsError({ user, err }));
+                return next();
+            }
+
+            const statTag = mapConfig.obj().stat_tag;
+
+            this.metadataBackend.incMapviewCount(user, statTag, (err) => {
+                if (err) {
+                    global.logger.log(incrementMapViewsError({ user, err }));
+                }
+
+                next();
+            });
+        });
+    }.bind(this);
+};
+
 function templateZoomCenter(view) {
-    if (!_.isUndefined(view.zoom) && view.center) {
+    if (view.zoom !== undefined && view.center) {
         return {
             zoom: view.zoom,
             center: view.center
@@ -288,9 +388,8 @@ function templateZoomCenter(view) {
 
 function templateBounds(view) {
     if (view.bounds) {
-        var hasAllBounds = _.every(['west', 'south', 'east', 'north'], function(prop) {
-            return Number.isFinite(view.bounds[prop]);
-        });
+        var hasAllBounds = ['west', 'south', 'east', 'north'].every(prop => Number.isFinite(view.bounds[prop]));
+
         if (hasAllBounds) {
             return {
                 bounds: {
@@ -306,3 +405,86 @@ function templateBounds(view) {
     }
     return false;
 }
+
+NamedMapsController.prototype.setCacheChannelHeader = function () {
+    return function setCacheChannelHeaderMiddleware (req, res, next) {
+        const { affectedTablesAndLastUpdate } = res.locals;
+
+        if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) {
+            res.set('X-Cache-Channel', affectedTablesAndLastUpdate.getCacheChannel());
+        }
+
+        next();
+    };
+};
+
+NamedMapsController.prototype.setSurrogateKey = function () {
+    return function setSurrogateKeyMiddleware(req, res, next) {
+        const { user, namedMapProvider, affectedTablesAndLastUpdate } = res.locals;
+
+        this.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, namedMapProvider.getTemplateName()));
+        if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) {
+            if (affectedTablesAndLastUpdate.tables.length > 0) {
+                this.surrogateKeysCache.tag(res, affectedTablesAndLastUpdate);
+            }
+        }
+
+        next();
+    }.bind(this);
+};
+
+NamedMapsController.prototype.setLastModifiedHeader = function () {
+    return function setLastModifiedHeaderMiddleware(req, res, next) {
+        const { affectedTablesAndLastUpdate } = res.locals;
+
+        if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) {
+            var lastModifiedDate;
+            if (Number.isFinite(affectedTablesAndLastUpdate.lastUpdatedTime)) {
+                lastModifiedDate = new Date(affectedTablesAndLastUpdate.getLastUpdatedAt());
+            } else {
+                lastModifiedDate = new Date();
+            }
+
+            res.set('Last-Modified', lastModifiedDate.toUTCString());
+        }
+
+        next();
+    };
+ };
+
+NamedMapsController.prototype.setCacheControlHeader = function () {
+    return function setCacheControlHeaderMiddleware(req, res, next) {
+        const { affectedTablesAndLastUpdate } = res.locals;
+
+        res.set('Cache-Control', 'public,max-age=7200,must-revalidate');
+
+        if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) {
+            // we increase cache control as we can invalidate it
+            res.set('Cache-Control', 'public,max-age=31536000');
+        }
+
+        next();
+    };
+ };
+
+NamedMapsController.prototype.setContentTypeHeader = function () {
+    return function setContentTypeHeaderMiddleware(req, res, next) {
+        const { headers = {} } = res.locals;
+
+        res.set('Content-Type', headers['content-type'] || headers['Content-Type'] || 'image/png');
+
+        next();
+    };
+};
+
+NamedMapsController.prototype.respond = function () {
+    return function respondMiddleware (req, res) {
+        const { body, stats = {}, format } = res.locals;
+
+        req.profiler.done('render-' + format);
+        req.profiler.add(stats);
+
+        res.status(200);
+        res.send(body);
+    };
+};

lib/cartodb/controllers/named_maps_admin.js

@@ -1,13 +1,6 @@
-var step = require('step');
-var assert = require('assert');
-var templateName = require('../backends/template_maps').templateName;
-
-var util = require('util');
-var BaseController = require('./base');
-
-var cors = require('../middleware/cors');
-var userMiddleware = require('../middleware/user');
-
+const { templateName } = require('../backends/template_maps');
+const cors = require('../middleware/cors');
+const userMiddleware = require('../middleware/user');
 
 /**
  * @param {AuthApi} authApi
@@ -15,189 +8,197 @@ var userMiddleware = require('../middleware/user');
  * @param {TemplateMaps} templateMaps
  * @constructor
  */
-function NamedMapsAdminController(authApi, pgConnection, templateMaps) {
-    BaseController.call(this, authApi, pgConnection);
-
+function NamedMapsAdminController(authApi, templateMaps) {
     this.authApi = authApi;
     this.templateMaps = templateMaps;
 }
 
-util.inherits(NamedMapsAdminController, BaseController);
-
 module.exports = NamedMapsAdminController;
 
-NamedMapsAdminController.prototype.register = function(app) {
-    app.post(app.base_url_templated, cors(), userMiddleware, this.create.bind(this));
-    app.put(app.base_url_templated + '/:template_id', cors(), userMiddleware, this.update.bind(this));
-    app.get(app.base_url_templated + '/:template_id', cors(), userMiddleware, this.retrieve.bind(this));
-    app.delete(app.base_url_templated + '/:template_id', cors(), userMiddleware, this.destroy.bind(this));
-    app.get(app.base_url_templated, cors(), userMiddleware, this.list.bind(this));
-    app.options(app.base_url_templated + '/:template_id', cors('Content-Type'));
-};
+NamedMapsAdminController.prototype.register = function (app) {
+    const { base_url_templated } = app;
 
-NamedMapsAdminController.prototype.create = function(req, res) {
-    var self = this;
+    app.post(
+        `${base_url_templated}/`,
+        cors(),
+        userMiddleware,
+        this.checkContentType('POST', 'POST TEMPLATE'),
+        this.authorizedByAPIKey('create', 'POST TEMPLATE'),
+        this.create()
+    );
 
-    var cdbuser = req.context.user;
+    app.put(
+        `${base_url_templated}/:template_id`,
+        cors(),
+        userMiddleware,
+        this.checkContentType('PUT', 'PUT TEMPLATE'),
+        this.authorizedByAPIKey('update', 'PUT TEMPLATE'),
+        this.update()
+    );
 
-    step(
-        function checkPerms(){
-            self.authApi.authorizedByAPIKey(cdbuser, req, this);
-        },
-        function addTemplate(err, authenticated) {
-            assert.ifError(err);
-            ifUnauthenticated(authenticated, 'Only authenticated users can get template maps');
-            ifInvalidContentType(req, 'template POST data must be of type application/json');
-            var cfg = req.body;
-            self.templateMaps.addTemplate(cdbuser, cfg, this);
-        },
-        function prepareResponse(err, tpl_id){
-            assert.ifError(err);
-            return { template_id: tpl_id };
-        },
-        finishFn(self, req, res, 'POST TEMPLATE')
+    app.get(
+        `${base_url_templated}/:template_id`,
+        cors(),
+        userMiddleware,
+        this.authorizedByAPIKey('get', 'GET TEMPLATE'),
+        this.retrieve()
+    );
+
+    app.delete(
+        `${base_url_templated}/:template_id`,
+        cors(),
+        userMiddleware,
+        this.authorizedByAPIKey('delete', 'DELETE TEMPLATE'),
+        this.destroy()
+    );
+
+    app.get(
+        `${base_url_templated}/`,
+        cors(),
+        userMiddleware,
+        this.authorizedByAPIKey('list', 'GET TEMPLATE LIST'),
+        this.list()
+    );
+
+    app.options(
+        `${base_url_templated}/:template_id`,
+        cors('Content-Type')
     );
 };
 
-NamedMapsAdminController.prototype.update = function(req, res) {
-    var self = this;
+NamedMapsAdminController.prototype.authorizedByAPIKey = function (action, label) {
+    return function authorizedByAPIKeyMiddleware (req, res, next) {
+        const { user } = res.locals;
 
-    var cdbuser = req.context.user;
-    var template;
-    var tpl_id;
+        this.authApi.authorizedByAPIKey(user, req, (err, authenticated) => {
+            if (err) {
+                return next(err);
+            }
 
-    step(
-        function checkPerms(){
-            self.authApi.authorizedByAPIKey(cdbuser, req, this);
-        },
-        function updateTemplate(err, authenticated) {
-            assert.ifError(err);
-            ifUnauthenticated(authenticated, 'Only authenticated user can update templated maps');
-            ifInvalidContentType(req, 'template PUT data must be of type application/json');
+            if (!authenticated) {
+                const error = new Error(`Only authenticated user can ${action} templated maps`);
+                error.http_status = 403;
+                error.label = label;
+                return next(error);
+            }
 
-            template = req.body;
-            tpl_id = templateName(req.params.template_id);
-            self.templateMaps.updTemplate(cdbuser, tpl_id, template, this);
-        },
-        function prepareResponse(err){
-            assert.ifError(err);
-
-            return { template_id: tpl_id };
-        },
-        finishFn(self, req, res, 'PUT TEMPLATE')
-    );
+            next();
+        });
+    }.bind(this);
 };
 
-NamedMapsAdminController.prototype.retrieve = function(req, res) {
-    var self = this;
+NamedMapsAdminController.prototype.checkContentType = function (action, label) {
+    return function checkContentTypeMiddleware (req, res, next) {
+        if (!req.is('application/json')) {
+            const error = new Error(`template ${action} data must be of type application/json`);
+            error.label = label;
+            return next(error);
+        }
+        next();
+    };
+};
 
-    if (req.profiler) {
+NamedMapsAdminController.prototype.create = function () {
+    return function createTemplateMiddleware (req, res, next) {
+        const { user } = res.locals;
+        const template = req.body;
+
+        this.templateMaps.addTemplate(user, template, (err, templateId) => {
+            if (err) {
+                return next(err);
+            }
+
+            res.status(200);
+
+            const method = req.query.callback ? 'jsonp' : 'json';
+            res[method]({ template_id: templateId });
+        });
+    }.bind(this);
+};
+
+NamedMapsAdminController.prototype.update = function () {
+    return function updateTemplateMiddleware (req, res, next) {
+        const { user } = res.locals;
+        const template = req.body;
+        const templateId = templateName(req.params.template_id);
+
+        this.templateMaps.updTemplate(user, templateId, template, (err) => {
+            if (err) {
+                return next(err);
+            }
+
+            res.status(200);
+
+            const method = req.query.callback ? 'jsonp' : 'json';
+            res[method]({ template_id: templateId });
+        });
+    }.bind(this);
+};
+
+NamedMapsAdminController.prototype.retrieve =  function () {
+    return function retrieveTemplateMiddleware (req, res, next) {
         req.profiler.start('windshaft-cartodb.get_template');
-    }
 
-    var cdbuser = req.context.user;
-    var tpl_id;
-    step(
-        function checkPerms(){
-            self.authApi.authorizedByAPIKey(cdbuser, req, this);
-        },
-        function getTemplate(err, authenticated) {
-            assert.ifError(err);
-            ifUnauthenticated(authenticated, 'Only authenticated users can get template maps');
+        const { user } = res.locals;
+        const templateId = templateName(req.params.template_id);
 
-            tpl_id = templateName(req.params.template_id);
-            self.templateMaps.getTemplate(cdbuser, tpl_id, this);
-        },
-        function prepareResponse(err, tpl_val) {
-            assert.ifError(err);
-            if ( ! tpl_val ) {
-                err = new Error("Cannot find template '" + tpl_id + "' of user '" + cdbuser + "'");
-                err.http_status = 404;
-                throw err;
+        this.templateMaps.getTemplate(user, templateId, (err, template) => {
+            if (err) {
+                return next(err);
+            }
+
+            if (!template) {
+                const error = new Error(`Cannot find template '${templateId}' of user '${user}'`);
+                error.http_status = 404;
+                return next(error);
             }
             // auth_id was added by ourselves,
             // so we remove it before returning to the user
-            delete tpl_val.auth_id;
-            return { template: tpl_val };
-        },
-        finishFn(self, req, res, 'GET TEMPLATE')
-    );
+            delete template.auth_id;
+
+            res.status(200);
+
+            const method = req.query.callback ? 'jsonp' : 'json';
+            res[method]({ template });
+        });
+    }.bind(this);
 };
 
-NamedMapsAdminController.prototype.destroy = function(req, res) {
-    var self = this;
-
-    if (req.profiler) {
+NamedMapsAdminController.prototype.destroy = function () {
+    return function destroyTemplateMiddleware (req, res, next) {
         req.profiler.start('windshaft-cartodb.delete_template');
-    }
 
-    var cdbuser = req.context.user;
-    var tpl_id;
-    step(
-        function checkPerms(){
-            self.authApi.authorizedByAPIKey(cdbuser, req, this);
-        },
-        function deleteTemplate(err, authenticated) {
-            assert.ifError(err);
-            ifUnauthenticated(authenticated, 'Only authenticated users can delete template maps');
+        const { user } = res.locals;
+        const templateId = templateName(req.params.template_id);
 
-            tpl_id = templateName(req.params.template_id);
-            self.templateMaps.delTemplate(cdbuser, tpl_id, this);
-        },
-        function prepareResponse(err/*, tpl_val*/){
-            assert.ifError(err);
-            return '';
-        },
-        finishFn(self, req, res, 'DELETE TEMPLATE', 204)
-    );
+        this.templateMaps.delTemplate(user, templateId, (err/* , tpl_val */) => {
+            if (err) {
+                return next(err);
+            }
+
+            res.status(204);
+
+            const method = req.query.callback ? 'jsonp' : 'json';
+            res[method]('');
+        });
+    }.bind(this);
 };
 
-NamedMapsAdminController.prototype.list = function(req, res) {
-    var self = this;
-    if ( req.profiler ) {
+NamedMapsAdminController.prototype.list = function () {
+    return function listTemplatesMiddleware (req, res, next) {
         req.profiler.start('windshaft-cartodb.get_template_list');
-    }
 
-    var cdbuser = req.context.user;
+        const { user } = res.locals;
 
-    step(
-        function checkPerms(){
-            self.authApi.authorizedByAPIKey(cdbuser, req, this);
-        },
-        function listTemplates(err, authenticated) {
-            assert.ifError(err);
-            ifUnauthenticated(authenticated, 'Only authenticated user can list templated maps');
+        this.templateMaps.listTemplates(user, (err, templateIds) => {
+            if (err) {
+                return next(err);
+            }
 
-            self.templateMaps.listTemplates(cdbuser, this);
-        },
-        function prepareResponse(err, tpl_ids){
-            assert.ifError(err);
-            return { template_ids: tpl_ids };
-        },
-        finishFn(self, req, res, 'GET TEMPLATE LIST')
-    );
+            res.status(200);
+
+            const method = req.query.callback ? 'jsonp' : 'json';
+            res[method]({ template_ids: templateIds });
+        });
+    }.bind(this);
 };
-
-function finishFn(controller, req, res, description, status) {
-    return function finish(err, response){
-        if (err) {
-            controller.sendError(req, res, err, description);
-        } else {
-            controller.send(req, res, response, status || 200);
-        }
-    };
-}
-
-function ifUnauthenticated(authenticated, description) {
-    if (!authenticated) {
-        var err = new Error(description);
-        err.http_status = 403;
-        throw err;
-    }
-}
-
-function ifInvalidContentType(req, description) {
-    if (!req.is('application/json')) {
-        throw new Error(description);
-    }
-}

lib/cartodb/controllers/server_info.js

@@ -1,21 +1,13 @@
-var windshaft = require('windshaft');
 var HealthCheck = require('../monitoring/health_check');
 
 var WELCOME_MSG = "This is the CartoDB Maps API, " +
     "see the documentation at http://docs.cartodb.com/cartodb-platform/maps-api.html";
 
 
-var versions = {
-    windshaft: windshaft.version,
-    grainstore: windshaft.grainstore.version(),
-    node_mapnik: windshaft.mapnik.version,
-    mapnik: windshaft.mapnik.versions.mapnik,
-    windshaft_cartodb: require('../../../package.json').version
-};
-
-function ServerInfoController() {
+function ServerInfoController(versions) {
     this.healthConfig = global.environment.health || {};
     this.healthCheck = new HealthCheck(global.environment.disabled_file);
+    this.versions = versions || {};
 }
 
 module.exports = ServerInfoController;
@@ -31,7 +23,7 @@ ServerInfoController.prototype.welcome = function(req, res) {
 };
 
 ServerInfoController.prototype.version = function(req, res) {
-    res.status(200).send(versions);
+    res.status(200).send(this.versions);
 };
 
 ServerInfoController.prototype.health = function(req, res) {

lib/cartodb/middleware/allow-query-params.js

@@ -0,0 +1,9 @@
+module.exports = function allowQueryParams(params) {
+    if (!Array.isArray(params)) {
+        throw new Error('allowQueryParams must receive an Array of params');
+    }
+    return function allowQueryParamsMiddleware(req, res, next) {
+        res.locals.allowedQueryParams = params;
+        next();
+    };
+};

lib/cartodb/middleware/context/authorize.js

@@ -0,0 +1,20 @@
+module.exports = function authorizeMiddleware (authApi) {
+    return function (req, res, next) {
+        req.profiler.done('req2params.setup');
+
+        authApi.authorize(req, res, (err, authorized) => {
+            req.profiler.done('authorize');
+            if (err) {
+                return next(err);
+            }
+
+            if(!authorized) {
+                err = new Error("Sorry, you are unauthorized (permission denied)");
+                err.http_status = 403;
+                return next(err);
+            }
+
+            return next();
+        });
+    };
+};

lib/cartodb/middleware/context/clean-up-query-params.js

@@ -0,0 +1,32 @@
+const _ = require('underscore');
+
+// Whitelist query parameters and attach format
+const REQUEST_QUERY_PARAMS_WHITELIST = [
+    'config',
+    'map_key',
+    'api_key',
+    'auth_token',
+    'callback',
+    'zoom',
+    'lon',
+    'lat',
+    // analysis
+    'filters' // json
+];
+
+module.exports = function cleanUpQueryParamsMiddleware () {
+    return function cleanUpQueryParams (req, res, next) {
+        var allowedQueryParams = REQUEST_QUERY_PARAMS_WHITELIST;
+
+        if (Array.isArray(res.locals.allowedQueryParams)) {
+            allowedQueryParams = allowedQueryParams.concat(res.locals.allowedQueryParams);
+        }
+
+        req.query = _.pick(req.query, allowedQueryParams);
+
+        // bring all query values onto res.locals object
+        _.extend(res.locals, req.query);
+        
+        next();
+    };
+};

lib/cartodb/middleware/context/db-conn-setup.js

@@ -0,0 +1,31 @@
+const _ = require('underscore');
+
+module.exports = function dbConnSetupMiddleware(pgConnection) {
+    return function dbConnSetup(req, res, next) {
+        const user = res.locals.user;
+        pgConnection.setDBConn(user, res.locals, (err) => {
+            if (err) {
+                if (err.message && -1 !== err.message.indexOf('name not found')) {
+                    err.http_status = 404;
+                }
+                req.profiler.done('req2params');
+                return next(err);
+            }
+
+            // Add default database connection parameters
+            // if none given
+            _.defaults(res.locals, {
+                dbuser: global.environment.postgres.user,
+                dbpassword: global.environment.postgres.password,
+                dbhost: global.environment.postgres.host,
+                dbport: global.environment.postgres.port
+            });
+            
+            res.set('X-Served-By-DB-Host', res.locals.dbhost);
+
+            req.profiler.done('req2params');
+            
+            next(null);
+        });
+    };
+};

lib/cartodb/middleware/context/index.js

@@ -0,0 +1,15 @@
+const locals = require('./locals');
+const cleanUpQueryParams = require('./clean-up-query-params');
+const layergroupToken = require('./layergroup-token');
+const authorize = require('./authorize');
+const dbConnSetup = require('./db-conn-setup');
+
+module.exports = function prepareContextMiddleware(authApi, pgConnection) {
+    return [
+        locals,
+        cleanUpQueryParams(),
+        layergroupToken,
+        authorize(authApi),
+        dbConnSetup(pgConnection)
+    ];
+};

lib/cartodb/middleware/context/layergroup-token.js

@@ -0,0 +1,32 @@
+var LayergroupToken = require('../../models/layergroup-token');
+
+module.exports = function layergroupTokenMiddleware(req, res, next) {
+    if (!res.locals.token) {
+        return next();
+    }
+
+    var user = res.locals.user;
+
+    var layergroupToken = LayergroupToken.parse(res.locals.token);
+    res.locals.token = layergroupToken.token;
+    res.locals.cache_buster = layergroupToken.cacheBuster;
+
+    if (layergroupToken.signer) {
+        res.locals.signer = layergroupToken.signer;
+        if (!res.locals.signer) {
+            res.locals.signer = user;
+        } else if (res.locals.signer !== user) {
+            var err = new Error(`Cannot use map signature of user "${res.locals.signer}" on db of user "${user}"`);
+            err.type = 'auth';
+            err.http_status = 403;
+            if (req.query && req.query.callback) {
+                err.http_status = 200;
+            }
+            
+            req.profiler.done('req2params');
+            return next(err);
+        }
+    }
+
+    return next();
+};

lib/cartodb/middleware/context/locals.js

@@ -0,0 +1,6 @@
+module.exports = function localsMiddleware(req, res, next) {
+    // save req.params in res.locals
+    res.locals = Object.assign(req.params || {}, res.locals);
+
+    next();
+};

lib/cartodb/middleware/error-middleware.js

@@ -0,0 +1,213 @@
+const _ = require('underscore');
+const debug = require('debug')('windshaft:cartodb:error-middleware');
+
+module.exports = function errorMiddleware (/* options */) {
+    return function error (err, req, res, next) {
+        // jshint unused:false
+        // jshint maxcomplexity:9
+        var allErrors = Array.isArray(err) ? err : [err];
+
+        allErrors = populateTimeoutErrors(allErrors);
+
+        const label = err.label || 'UNKNOWN';
+        err = allErrors[0] || new Error(label);
+        allErrors[0] = err;
+
+        var statusCode = findStatusCode(err);
+
+        if (err.message === 'Tile does not exist' && res.locals.format === 'mvt') {
+            statusCode = 204;
+        }
+        
+        setErrorHeader(allErrors, statusCode, res);
+        debug('[%s ERROR] -- %d: %s, %s', label, statusCode, err, err.stack);
+
+        // If a callback was requested, force status to 200
+        if (req.query && req.query.callback) {
+            statusCode = 200;
+        }
+
+        var errorResponseBody = {
+            errors: allErrors.map(errorMessage),
+            errors_with_context: allErrors.map(errorMessageWithContext)
+        };
+
+        res.status(statusCode);
+
+        if (req.query && req.query.callback) {
+            res.jsonp(errorResponseBody);
+        } else {
+            res.json(errorResponseBody);
+        }
+    };
+};
+
+function isRenderTimeoutError (err) {
+    return err.message === 'Render timed out';
+}
+
+function isDatasourceTimeoutError (err) {
+    return err.message && err.message.match(/canceling statement due to statement timeout/i);
+}
+
+function isTimeoutError (err) {
+    return isRenderTimeoutError(err) || isDatasourceTimeoutError(err);
+}
+
+function populateTimeoutErrors (errors) {
+    return errors.map(function (error) {
+        if (isRenderTimeoutError(error)) {
+            error.subtype = 'render';
+        }
+
+        if (isDatasourceTimeoutError(error)) {
+            error.subtype = 'datasource';
+        }
+
+        if (isTimeoutError(error)) {
+            error.message = 'You are over platform\'s limits. Please contact us to know more details';
+            error.type = 'limit';
+            error.http_status = 429;
+        }
+
+        return error;
+    });
+}
+
+function findStatusCode(err) {
+    var statusCode;
+    if ( err.http_status ) {
+        statusCode = err.http_status;
+    } else {
+        statusCode = statusFromErrorMessage('' + err);
+    }
+    return statusCode;
+}
+
+module.exports.findStatusCode = findStatusCode;
+
+function statusFromErrorMessage(errMsg) {
+    // Find an appropriate statusCode based on message
+    // jshint maxcomplexity:7
+    var statusCode = 400;
+    if ( -1 !== errMsg.indexOf('permission denied') ) {
+        statusCode = 403;
+    }
+    else if ( -1 !== errMsg.indexOf('authentication failed') ) {
+        statusCode = 403;
+    }
+    else if (errMsg.match(/Postgis Plugin.*[\s|\n].*column.*does not exist/)) {
+        statusCode = 400;
+    }
+    else if ( -1 !== errMsg.indexOf('does not exist') ) {
+        if ( -1 !== errMsg.indexOf(' role ') ) {
+            statusCode = 403; // role 'xxx' does not exist
+        } else if ( errMsg.match(/function .* does not exist/) ) {
+            statusCode = 400; // invalid SQL (SQL function does not exist)
+        } else {
+            statusCode = 404;
+        }
+    }
+
+
+    return statusCode;
+}
+
+function errorMessage(err) {
+    // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
+    var message = (_.isString(err) ? err : err.message) || 'Unknown error';
+
+    return stripConnectionInfo(message);
+}
+
+module.exports.errorMessage = errorMessage;
+
+function stripConnectionInfo(message) {
+    // Strip connection info, if any
+    return message
+        // See https://github.com/CartoDB/Windshaft/issues/173
+        .replace(/Connection string: '[^']*'\n\s/im, '')
+        // See https://travis-ci.org/CartoDB/Windshaft/jobs/20703062#L1644
+        .replace(/is the server.*encountered/im, 'encountered');
+}
+
+var ERROR_INFO_TO_EXPOSE = {
+    message: true,
+    layer: true,
+    type: true,
+    analysis: true,
+    subtype: true
+};
+
+function shouldBeExposed (prop) {
+    return !!ERROR_INFO_TO_EXPOSE[prop];
+}
+
+function errorMessageWithContext(err) {
+    // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
+    var message = (_.isString(err) ? err : err.message) || 'Unknown error';
+
+    var error = {
+        type: err.type || 'unknown',
+        message: stripConnectionInfo(message),
+    };
+
+    for (var prop in err) {
+        // type & message are properties from Error's prototype and will be skipped
+        if (err.hasOwnProperty(prop) && shouldBeExposed(prop)) {
+            error[prop] = err[prop];
+        }
+    }
+
+    return error;
+}
+
+function setErrorHeader(errors, statusCode, res) {
+    let errorsCopy = errors.slice(0);
+    const mainError = errorsCopy.shift();
+
+    let errorsLog = {
+        mainError: {
+            statusCode: statusCode || 200,
+            message:    mainError.message,
+            name:       mainError.name,
+            label:      mainError.label,
+            type:       mainError.type,
+            subtype:    mainError.subtype
+        }
+    };
+
+    errorsLog.moreErrors = errorsCopy.map(error => {
+        return {
+            message: error.message,
+            name:    error.name,
+            label:   error.label,
+            type:    error.type,
+            subtype: error.subtype
+        };
+    });
+    
+    res.set('X-Tiler-Errors', stringifyForLogs(errorsLog));
+}
+
+/**
+ * Remove problematic nested characters 
+ * from object for logs RegEx
+ * 
+ * @param {Object} object 
+ */
+function stringifyForLogs(object) {
+    Object.keys(object).map(key => {
+        if(typeof object[key] === 'string') {
+            object[key] = object[key].replace(/[^a-zA-Z0-9]/g, ' ');
+        } else if (typeof object[key] === 'object') {
+            stringifyForLogs(object[key]);
+        } else if (object[key] instanceof Array) {
+            for (let element of object[key]) {
+                stringifyForLogs(element);
+            }
+        }
+    });
+
+    return JSON.stringify(object);
+}

lib/cartodb/middleware/lzma.js

@@ -0,0 +1,30 @@
+'use strict';
+
+const LZMA = require('lzma').LZMA;
+
+const lzmaWorker = new LZMA();
+
+module.exports = function lzmaMiddleware(req, res, next) {
+    if (!req.query.hasOwnProperty('lzma')) {
+        return next();
+    }
+
+    // Decode (from base64)
+    var lzma = new Buffer(req.query.lzma, 'base64')
+        .toString('binary')
+        .split('')
+        .map(function(c) {
+            return c.charCodeAt(0) - 128;
+        });
+
+    // Decompress
+    lzmaWorker.decompress(lzma, function(result) {
+        try {
+            delete req.query.lzma;
+            Object.assign(req.query, JSON.parse(result));
+            next();
+        } catch (err) {
+            next(new Error('Error parsing lzma as JSON: ' + err));
+        }
+    });
+};

lib/cartodb/middleware/stats.js

@@ -0,0 +1,27 @@
+const Profiler = require('../stats/profiler_proxy');
+const debug = require('debug')('windshaft:cartodb:stats');
+const onHeaders = require('on-headers');
+
+module.exports = function statsMiddleware(options) {
+    const { enabled = true, statsClient } = options;
+
+    return function stats(req, res, next) {
+        req.profiler = new Profiler({
+            statsd_client: statsClient,
+            profile: enabled
+        });
+
+        onHeaders(res, () => res.set('X-Tiler-Profiler', req.profiler.toJSONString()));
+
+        res.on('finish', () => {
+            try {
+                // May throw due to dns, see: http://github.com/CartoDB/Windshaft/issues/166
+                req.profiler.sendStats();
+            } catch (err) {
+                debug("error sending profiling stats: " + err);
+            }
+        });
+
+        next();
+    };
+};

lib/cartodb/middleware/user.js

@@ -2,6 +2,7 @@ var CdbRequest = require('../models/cdb_request');
 var cdbRequest = new CdbRequest();
 
 module.exports = function userMiddleware(req, res, next) {
-    req.context.user = cdbRequest.userByReq(req);
+    res.locals.user = cdbRequest.userByReq(req);
+
     next();
 };

lib/cartodb/middleware/vector-error.js

@@ -0,0 +1,30 @@
+const fs = require('fs');
+
+const timeoutErrorVectorTile = fs.readFileSync(__dirname + '/../../../assets/render-timeout-fallback.mvt');
+
+module.exports = function vectorError() {
+    return function vectorErrorMiddleware(err, req, res, next) {
+        if(req.params.format === 'mvt') {
+
+            if (isTimeoutError(err)) {
+                res.set('Content-Type', 'application/x-protobuf');
+                return res.status(429).send(timeoutErrorVectorTile);
+            }
+        }
+
+        next(err);
+    };
+};
+
+
+function isRenderTimeoutError (err) {
+    return err.message === 'Render timed out';
+}
+
+function isDatasourceTimeoutError (err) {
+    return err.message && err.message.match(/canceling statement due to statement timeout/i);
+}
+
+function isTimeoutError (err) {
+    return isRenderTimeoutError(err) || isDatasourceTimeoutError(err);
+}

lib/cartodb/models/aggregation/aggregation-mapconfig.js

@@ -0,0 +1,233 @@
+const MapConfig = require('windshaft').model.MapConfig;
+const aggregationQuery = require('./aggregation-query');
+const aggregationValidator = require('./aggregation-validator');
+const {
+    createPositiveNumberValidator,
+    createIncludesValueValidator,
+    createAggregationColumnsValidator
+} = aggregationValidator;
+
+const SubstitutionTokens = require('../../utils/substitution-tokens');
+
+const removeDuplicates = arr => [...new Set(arr)];
+
+function prepareSql(sql) {
+    return sql && SubstitutionTokens.replace(sql, {
+        bbox: 'ST_MakeEnvelope(0,0,0,0)',
+        scale_denominator: '0',
+        pixel_width: '1',
+        pixel_height: '1'
+    });
+}
+
+module.exports = class AggregationMapConfig extends MapConfig {
+    static get AGGREGATIONS () {
+        return aggregationQuery.SUPPORTED_AGGREGATE_FUNCTIONS;
+    }
+
+    static get PLACEMENTS () {
+        return aggregationQuery.SUPPORTED_PLACEMENTS;
+    }
+
+    static get THRESHOLD () {
+        return 1e5; // 100K
+    }
+
+    static get RESOLUTION () {
+        return 1;
+    }
+
+    static get SUPPORTED_GEOMETRY_TYPES () {
+        return [
+            'ST_Point'
+        ];
+    }
+
+    static supportsGeometryType(geometryType) {
+        return AggregationMapConfig.SUPPORTED_GEOMETRY_TYPES.includes(geometryType);
+    }
+
+    static  getAggregationGeometryColumn() {
+        return aggregationQuery.GEOMETRY_COLUMN;
+    }
+
+    constructor (user, config, connection, datasource) {
+        super(config, datasource);
+
+        const validate = aggregationValidator(this);
+        const positiveNumberValidator = createPositiveNumberValidator(this);
+        const includesValidPlacementsValidator = createIncludesValueValidator(this, AggregationMapConfig.PLACEMENTS);
+        const aggregationColumnsValidator = createAggregationColumnsValidator(this, AggregationMapConfig.AGGREGATIONS);
+
+        validate('resolution', positiveNumberValidator);
+        validate('placement', includesValidPlacementsValidator);
+        validate('threshold', positiveNumberValidator);
+        validate('columns', aggregationColumnsValidator);
+
+        this.user = user;
+        this.pgConnection = connection;
+    }
+
+    getAggregatedQuery (index) {
+        const { sql_raw, sql } = this.getLayer(index).options;
+        const {
+            // The default aggregation has no placement, columns or dimensions;
+            // this enables the special "full-sample" aggregation.
+            resolution = AggregationMapConfig.RESOLUTION,
+            threshold = AggregationMapConfig.THRESHOLD,
+            placement,
+            columns = {},
+            dimensions = {}
+        } = this.getAggregation(index);
+
+        return aggregationQuery({
+            query: sql_raw || sql,
+            resolution,
+            threshold,
+            placement,
+            columns,
+            dimensions,
+            isDefaultAggregation: this._isDefaultLayerAggregation(index)
+        });
+    }
+
+    isAggregationMapConfig () {
+        return this.isVectorOnlyMapConfig() || this.hasAnyLayerAggregation();
+    }
+
+    isAggregationLayer (index) {
+        return this.isVectorOnlyMapConfig() || this.hasLayerAggregation(index);
+    }
+
+    hasAnyLayerAggregation () {
+        const layers = this.getLayers();
+
+        for (let index = 0; index < layers.length; index++) {
+            if (this.hasLayerAggregation(index)) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    hasLayerAggregation (index) {
+        const layer = this.getLayer(index);
+        const { aggregation } = layer.options;
+
+        return aggregation !== undefined && (typeof aggregation === 'object' || typeof aggregation === 'boolean');
+    }
+
+    getAggregation (index) {
+        if (this.isVectorOnlyMapConfig() && !this.hasLayerAggregation(index)) {
+            return {};
+        }
+
+        const { aggregation } = this.getLayer(index).options;
+
+        if (typeof aggregation === 'boolean') {
+            return {};
+        }
+
+        return aggregation;
+    }
+
+    getLayerAggregationColumns (index, callback) {
+        if (this._isDefaultLayerAggregation(index)) {
+            const skipGeoms = true;
+            return this.getLayerColumns(index, skipGeoms, (err, columns) => {
+                if (err) {
+                    return callback(err);
+                }
+
+                return callback(null, columns);
+            });
+        }
+
+        const columns = this._getLayerAggregationRequiredColumns(index);
+
+        return callback(null, columns);
+    }
+
+    _getLayerAggregationRequiredColumns (index) {
+        const { columns, dimensions } = this.getAggregation(index);
+
+        let aggregatedColumns = [];
+        if (columns) {
+            aggregatedColumns = Object.keys(columns)
+                .map(key => columns[key].aggregated_column)
+                .filter(aggregatedColumn => typeof aggregatedColumn === 'string');
+        }
+
+        let dimensionsColumns = [];
+        if (dimensions) {
+            dimensionsColumns = Object.keys(dimensions)
+                .map(key => dimensions[key])
+                .filter(dimension => typeof dimension === 'string');
+        }
+
+        return removeDuplicates(aggregatedColumns.concat(dimensionsColumns));
+    }
+
+    doesLayerReachThreshold(index, featureCount) {
+        const threshold = this.getAggregation(index) && this.getAggregation(index).threshold ?
+            this.getAggregation(index).threshold :
+            AggregationMapConfig.THRESHOLD;
+
+        return featureCount >= threshold;
+    }
+
+    getLayerColumns (index, skipGeoms, callback) {
+        const geomColumns = ['the_geom', 'the_geom_webmercator'];
+        const limitedQuery = ctx => `SELECT * FROM (${ctx.query}) __cdb_schema LIMIT 0`;
+        const layer = this.getLayer(index);
+
+        this.pgConnection.getConnection(this.user, (err, connection) => {
+            if (err) {
+                return callback(err);
+            }
+
+            const sql = limitedQuery({
+                query: prepareSql(layer.options.sql)
+            });
+
+            connection.query(sql, (err, result) => {
+                if (err) {
+                    return callback(err);
+                }
+
+                let columns = result.fields || [];
+
+                columns = columns.map(({ name }) => name);
+
+                if (skipGeoms) {
+                    columns = columns.filter((column) => !geomColumns.includes(column));
+                }
+
+                return callback(err, columns);
+            });
+        });
+    }
+
+    _isDefaultLayerAggregation (index) {
+        const aggregation = this.getAggregation(index);
+
+        return (this.isVectorOnlyMapConfig() && !this.hasLayerAggregation(index)) ||
+            aggregation === true ||
+            this._isDefaultAggregation(aggregation);
+    }
+
+    _isDefaultAggregation (aggregation) {
+        return aggregation.placement === undefined &&
+            aggregation.columns === undefined &&
+            this._isEmptyParameter(aggregation.dimensions);
+    }
+
+    _isEmptyParameter(parameter) {
+        return parameter === undefined || parameter === null || this._isEmptyObject(parameter);
+    }
+
+    _isEmptyObject (parameter) {
+        return typeof parameter === 'object' && Object.keys(parameter).length === 0;
+    }
+};

lib/cartodb/models/aggregation/aggregation-query.js

@@ -0,0 +1,253 @@
+const DEFAULT_PLACEMENT = 'point-sample';
+
+/**
+ * Returns a template function (function that accepts template parameters and returns a string)
+ * to generate an aggregation query.
+ * Valid options to define the query template are:
+ * - placement
+ * - columns
+ * - dimensions*
+ * The query template parameters taken by the result template function are:
+ * - sourceQuery
+ * - res
+ * - columns
+ * - dimensions
+ */
+const templateForOptions = (options) => {
+    let templateFn = defaultAggregationQueryTemplate;
+    if (!options.isDefaultAggregation) {
+        templateFn = aggregationQueryTemplates[options.placement || DEFAULT_PLACEMENT];
+        if (!templateFn) {
+            throw new Error("Invalid Aggregation placement: '" + options.placement + "'");
+        }
+    }
+    return templateFn;
+};
+
+/**
+ * Generates an aggregation query given the aggregation options:
+ * - query
+ * - resolution - defined as in torque:
+ *   aggregation cell is resolution*resolution pixels, where tiles are always 256x256 pixels
+ * - columns
+ * - placement
+ * - dimensions
+ *
+ * The default aggregation (when no explicit placement, columns or dimensions are present) returns
+ * a sample record (with all the original columns and _cdb_feature_count) for each aggregation group.
+ * When placement, columns or dimensions are specified, columns are aggregated as requested
+ * (by default only _cdb_feature_count) and with the_geom_webmercator as defined by placement.
+ */
+const queryForOptions = (options) => templateForOptions(options)({
+    sourceQuery: options.query,
+    res: 256/options.resolution,
+    columns: options.columns,
+    dimensions: options.dimensions
+});
+
+module.exports = queryForOptions;
+
+const SUPPORTED_AGGREGATE_FUNCTIONS = {
+    'count': {
+        sql: (column_name, params) => `count(${params.aggregated_column || '*'})`
+    },
+    'avg': {
+        sql: (column_name, params) => `avg(${params.aggregated_column || column_name})`
+    },
+    'sum': {
+        sql: (column_name, params) => `sum(${params.aggregated_column || column_name})`
+    },
+    'min': {
+        sql: (column_name, params) => `min(${params.aggregated_column || column_name})`
+    },
+    'max': {
+        sql: (column_name, params) => `max(${params.aggregated_column || column_name})`
+    },
+    'mode': {
+        sql: (column_name, params) => `_cdb_mode(${params.aggregated_column || column_name})`
+    }
+};
+
+module.exports.SUPPORTED_AGGREGATE_FUNCTIONS = Object.keys(SUPPORTED_AGGREGATE_FUNCTIONS);
+
+const sep = (list) => {
+    let expr = list.join(', ');
+    return expr ? ', ' + expr : expr;
+};
+
+const aggregateColumns = ctx => {
+    return Object.assign({
+        _cdb_feature_count: {
+            aggregate_function: 'count'
+        }
+    }, ctx.columns || {});
+};
+
+const aggregateColumnNames = (ctx, table) => {
+    let columns = aggregateColumns(ctx);
+    if (table) {
+        return sep(Object.keys(columns).map(
+            column_name => `${table}.${column_name}`
+        ));
+    }
+    return sep(Object.keys(columns));
+};
+
+const aggregateColumnDefs = ctx => {
+    let columns = aggregateColumns(ctx);
+    return sep(Object.keys(columns).map(column_name => {
+        const aggregate_function = columns[column_name].aggregate_function || 'count';
+        const aggregate_definition = SUPPORTED_AGGREGATE_FUNCTIONS[aggregate_function];
+        if (!aggregate_definition) {
+            throw new Error("Invalid Aggregate function: '" + aggregate_function + "'");
+        }
+        const aggregate_expression = aggregate_definition.sql(column_name, columns[column_name]);
+        return `${aggregate_expression} AS ${column_name}`;
+    }));
+};
+
+
+const aggregateDimensions = ctx => ctx.dimensions || {};
+
+const dimensionNames = (ctx, table) => {
+    let dimensions = aggregateDimensions(ctx);
+    if (table) {
+        return sep(Object.keys(dimensions).map(
+            dimension_name => `${table}.${dimension_name}`
+        ));
+    }
+    return sep(Object.keys(dimensions));
+};
+
+const dimensionDefs = ctx => {
+    let dimensions = aggregateDimensions(ctx);
+    return sep(Object.keys(dimensions).map(dimension_name => {
+        const expression = dimensions[dimension_name];
+        return `${expression} AS ${dimension_name}`;
+    }));
+};
+
+// SQL expression to compute the aggregation resolution (grid cell size).
+// This is equivalent to `${256/ctx.res}*CDB_XYZ_Resolution(CDB_ZoomFromScale(!scale_denominator!))`
+// This is defined by the ctx.res parameter, which is the number of grid cells per tile linear dimension
+// (i.e. each tile is divided into ctx.res*ctx.res cells).
+const gridResolution = ctx => `(${256*0.00028/ctx.res}*!scale_denominator!)::double precision`;
+
+// Notes:
+// * We need to filter spatially using !bbox! to make the queries efficient because
+//   the filter added by Mapnik (wrapping the query)
+//   is only applied after the aggregation.
+// * This queries are used for rendering and the_geom is omitted in the results for better performance
+
+// The special default aggregation includes all the columns of a sample row per grid cell and
+// the count (_cdb_feature_count) of the aggregated rows.
+const defaultAggregationQueryTemplate = ctx => `
+    WITH
+    _cdb_params AS (
+        SELECT
+        ${gridResolution(ctx)} AS res,
+        !bbox! AS bbox
+    ),
+    _cdb_clusters AS (
+        SELECT
+            MIN(cartodb_id) AS cartodb_id
+            ${dimensionDefs(ctx)}
+            ${aggregateColumnDefs(ctx)}
+        FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params
+        WHERE _cdb_query.the_geom_webmercator && _cdb_params.bbox
+        GROUP BY
+            Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res),
+            Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)
+            ${dimensionNames(ctx)}
+    ) SELECT
+        _cdb_query.*
+        ${aggregateColumnNames(ctx)}
+    FROM
+        _cdb_clusters INNER JOIN (${ctx.sourceQuery}) _cdb_query
+        ON (_cdb_clusters.cartodb_id = _cdb_query.cartodb_id)
+`;
+
+const aggregationQueryTemplates = {
+    'centroid': ctx => `
+        WITH
+        _cdb_params AS (
+            SELECT
+            ${gridResolution(ctx)} AS res,
+            !bbox! AS bbox
+        )
+        SELECT
+            row_number() over() AS cartodb_id,
+            ST_SetSRID(
+                ST_MakePoint(
+                    AVG(ST_X(_cdb_query.the_geom_webmercator)),
+                    AVG(ST_Y(_cdb_query.the_geom_webmercator))
+                ), 3857
+            ) AS the_geom_webmercator
+            ${dimensionDefs(ctx)}
+            ${aggregateColumnDefs(ctx)}
+        FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params
+        WHERE _cdb_query.the_geom_webmercator && _cdb_params.bbox
+        GROUP BY
+            Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res),
+            Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)
+            ${dimensionNames(ctx)}
+    `,
+
+    'point-grid': ctx => `
+        WITH
+        _cdb_params AS (
+            SELECT
+            ${gridResolution(ctx)} AS res,
+            !bbox! AS bbox
+        ),
+        _cdb_clusters AS (
+            SELECT
+            Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res)::int AS _cdb_gx,
+            Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)::int AS _cdb_gy
+            ${dimensionDefs(ctx)}
+            ${aggregateColumnDefs(ctx)}
+            FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params
+            WHERE the_geom_webmercator && _cdb_params.bbox
+            GROUP BY _cdb_gx, _cdb_gy ${dimensionNames(ctx)}
+        )
+        SELECT
+            row_number() over() AS cartodb_id,
+            ST_SetSRID(ST_MakePoint((_cdb_gx+0.5)*res, (_cdb_gy+0.5)*res), 3857) AS the_geom_webmercator
+            ${dimensionNames(ctx)}
+            ${aggregateColumnNames(ctx)}
+            FROM _cdb_clusters, _cdb_params
+    `,
+
+    'point-sample': ctx => `
+        WITH
+        _cdb_params AS (
+            SELECT
+            ${gridResolution(ctx)} AS res,
+            !bbox! AS bbox
+        ),
+        _cdb_clusters AS (
+            SELECT
+                MIN(cartodb_id) AS cartodb_id
+                ${dimensionDefs(ctx)}
+                ${aggregateColumnDefs(ctx)}
+            FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params
+            WHERE _cdb_query.the_geom_webmercator && _cdb_params.bbox
+            GROUP BY
+                Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res),
+                Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)
+                ${dimensionNames(ctx)}
+        )
+        SELECT
+            _cdb_clusters.cartodb_id,
+            the_geom, the_geom_webmercator
+            ${dimensionNames(ctx, '_cdb_query')}
+            ${aggregateColumnNames(ctx, '_cdb_clusters')}
+        FROM
+            _cdb_clusters INNER JOIN (${ctx.sourceQuery}) _cdb_query
+            ON (_cdb_clusters.cartodb_id = _cdb_query.cartodb_id)
+    `
+
+};
+
+module.exports.SUPPORTED_PLACEMENTS = Object.keys(aggregationQueryTemplates);
+module.exports.GEOMETRY_COLUMN = 'the_geom_webmercator';

lib/cartodb/models/aggregation/aggregation-validator.js

@@ -0,0 +1,93 @@
+module.exports = function aggregationValidator (mapconfig) {
+    return function validateProperty (key, validator) {
+        for (let index = 0; index < mapconfig.getLayers().length; index++) {
+            const aggregation = mapconfig.getAggregation(index);
+
+            if (aggregation === undefined || aggregation[key] === undefined) {
+                continue;
+            }
+
+            validator(aggregation[key], key, index);
+        }
+    };
+};
+
+module.exports.createIncludesValueValidator = function (mapconfig, validValues) {
+    return function validateIncludesValue (value, key, index) {
+        if (!validValues.includes(value)) {
+            const message = `Invalid ${key}. Valid values: ${validValues.join(', ')}`;
+            throw createLayerError(message, mapconfig, index);
+        }
+    };
+};
+
+module.exports.createPositiveNumberValidator = function (mapconfig) {
+    return function validatePositiveNumber (value, key, index) {
+        if (!Number.isFinite(value) || value <= 0) {
+            const message = `Invalid ${key}, should be a number greather than 0`;
+            throw createLayerError(message, mapconfig, index);
+        }
+    };
+};
+
+module.exports.createAggregationColumnsValidator = function (mapconfig, validAggregatedFunctions) {
+    const validateAggregationColumnNames = createAggregationColumnNamesValidator(mapconfig);
+    const validateAggregateFunction = createAggregateFunctionValidator(mapconfig, validAggregatedFunctions);
+    const validateAggregatedColumn = createAggregatedColumnValidator(mapconfig);
+
+    return function validateAggregationColumns (value, key, index) {
+        validateAggregationColumnNames(value, key, index);
+        validateAggregateFunction(value, key, index);
+        validateAggregatedColumn(value, key, index);
+    };
+};
+
+function createAggregationColumnNamesValidator(mapconfig) {
+    return function validateAggregationColumnNames (value, key, index) {
+        Object.keys(value).forEach((columnName) => {
+            if (columnName.length <= 0) {
+                const message = `Invalid column name, should be a non empty string`;
+                throw createLayerError(message, mapconfig, index);
+            }
+        });
+    };
+}
+
+function createAggregateFunctionValidator (mapconfig, validAggregatedFunctions) {
+    return function validateAggregateFunction (value, key, index) {
+        Object.keys(value).forEach((columnName) => {
+            const { aggregate_function } = value[columnName];
+
+            if (!validAggregatedFunctions.includes(aggregate_function)) {
+                const message = `Unsupported aggregation function ${aggregate_function},` +
+                                ` valid ones: ${validAggregatedFunctions.join(', ')}`;
+                throw createLayerError(message, mapconfig, index);
+            }
+        });
+    };
+}
+
+function createAggregatedColumnValidator (mapconfig) {
+    return function validateAggregatedColumn (value, key, index) {
+        Object.keys(value).forEach((columnName) => {
+            const { aggregated_column } = value[columnName];
+
+            if (typeof aggregated_column !== 'string' || aggregated_column <= 0) {
+                const message = `Invalid aggregated column, should be a non empty string`;
+                throw createLayerError(message, mapconfig, index);
+            }
+        });
+    };
+}
+
+function createLayerError(message, mapconfig, index) {
+    const error = new Error(message);
+    error.type = 'layer';
+    error.layer = {
+        id: mapconfig.getLayerId(index),
+        index: index,
+        type: mapconfig.layerType(index)
+    };
+
+    return error;
+}

lib/cartodb/models/dataview/aggregation.js

@@ -1,63 +1,186 @@
-var _ = require('underscore');
-var BaseWidget = require('./base');
-var debug = require('debug')('windshaft:widget:aggregation');
+const BaseDataview = require('./base');
+const debug = require('debug')('windshaft:dataview:aggregation');
 
-var dot = require('dot');
-dot.templateSettings.strip = false;
+const filteredQueryTpl = ctx => `
+    filtered_source AS (
+        SELECT *
+        FROM (${ctx.query}) _cdb_filtered_source
+        ${ctx.aggregationColumn && ctx.isFloatColumn ? `
+        WHERE
+            ${ctx.aggregationColumn} != 'infinity'::float
+        AND
+            ${ctx.aggregationColumn} != '-infinity'::float
+        AND
+            ${ctx.aggregationColumn} != 'NaN'::float` :
+        ''
+        }
+    )
+`;
 
-var summaryQueryTpl = dot.template([
-    'summary AS (',
-    '  SELECT',
-    '  count(1) AS count,',
-    '  sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count',
-    '  FROM ({{=it._query}}) _cdb_aggregation_nulls',
-    ')'
-].join('\n'));
+const summaryQueryTpl = ctx => `
+    summary AS (
+        SELECT
+            count(1) AS count,
+            sum(CASE WHEN ${ctx.column} IS NULL THEN 1 ELSE 0 END) AS nulls_count
+            ${ctx.isFloatColumn ? `,
+            sum(
+                CASE
+                    WHEN ${ctx.aggregationColumn} = 'infinity'::float OR ${ctx.aggregationColumn} = '-infinity'::float
+                    THEN 1
+                    ELSE 0
+                    END
+            ) AS infinities_count,
+            sum(CASE WHEN ${ctx.aggregationColumn} = 'NaN'::float THEN 1 ELSE 0 END) AS nans_count` :
+            ''
+            }
+        FROM (${ctx.query}) _cdb_aggregation_nulls
+    )
+`;
 
-var rankedCategoriesQueryTpl = dot.template([
-    'categories AS(',
-    '  SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,',
-    '    row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank',
-    '  FROM ({{=it._query}}) _cdb_aggregation_all',
-    '  GROUP BY {{=it._column}}',
-    '  ORDER BY 2 DESC',
-    ')'
-].join('\n'));
+const rankedCategoriesQueryTpl = ctx => `
+    categories AS(
+        SELECT
+            ${ctx.column} AS category,
+            ${ctx.aggregationFn} AS value,
+            row_number() OVER (ORDER BY ${ctx.aggregationFn} desc) as rank
+        FROM filtered_source
+        ${ctx.aggregationColumn !== null ? `WHERE ${ctx.aggregationColumn} IS NOT NULL` : ''}
+        GROUP BY ${ctx.column}
+        ORDER BY 2 DESC
+    )
+`;
 
-var categoriesSummaryQueryTpl = dot.template([
-    'categories_summary AS(',
-    '  SELECT count(1) categories_count, max(value) max_val, min(value) min_val',
-    '  FROM categories',
-    ')'
-].join('\n'));
+const categoriesSummaryMinMaxQueryTpl = () => `
+    categories_summary_min_max AS(
+        SELECT
+            max(value) max_val,
+            min(value) min_val
+        FROM categories
+    )
+`;
 
-var rankedAggregationQueryTpl = dot.template([
-    'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count',
-    '  FROM categories, summary, categories_summary',
-    '  WHERE rank < {{=it._limit}}',
-    'UNION ALL',
-    'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val, count, categories_count',
-    '  FROM categories, summary, categories_summary',
-    '  WHERE rank >= {{=it._limit}}',
-    'GROUP BY nulls_count, min_val, max_val, count, categories_count'
-].join('\n'));
+const categoriesSummaryCountQueryTpl = ctx => `
+    categories_summary_count AS(
+        SELECT count(1) AS categories_count
+        FROM (
+            SELECT ${ctx.column} AS category
+            FROM (${ctx.query}) _cdb_categories
+            GROUP BY ${ctx.column}
+        ) _cdb_categories_count
+    )
+`;
 
-var aggregationQueryTpl = dot.template([
-    'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,',
-    '  nulls_count, min_val, max_val, count, categories_count',
-    'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary',
-    'GROUP BY category, nulls_count, min_val, max_val, count, categories_count',
-    'ORDER BY value DESC'
-].join('\n'));
+const specialNumericValuesColumns = () => `, nans_count, infinities_count`;
 
-var CATEGORIES_LIMIT = 6;
+const rankedAggregationQueryTpl = ctx => `
+    SELECT
+        category,
+        value,
+        false as agg,
+        nulls_count,
+        min_val,
+        max_val,
+        count,
+        categories_count
+        ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' }
+        FROM categories, summary, categories_summary_min_max, categories_summary_count
+        WHERE rank < ${ctx.limit}
+    UNION ALL
+    SELECT
+        null category,
+        ${ctx.aggregation !== 'count' ? ctx.aggregation : 'sum'}(value) as value,
+        true as agg,
+        nulls_count,
+        min_val,
+        max_val,
+        count,
+        categories_count
+        ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' }
+        FROM categories, summary, categories_summary_min_max, categories_summary_count
+        WHERE rank >= ${ctx.limit}
+    GROUP BY
+        nulls_count,
+        min_val,
+        max_val,
+        count,
+        categories_count
+        ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' }
+`;
 
-var VALID_OPERATIONS = {
+const aggregationQueryTpl = ctx => `
+    SELECT
+        ${ctx.column} AS category,
+        ${ctx.aggregationFn} AS value,
+        false as agg,
+        nulls_count,
+        min_val,
+        max_val,
+        count,
+        categories_count
+        ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' }
+    FROM (${ctx.query}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count
+    GROUP BY
+        ${ctx.column},
+        nulls_count,
+        min_val,
+        max_val,
+        count,
+        categories_count
+        ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' }
+    ORDER BY value DESC
+`;
+
+const aggregationFnQueryTpl = ctx => `${ctx.aggregation}(${ctx.aggregationColumn})`;
+
+const aggregationDataviewQueryTpl = ctx => `
+    WITH
+    ${filteredQueryTpl(ctx)},
+    ${summaryQueryTpl(ctx)},
+    ${rankedCategoriesQueryTpl(ctx)},
+    ${categoriesSummaryMinMaxQueryTpl(ctx)},
+    ${categoriesSummaryCountQueryTpl(ctx)}
+    ${!!ctx.override.ownFilter ? `${aggregationQueryTpl(ctx)}` : `${rankedAggregationQueryTpl(ctx)}`}
+`;
+
+const filterCategoriesQueryTpl = ctx => `
+    SELECT
+        ${ctx.column} AS category,
+        ${ctx.value} AS value
+    FROM (${ctx.query}) _cdb_aggregation_search
+    WHERE CAST(${ctx.column} as text) ILIKE ${ctx.userQuery}
+    GROUP BY ${ctx.column}
+`;
+
+const searchQueryTpl = ctx => `
+    WITH
+    search_unfiltered AS (
+        ${ctx.searchUnfiltered}
+    ),
+    search_filtered AS (
+        ${ctx.searchFiltered}
+    ),
+    search_union AS (
+        SELECT * FROM search_unfiltered
+        UNION ALL
+        SELECT * FROM search_filtered
+    )
+    SELECT category, sum(value) AS value
+    FROM search_union
+    GROUP BY category
+    ORDER BY value desc
+`;
+
+const CATEGORIES_LIMIT = 6;
+
+const VALID_OPERATIONS = {
     count: [],
-    sum: ['aggregationColumn']
+    sum: ['aggregationColumn'],
+    avg: ['aggregationColumn'],
+    min: ['aggregationColumn'],
+    max: ['aggregationColumn']
 };
 
-var TYPE = 'aggregation';
+const TYPE = 'aggregation';
 
 /**
  {
@@ -68,211 +191,160 @@ var TYPE = 'aggregation';
      }
  }
  */
-function Aggregation(query, options) {
-    if (!_.isString(options.column)) {
-        throw new Error('Aggregation expects `column` in widget options');
+module.exports = class Aggregation extends BaseDataview {
+    constructor (query, options = {}, queries = {}) {
+        super();
+
+        this._checkOptions(options);
+
+        this.query = query;
+        this.queries = queries;
+        this.column = options.column;
+        this.aggregation = options.aggregation;
+        this.aggregationColumn = options.aggregationColumn;
+        this._isFloatColumn = null;
     }
 
-    if (!_.isString(options.aggregation)) {
-        throw new Error('Aggregation expects `aggregation` operation in widget options');
-    }
-
-    if (!VALID_OPERATIONS[options.aggregation]) {
-        throw new Error("Aggregation does not support '" + options.aggregation + "' operation");
-    }
-
-    var requiredOptions = VALID_OPERATIONS[options.aggregation];
-    var missingOptions = _.difference(requiredOptions, Object.keys(options));
-    if (missingOptions.length > 0) {
-        throw new Error(
-            "Aggregation '" + options.aggregation + "' is missing some options: " + missingOptions.join(',')
-        );
-    }
-
-    BaseWidget.apply(this);
-
-    this.query = query;
-    this.column = options.column;
-    this.aggregation = options.aggregation;
-    this.aggregationColumn = options.aggregationColumn;
-}
-
-Aggregation.prototype = new BaseWidget();
-Aggregation.prototype.constructor = Aggregation;
-
-module.exports = Aggregation;
-
-Aggregation.prototype.sql = function(psql, filters, override, callback) {
-    if (!callback) {
-        callback = override;
-        override = {};
-    }
-
-    var _query = this.query;
-
-    var aggregationSql;
-    if (!!override.ownFilter) {
-        aggregationSql = [
-            "WITH",
-            [
-                summaryQueryTpl({
-                    _query: _query,
-                    _column: this.column
-                }),
-                rankedCategoriesQueryTpl({
-                    _query: _query,
-                    _column: this.column,
-                    _aggregation: this.getAggregationSql()
-                }),
-                categoriesSummaryQueryTpl({
-                    _query: _query,
-                    _column: this.column
-                })
-            ].join(',\n'),
-            aggregationQueryTpl({
-                _query: _query,
-                _column: this.column,
-                _aggregation: this.getAggregationSql(),
-                _limit: CATEGORIES_LIMIT
-            })
-        ].join('\n');
-    } else {
-        aggregationSql = [
-            "WITH",
-            [
-                summaryQueryTpl({
-                    _query: _query,
-                    _column: this.column
-                }),
-                rankedCategoriesQueryTpl({
-                    _query: _query,
-                    _column: this.column,
-                    _aggregation: this.getAggregationSql()
-                }),
-                categoriesSummaryQueryTpl({
-                    _query: _query,
-                    _column: this.column
-                })
-            ].join(',\n'),
-            rankedAggregationQueryTpl({
-                _query: _query,
-                _column: this.column,
-                _limit: CATEGORIES_LIMIT
-            })
-        ].join('\n');
-    }
-
-    debug(aggregationSql);
-
-    return callback(null, aggregationSql);
-};
-
-var aggregationFnQueryTpl = dot.template('{{=it._aggregationFn}}({{=it._aggregationColumn}})');
-Aggregation.prototype.getAggregationSql = function() {
-    return aggregationFnQueryTpl({
-        _aggregationFn: this.aggregation,
-        _aggregationColumn: this.aggregationColumn || 1
-    });
-};
-
-Aggregation.prototype.format = function(result) {
-    var categories = [];
-    var count = 0;
-    var nulls = 0;
-    var minValue = 0;
-    var maxValue = 0;
-    var categoriesCount = 0;
-
-
-    if (result.rows.length) {
-        var firstRow = result.rows[0];
-        count = firstRow.count;
-        nulls = firstRow.nulls_count;
-        minValue = firstRow.min_val;
-        maxValue = firstRow.max_val;
-        categoriesCount = firstRow.categories_count;
-
-        result.rows.forEach(function(row) {
-            categories.push(_.omit(row, 'count', 'nulls_count', 'min_val', 'max_val', 'categories_count'));
-        });
-    }
-
-    return {
-        count: count,
-        nulls: nulls,
-        min: minValue,
-        max: maxValue,
-        categoriesCount: categoriesCount,
-        categories: categories
-    };
-};
-
-var filterCategoriesQueryTpl = dot.template([
-    'SELECT {{=it._column}} AS category, {{=it._value}} AS value',
-    'FROM ({{=it._query}}) _cdb_aggregation_search',
-    'WHERE CAST({{=it._column}} as text) ILIKE {{=it._userQuery}}',
-    'GROUP BY {{=it._column}}'
-].join('\n'));
-
-var searchQueryTpl = dot.template([
-    'WITH',
-    'search_unfiltered AS (',
-    '  {{=it._searchUnfiltered}}',
-    '),',
-    'search_filtered AS (',
-    '  {{=it._searchFiltered}}',
-    '),',
-    'search_union AS (',
-    '  SELECT * FROM search_unfiltered',
-    '  UNION ALL',
-    '  SELECT * FROM search_filtered',
-    ')',
-    'SELECT category, sum(value) AS value',
-    'FROM search_union',
-    'GROUP BY category',
-    'ORDER BY value desc'
-].join('\n'));
-
-
-Aggregation.prototype.search = function(psql, userQuery, callback) {
-    var self = this;
-
-    var _userQuery = psql.escapeLiteral('%' + userQuery + '%');
-
-    // TODO unfiltered will be wrong as filters are already applied at this point
-    var query = searchQueryTpl({
-        _searchUnfiltered: filterCategoriesQueryTpl({
-            _query: this.query,
-            _column: this.column,
-            _value: '0',
-            _userQuery: _userQuery
-        }),
-        _searchFiltered: filterCategoriesQueryTpl({
-            _query: this.query,
-            _column: this.column,
-            _value: 'count(1)',
-            _userQuery: _userQuery
-        })
-    });
-
-    psql.query(query, function(err, result) {
-        if (err) {
-            return callback(err, result);
+    _checkOptions (options) {
+        if (typeof options.column !== 'string') {
+            throw new Error(`Aggregation expects 'column' in dataview options`);
         }
 
-        return callback(null, {type: self.getType(), categories: result.rows });
-    }, true); // use read-only transaction
-};
+        if (typeof options.aggregation !== 'string') {
+            throw new Error(`Aggregation expects 'aggregation' operation in dataview options`);
+        }
 
-Aggregation.prototype.getType = function() {
-    return TYPE;
-};
+        if (!VALID_OPERATIONS[options.aggregation]) {
+            throw new Error(`Aggregation does not support '${options.aggregation}' operation`);
+        }
 
-Aggregation.prototype.toString = function() {
-    return JSON.stringify({
-        _type: TYPE,
-        _query: this.query,
-        _column: this.column,
-        _aggregation: this.aggregation
-    });
+        const requiredOptions = VALID_OPERATIONS[options.aggregation];
+        const missingOptions = requiredOptions.filter(requiredOption => !options.hasOwnProperty(requiredOption));
+
+        if (missingOptions.length > 0) {
+            throw new Error(
+                `Aggregation '${options.aggregation}' is missing some options: ${missingOptions.join(',')}`
+            );
+        }
+    }
+
+    sql (psql, override, callback) {
+        if (!callback) {
+            callback = override;
+            override = {};
+        }
+
+        if (this._shouldCheckColumnType()) {
+            this._isFloatColumn = false;
+            this.getColumnType(psql, this.aggregationColumn, this.queries.no_filters, (err, type) => {
+                if (!err && !!type) {
+                    this._isFloatColumn = type.float;
+                }
+                this.sql(psql, override, callback);
+            });
+            return null;
+        }
+
+        const limit = Number.isFinite(override.categories) && override.categories > 0 ?
+            override.categories :
+            CATEGORIES_LIMIT;
+
+        const aggregationSql = aggregationDataviewQueryTpl({
+            override: override,
+            query: this.query,
+            column: this.column,
+            aggregation: this.aggregation,
+            aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null,
+            aggregationFn: aggregationFnQueryTpl({
+                aggregation: this.aggregation,
+                aggregationColumn: this.aggregationColumn || 1
+            }),
+            isFloatColumn: this._isFloatColumn,
+            limit
+        });
+
+        debug(aggregationSql);
+
+        return callback(null, aggregationSql);
+    }
+
+    _shouldCheckColumnType () {
+        return this.aggregationColumn && this._isFloatColumn === null;
+    }
+
+    format (result) {
+        const {
+            count = 0,
+            nulls_count = 0,
+            nans_count = 0,
+            infinities_count = 0,
+            min_val = 0,
+            max_val = 0,
+            categories_count = 0
+         } = result.rows[0] || {};
+        
+        return {
+            aggregation: this.aggregation,
+            count: count,
+            nulls: nulls_count,
+            nans: nans_count,
+            infinities: infinities_count,
+            min: min_val,
+            max: max_val,
+            categoriesCount: categories_count,
+            categories: result.rows.map(({ category, value, agg }) => {
+                return { 
+                    category: agg ? 'Other' : category,
+                    value, 
+                    agg 
+                };
+            })
+        };
+    }
+
+    search (psql, userQuery, callback) {
+        const escapedUserQuery = psql.escapeLiteral(`%${userQuery}%`);
+        const value = this.aggregation !== 'count' && this.aggregationColumn ?
+            `${this.aggregation}(${this.aggregationColumn})` :
+            'count(1)';
+
+        // TODO unfiltered will be wrong as filters are already applied at this point
+        const query = searchQueryTpl({
+            searchUnfiltered: filterCategoriesQueryTpl({
+                query: this.query,
+                column: this.column,
+                value: '0',
+                userQuery: escapedUserQuery
+            }),
+            searchFiltered: filterCategoriesQueryTpl({
+                query: this.query,
+                column: this.column,
+                value: value,
+                userQuery: escapedUserQuery
+            })
+        });
+
+        debug(query);
+
+        psql.query(query, (err, result) => {
+            if (err) {
+                return callback(err, result);
+            }
+
+            return callback(null, {type: this.getType(), categories: result.rows });
+        }, true); // use read-only transaction
+    }
+
+    getType () {
+        return TYPE;
+    }
+
+    toString () {
+        return JSON.stringify({
+            _type: TYPE,
+            _query: this.query,
+            _column: this.column,
+            _aggregation: this.aggregation
+        });
+    }
 };

lib/cartodb/models/dataview/base.js

@@ -1,26 +1,59 @@
-function BaseDataview() {}
+const FLOAT_OIDS = {
+    700: true,
+    701: true,
+    1700: true
+};
 
-module.exports = BaseDataview;
+const DATE_OIDS = {
+    1082: true,
+    1114: true,
+    1184: true
+};
 
-BaseDataview.prototype.getResult = function(psql, override, callback) {
-    var self = this;
-    this.sql(psql, override, function(err, query) {
-        psql.query(query, function(err, result) {
+const columnTypeQueryTpl = ctx => `SELECT pg_typeof(${ctx.column})::oid FROM (${ctx.query}) _cdb_column_type limit 1`;
 
+function getPGTypeName (pgType) {
+    return {
+        float: FLOAT_OIDS.hasOwnProperty(pgType),
+        date: DATE_OIDS.hasOwnProperty(pgType)
+    };
+}
+
+module.exports = class BaseDataview {
+    getResult (psql, override, callback) {
+        this.sql(psql, override, (err, query) => {
             if (err) {
-                return callback(err, result);
+                return callback(err);
             }
 
-            result = self.format(result, override);
-            result.type = self.getType();
+            psql.query(query, (err, result) => {
+                if (err) {
+                    return callback(err, result);
+                }
 
-            return callback(null, result);
+                result = this.format(result, override);
+                result.type = this.getType();
 
-        }, true); // use read-only transaction
-    });
+                return callback(null, result);
 
-};
-
-BaseDataview.prototype.search = function(psql, userQuery, callback) {
-    return callback(null, this.format({ rows: [] }));
+            }, true); // use read-only transaction
+        });
+    }
+
+    search (psql, userQuery, callback) {
+        return callback(null, this.format({ rows: [] }));
+    }
+
+    getColumnType (psql, column, query, callback) {
+        const readOnlyTransaction = true;
+        const columnTypeQuery = columnTypeQueryTpl({ column, query });
+
+        psql.query(columnTypeQuery, (err, result) => {
+            if (err) {
+                return callback(err);
+            }
+            const pgType = result.rows[0].pg_typeof;
+            callback(null, getPGTypeName(pgType));
+        }, readOnlyTransaction);
+    }
 };

lib/cartodb/models/dataview/factory.js

@@ -1,18 +1,20 @@
-var dataviews = require('./');
+const dataviews = require('./');
 
-var DataviewFactory = {
-    dataviews: Object.keys(dataviews).reduce(function(allDataviews, dataviewClassName) {
-        allDataviews[dataviewClassName.toLowerCase()] = dataviews[dataviewClassName];
-        return allDataviews;
-    }, {}),
+module.exports = class DataviewFactory {
+    static get dataviews() {
+        return Object.keys(dataviews).reduce((allDataviews, dataviewClassName) => {
+            allDataviews[dataviewClassName.toLowerCase()] = dataviews[dataviewClassName];
+            return allDataviews;
+        }, {});
+    }
+
+    static getDataview (query, dataviewDefinition) {
+        const { type, options, sql } = dataviewDefinition;
 
-    getDataview: function(query, dataviewDefinition) {
-        var type = dataviewDefinition.type;
         if (!this.dataviews[type]) {
             throw new Error('Invalid dataview type: "' + type + '"');
         }
-        return new this.dataviews[type](query, dataviewDefinition.options);
+
+        return new this.dataviews[type](query, options, sql);
     }
 };
-
-module.exports = DataviewFactory;

lib/cartodb/models/dataview/formula.js

@@ -1,18 +1,16 @@
-var _ = require('underscore');
-var BaseWidget = require('./base');
-var debug = require('debug')('windshaft:widget:formula');
+const BaseDataview = require('./base');
+const debug = require('debug')('windshaft:dataview:formula');
+const utils = require('../../utils/query-utils');
 
-var dot = require('dot');
-dot.templateSettings.strip = false;
+const formulaQueryTpl = ctx =>
+`SELECT
+    ${ctx.operation}(${utils.handleFloatColumn(ctx)}) AS result,
+    ${utils.countNULLs(ctx)} AS nulls_count
+    ${ctx.isFloatColumn ? `,${utils.countInfinites(ctx)} AS infinities_count,` : ``}
+    ${ctx.isFloatColumn ? `${utils.countNaNs(ctx)} AS nans_count` : ``}
+FROM (${ctx.query}) __cdb_formula`;
 
-var formulaQueryTpl = dot.template([
-    'SELECT',
-    '{{=it._operation}}({{=it._column}}) AS result,',
-    '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
-    'FROM ({{=it._query}}) _cdb_formula'
-].join('\n'));
-
-var VALID_OPERATIONS = {
+const VALID_OPERATIONS = {
     count: true,
     avg: true,
     sum: true,
@@ -20,7 +18,7 @@ var VALID_OPERATIONS = {
     max: true
 };
 
-var TYPE = 'formula';
+const TYPE = 'formula';
 
 /**
  {
@@ -31,74 +29,90 @@ var TYPE = 'formula';
      }
  }
  */
-function Formula(query, options) {
-    if (!_.isString(options.operation)) {
-        throw new Error('Formula expects `operation` in widget options');
+module.exports = class Formula extends BaseDataview {
+    constructor (query, options = {}, queries = {}) {
+        super();
+
+        this._checkOptions(options);
+
+        this.query = query;
+        this.queries = queries;
+        this.column = options.column || '1';
+        this.operation = options.operation;
+        this._isFloatColumn = null;
     }
 
-    if (!VALID_OPERATIONS[options.operation]) {
-        throw new Error("Formula does not support '" + options.operation + "' operation");
+    _checkOptions (options) {
+        if (typeof options.operation !== 'string') {
+            throw new Error(`Formula expects 'operation' in dataview options`);
+        }
+
+        if (!VALID_OPERATIONS[options.operation]) {
+            throw new Error(`Formula does not support '${options.operation}' operation`);
+        }
+
+        if (options.operation !== 'count' && typeof options.column !== 'string') {
+            throw new Error(`Formula expects 'column' in dataview options`);
+        }
     }
 
-    if (options.operation !== 'count' && !_.isString(options.column)) {
-        throw new Error('Formula expects `column` in widget options');
+
+    sql (psql, override, callback) {
+        if (!callback) {
+            callback = override;
+            override = {};
+        }
+
+        if (this._isFloatColumn === null) {
+            this._isFloatColumn = false;
+            this.getColumnType(psql, this.column, this.queries.no_filters, (err, type) => {
+                if (!err && !!type) {
+                    this._isFloatColumn = type.float;
+                }
+                this.sql(psql, override, callback);
+            });
+            return null;
+        }
+
+        const formulaSql = formulaQueryTpl({
+            isFloatColumn: this._isFloatColumn,
+            query: this.query,
+            operation: this.operation,
+            column: this.column
+        });
+
+        debug(formulaSql);
+
+        return callback(null, formulaSql);
     }
 
-    BaseWidget.apply(this);
+    format (res) {
+        const {
+            result = 0,
+            nulls_count = 0,
+            nans_count,
+            infinities_count
+        } = res.rows[0] || {};
 
-    this.query = query;
-    this.column = options.column || '1';
-    this.operation = options.operation;
-}
-
-Formula.prototype = new BaseWidget();
-Formula.prototype.constructor = Formula;
-
-module.exports = Formula;
-
-Formula.prototype.sql = function(psql, filters, override, callback) {
-    if (!callback) {
-        callback = override;
-        override = {};
+        return {
+            operation: this.operation,
+            result,
+            nulls: nulls_count,
+            nans: nans_count,
+            infinities: infinities_count
+        };
     }
 
-    var _query = this.query;
-    var formulaSql = formulaQueryTpl({
-        _query: _query,
-        _operation: this.operation,
-        _column: this.column
-    });
+    getType () {
+        return TYPE;
+    }
 
-    debug(formulaSql);
-
-    return callback(null, formulaSql);
-};
-
-Formula.prototype.format = function(result) {
-    var formattedResult = {
-        operation: this.operation,
-        result: 0,
-        nulls: 0
-    };
-
-    if (result.rows.length) {
-        formattedResult.operation = this.operation;
-        formattedResult.result = result.rows[0].result;
-        formattedResult.nulls = result.rows[0].nulls_count;
-    }
-
-    return formattedResult;
-};
-
-Formula.prototype.getType = function() {
-    return TYPE;
-};
-
-Formula.prototype.toString = function() {
-    return JSON.stringify({
-        _type: TYPE,
-        _query: this.query,
-        _column: this.column,
-        _operation: this.operation
-    });
+    toString () {
+        return JSON.stringify({
+            _type: TYPE,
+            _query: this.query,
+            _column: this.column,
+            _operation: this.operation
+        });
+    }
 };

lib/cartodb/models/dataview/histogram.js

@@ -1,297 +1,72 @@
-var _ = require('underscore');
-var BaseWidget = require('./base');
-var debug = require('debug')('windshaft:dataview:histogram');
+const debug = require('debug')('windshaft:dataview:histogram');
+const NumericHistogram = require('./histograms/numeric-histogram');
+const DateHistogram = require('./histograms/date-histogram');
 
-var dot = require('dot');
-dot.templateSettings.strip = false;
+const DATE_HISTOGRAM = 'DateHistogram';
+const NUMERIC_HISTOGRAM = 'NumericHistogram';
 
-var columnTypeQueryTpl = dot.template(
-    'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1'
-);
-var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})");
+module.exports = class Histogram {
+    constructor (query, options, queries) {
+        this.query = query;
+        this.options = options || {};
+        this.queries = queries;
 
-var BIN_MIN_NUMBER = 6;
-var BIN_MAX_NUMBER = 48;
-
-var basicsQueryTpl = dot.template([
-    'basics AS (',
-    '  SELECT',
-    '    max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,',
-    '    avg({{=it._column}}) AS avg_val, count(1) AS total_rows',
-    '  FROM ({{=it._query}}) _cdb_basics',
-    ')'
-].join(' \n'));
-
-var overrideBasicsQueryTpl = dot.template([
-    'basics AS (',
-    '  SELECT',
-    '    max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,',
-    '    avg({{=it._column}}) AS avg_val, count(1) AS total_rows',
-    '  FROM ({{=it._query}}) _cdb_basics',
-    ')'
-].join('\n'));
-
-var iqrQueryTpl = dot.template([
-    'iqrange AS (',
-    '  SELECT max(quartile_max) - min(quartile_max) AS iqr',
-    '  FROM (',
-    '    SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (',
-    '      SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}',
-    '    ) AS quartile',
-    '    FROM ({{=it._query}}) _cdb_rank) _cdb_quartiles',
-    '    WHERE quartile = 1 or quartile = 3',
-    '    GROUP BY quartile',
-    '  ) _cdb_iqr',
-    ')'
-].join('\n'));
-
-var binsQueryTpl = dot.template([
-    'bins AS (',
-    '  SELECT CASE WHEN total_rows = 0 OR iqr = 0',
-    '      THEN 1',
-    '      ELSE GREATEST(',
-    '        LEAST({{=it._minBins}}, CAST(total_rows AS INT)),',
-    '        LEAST(',
-    '          CAST(((max_val - min_val) / (2 * iqr * power(total_rows, 1/3))) AS INT),',
-    '          {{=it._maxBins}}',
-    '        )',
-    '      )',
-    '    END AS bins_number',
-    '  FROM basics, iqrange, ({{=it._query}}) _cdb_bins',
-    '  LIMIT 1',
-    ')'
-].join('\n'));
-
-var overrideBinsQueryTpl = dot.template([
-    'bins AS (',
-    '  SELECT {{=it._bins}} AS bins_number',
-    ')'
-].join('\n'));
-
-var nullsQueryTpl = dot.template([
-    'nulls AS (',
-    '  SELECT',
-    '  count(*) AS nulls_count',
-    '  FROM ({{=it._query}}) _cdb_histogram_nulls',
-    '  WHERE {{=it._column}} IS NULL',
-    ')'
-].join('\n'));
-
-var histogramQueryTpl = dot.template([
-    'SELECT',
-    '    (max_val - min_val) / cast(bins_number as float) AS bin_width,',
-    '    bins_number,',
-    '    nulls_count,',
-    '    avg_val,',
-    '    CASE WHEN min_val = max_val',
-    '      THEN 0',
-    '      ELSE GREATEST(1, LEAST(WIDTH_BUCKET({{=it._column}}, min_val, max_val, bins_number), bins_number)) - 1',
-    '    END AS bin,',
-    '    min({{=it._column}})::numeric AS min,',
-    '    max({{=it._column}})::numeric AS max,',
-    '    avg({{=it._column}})::numeric AS avg,',
-    '    count(*) AS freq',
-    'FROM ({{=it._query}}) _cdb_histogram, basics, nulls, bins',
-    'WHERE {{=it._column}} IS NOT NULL',
-    'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val',
-    'ORDER BY bin'
-].join('\n'));
-
-
-var TYPE = 'histogram';
-
-/**
- {
-     type: 'histogram',
-     options: {
-         column: 'name',
-         bins: 10 // OPTIONAL
-     }
- }
- */
-function Histogram(query, options) {
-    if (!_.isString(options.column)) {
-        throw new Error('Histogram expects `column` in widget options');
+        this.histogramImplementation = this._getHistogramImplementation();
     }
 
-    this.query = query;
-    this.column = options.column;
-    this.bins = options.bins;
+    _getHistogramImplementation (override) {
+        let implementation = null;
 
-    this._columnType = null;
-}
-
-Histogram.prototype = new BaseWidget();
-Histogram.prototype.constructor = Histogram;
-
-module.exports = Histogram;
-
-var DATE_OIDS = {
-    1082: true,
-    1114: true,
-    1184: true
-};
-
-Histogram.prototype.sql = function(psql, override, callback) {
-    if (!callback) {
-        callback = override;
-        override = {};
-    }
-
-    var self = this;
-
-    var _column = this.column;
-
-    var columnTypeQuery = columnTypeQueryTpl({
-        column: _column, query: this.query
-    });
-
-    if (this._columnType === null) {
-        psql.query(columnTypeQuery, function(err, result) {
-            // assume numeric, will fail later
-            self._columnType = 'numeric';
-            if (!err && !!result.rows[0]) {
-                var pgType = result.rows[0].pg_typeof;
-                if (DATE_OIDS.hasOwnProperty(pgType)) {
-                    self._columnType = 'date';
-                }
-            }
-            self.sql(psql, override, callback);
-        }, true); // use read-only transaction
-        return null;
-    }
-
-    if (this._columnType === 'date') {
-        _column = columnCastTpl({column: _column});
-    }
-
-    var _query = this.query;
-
-    var basicsQuery, binsQuery;
-
-    if (override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins')) {
-        debug('overriding with %j', override);
-        basicsQuery = overrideBasicsQueryTpl({
-            _query: _query,
-            _column: _column,
-            _start: override.start,
-            _end: override.end
-        });
-
-        binsQuery = [
-            overrideBinsQueryTpl({
-                _bins: override.bins
-            })
-        ].join(',\n');
-    } else {
-        basicsQuery = basicsQueryTpl({
-            _query: _query,
-            _column: _column
-        });
-
-        if (override && _.has(override, 'bins')) {
-            binsQuery = [
-                overrideBinsQueryTpl({
-                    _bins: override.bins
-                })
-            ].join(',\n');
-        } else {
-            binsQuery = [
-                iqrQueryTpl({
-                    _query: _query,
-                    _column: _column
-                }),
-                binsQueryTpl({
-                    _query: _query,
-                    _minBins: BIN_MIN_NUMBER,
-                    _maxBins: BIN_MAX_NUMBER
-                })
-            ].join(',\n');
+        switch (this._getHistogramSubtype(override)) {
+            case DATE_HISTOGRAM:
+                debug('Delegating to DateHistogram with options: %j and overriding: %j', this.options, override);
+                implementation = new DateHistogram(this.query, this.options, this.queries);
+                break;
+            case NUMERIC_HISTOGRAM:
+                debug('Delegating to NumericHistogram with options: %j and overriding: %j', this.options, override);
+                implementation = new NumericHistogram(this.query, this.options, this.queries);
+                break;
+            default:
+                throw new Error('Unsupported Histogram type');
         }
+
+        return implementation;
     }
 
+    _getHistogramSubtype (override) {
+        if(this._isDateHistogram(override)) {
+            return DATE_HISTOGRAM;
+        }
 
-    var histogramSql = [
-        "WITH",
-        [
-            basicsQuery,
-            binsQuery,
-            nullsQueryTpl({
-                _query: _query,
-                _column: _column
-            })
-        ].join(',\n'),
-        histogramQueryTpl({
-            _query: _query,
-            _column: _column
-        })
-    ].join('\n');
-
-    debug(histogramSql);
-
-    return callback(null, histogramSql);
-};
-
-Histogram.prototype.format = function(result, override) {
-    override = override || {};
-    var buckets = [];
-
-    var binsCount = getBinsCount(override);
-    var width = getWidth(override);
-    var binsStart = getBinStart(override);
-    var nulls = 0;
-    var avg;
-
-    if (result.rows.length) {
-        var firstRow = result.rows[0];
-        binsCount = firstRow.bins_number;
-        width = firstRow.bin_width || width;
-        avg = firstRow.avg_val;
-        nulls = firstRow.nulls_count;
-        binsStart = override.hasOwnProperty('start') ? override.start : firstRow.min;
-
-        buckets = result.rows.map(function(row) {
-            return _.omit(row, 'bins_number', 'bin_width', 'nulls_count', 'avg_val');
-        });
+        return NUMERIC_HISTOGRAM;
     }
 
-    return {
-        bin_width: width,
-        bins_count: binsCount,
-        bins_start: binsStart,
-        nulls: nulls,
-        avg: avg,
-        bins: buckets
-    };
-};
-
-function getBinStart(override) {
-    return override.start || 0;
-}
-
-function getBinsCount(override) {
-    return override.bins || 0;
-}
-
-function getWidth(override) {
-    var width = 0;
-    var binsCount = override.bins;
-
-    if (binsCount && Number.isFinite(override.start) && Number.isFinite(override.end)) {
-        width = (override.end - override.start) / binsCount;
+    _isDateHistogram (override = {}) {
+        return (this.options.hasOwnProperty('aggregation') || override.hasOwnProperty('aggregation'));
     }
 
-    return width;
-}
+    getResult (psql, override, callback) {
+        this.histogramImplementation = this._getHistogramImplementation(override);
+        this.histogramImplementation.getResult(psql, override, callback);
+    }
 
-Histogram.prototype.getType = function() {
-    return TYPE;
-};
+    // In order to keep previous behaviour with overviews,
+    // we have to expose the following methods to bypass
+    // the concrete overview implementation
 
-Histogram.prototype.toString = function() {
-    return JSON.stringify({
-        _type: TYPE,
-        _column: this.column,
-        _query: this.query
-    });
+    sql (psql, override, callback) {
+        this.histogramImplementation.sql(psql, override, callback);
+    }
+
+    format (result, override) {
+        return this.histogramImplementation.format(result, override);
+    }
+
+    getType () {
+        return this.histogramImplementation.getType();
+    }
+
+    toString () {
+        return this.histogramImplementation.toString();
+    }
 };

lib/cartodb/models/dataview/histograms/base-histogram.js

@@ -0,0 +1,85 @@
+const BaseDataview = require('../base');
+
+const TYPE = 'histogram';
+
+module.exports = class BaseHistogram extends BaseDataview {
+    constructor (query, options, queries) {
+        super();
+
+        if (typeof options.column !== 'string') {
+            throw new Error('Histogram expects `column` in widget options');
+        }
+
+        this.query = query;
+        this.queries = queries;
+        this.column = options.column;
+        this.bins = options.bins;
+
+        this._columnType = null;
+    }
+
+    sql (psql, override, callback) {
+        if (!callback) {
+            callback = override;
+            override = {};
+        }
+
+        if (this._columnType === null) {
+            this.getColumnType(psql, this.column, this.queries.no_filters, (err, type) => {
+                // assume numeric, will fail later
+                this._columnType = 'numeric';
+                if (!err && !!type) {
+                    this._columnType = Object.keys(type).find(function (key) {
+                        return type[key];
+                    });
+                }
+                this.sql(psql, override, callback);
+            }, true); // use read-only transaction
+            return null;
+        }
+
+        return this._buildQuery(psql, override, callback);
+    }
+
+    format (result, override) {
+        const histogram = this._getSummary(result, override);
+        histogram.bins = this._getBuckets(result);
+        return histogram;
+    }
+
+    getType () {
+        return TYPE;
+    }
+
+    toString () {
+        return JSON.stringify({
+            _type: TYPE,
+            _column: this.column,
+            _query: this.query
+        });
+    }
+
+    _hasOverridenRange (override) {
+        return override && override.hasOwnProperty('start') && override.hasOwnProperty('end');
+    }
+
+    _getBinStart (override = {}) {
+        if (this._hasOverridenRange(override)) {
+            return Math.min(override.start, override.end);
+        }
+
+        return override.start || 0;
+    }
+
+    _getBinEnd (override = {}) {
+        if (this._hasOverridenRange(override)) {
+            return Math.max(override.start, override.end);
+        }
+
+        return override.end || 0;
+    }
+
+    _getBinsCount (override = {}) {
+        return override.bins || 0;
+    }
+};

lib/cartodb/models/dataview/histograms/date-histogram.js

@@ -0,0 +1,322 @@
+const BaseHistogram = require('./base-histogram');
+const debug = require('debug')('windshaft:dataview:date-histogram');
+const utils = require('../../../utils/query-utils');
+
+/**
+ * Gets the name of a timezone with the same offset as the required
+ * using the pg_timezone_names table. We do this because it's simpler to pass
+ * the name than to pass the offset itself as PostgreSQL uses different
+ * sign convention. For example: TIME ZONE 'CET' is equal to TIME ZONE 'UTC-1',
+ * not 'UTC+1' which would be expected.
+ * Gives priority to Etc/GMT±N timezones but still support odd offsets like 8.5
+ * hours for Asia/Pyongyang.
+ * It also makes it easier to, in the future, support the input of expected timezone
+ * instead of the offset; that is using 'Europe/Madrid' instead of
+ * '+3600' or '+7200'. The daylight saving status can be handled by postgres.
+ */
+const offsetNameQueryTpl = ctx => `
+WITH __wd_tz AS
+(
+    SELECT name
+    FROM pg_timezone_names
+    WHERE utc_offset = interval '${ctx.offset} hours'
+    ORDER BY CASE WHEN name LIKE 'Etc/GMT%' THEN 0 ELSE 1 END
+    LIMIT 1
+),`;
+
+/**
+ * Function to get the subquery that places each row in its bin depending on
+ * the aggregation. Since the data stored is in epoch we need to adapt it to
+ * our timezone so when calling date_trunc it falls into the correct bin
+ */
+function dataBucketsQuery(ctx) {
+    var condition_str = '';
+
+    if (ctx.start !== 0) {
+        condition_str = `WHERE ${ctx.column} >= to_timestamp(${ctx.start})`;
+    }
+    if (ctx.end !== 0) {
+        if (condition_str === '') {
+            condition_str = `WHERE ${ctx.column} <= to_timestamp(${ctx.end})`;
+        }
+        else {
+            condition_str += ` and ${ctx.column} <= to_timestamp(${ctx.end})`;
+        }
+    }
+
+    return `
+__wd_buckets AS
+(
+    SELECT
+        date_trunc('${ctx.aggregation}', timezone(__wd_tz.name, ${ctx.column}::timestamptz)) as timestamp,
+        count(*) as freq,
+        ${utils.countNULLs(ctx)} as nulls_count
+    FROM
+    (
+        ${ctx.query}
+    ) __source, __wd_tz
+    ${condition_str}
+    GROUP BY 1, __wd_tz.name
+),`;
+}
+
+/**
+ * Function that generates an array with all the possible bins between the
+ * start and end date. If not provided we use the min and max generated from
+ * the dataBucketsQuery
+ */
+function allBucketsArrayQuery(ctx) {
+    var extra_from = ``;
+    var series_start = ``;
+    var series_end = ``;
+
+    if (ctx.start === 0) {
+        extra_from = `, __wd_buckets GROUP BY __wd_tz.name`;
+        series_start = `min(__wd_buckets.timestamp)`;
+    } else {
+        series_start = `date_trunc('${ctx.aggregation}', timezone(__wd_tz.name, to_timestamp(${ctx.start})))`;
+    }
+
+    if (ctx.end === 0) {
+        extra_from = `, __wd_buckets GROUP BY __wd_tz.name`;
+        series_end = `max(__wd_buckets.timestamp)`;
+    } else {
+        series_end = `date_trunc('${ctx.aggregation}', timezone(__wd_tz.name, to_timestamp(${ctx.end})))`;
+    }
+
+    return `
+__wd_all_buckets AS
+(
+    SELECT ARRAY(
+        SELECT
+            generate_series(
+                    ${series_start},
+                    ${series_end},
+                    interval '${ctx.interval}') as bin_start
+            FROM __wd_tz${extra_from}
+    ) as bins
+)`;
+}
+
+const dateIntervalQueryTpl = ctx => `
+    WITH
+    __cdb_dates AS (
+        SELECT
+            MAX(${ctx.column}::timestamp) AS __cdb_end,
+            MIN(${ctx.column}::timestamp) AS __cdb_start
+        FROM (${ctx.query}) __cdb_source
+    ),
+    __cdb_interval_in_days AS (
+        SELECT
+            DATE_PART('day', __cdb_end - __cdb_start) AS __cdb_days
+        FROM __cdb_dates
+    ),
+    __cdb_interval_in_hours AS (
+        SELECT
+            __cdb_days * 24 + DATE_PART('hour', __cdb_end - __cdb_start) AS __cdb_hours
+        FROM __cdb_interval_in_days, __cdb_dates
+    ),
+    __cdb_interval_in_minutes AS (
+        SELECT
+            __cdb_hours * 60 + DATE_PART('minute', __cdb_end - __cdb_start) AS __cdb_minutes
+        FROM __cdb_interval_in_hours, __cdb_dates
+    ),
+    __cdb_interval_in_seconds AS (
+        SELECT
+            __cdb_minutes * 60 + DATE_PART('second', __cdb_end - __cdb_start) AS __cdb_seconds
+        FROM __cdb_interval_in_minutes, __cdb_dates
+    )
+    SELECT
+        ROUND(__cdb_days / 365243) AS millennium,
+        ROUND(__cdb_days / 36525) AS century,
+        ROUND(__cdb_days / 3652) AS decade,
+        ROUND(__cdb_days / 365) AS year,
+        ROUND(__cdb_days / 91) AS quarter,
+        ROUND(__cdb_days / 30) AS month,
+        ROUND(__cdb_days / 7) AS week,
+        __cdb_days AS day,
+        __cdb_hours AS hour,
+        __cdb_minutes AS minute,
+        __cdb_seconds AS second
+    FROM __cdb_interval_in_days, __cdb_interval_in_hours, __cdb_interval_in_minutes, __cdb_interval_in_seconds
+`;
+
+/** Constant to switch between aggregations in auto mode */
+const MAX_INTERVAL_VALUE = 100;
+
+const DATE_AGGREGATIONS = {
+    'auto': true,
+    'second' : true,
+    'minute': true,
+    'hour': true,
+    'day': true,
+    'week': true,
+    'month': true,
+    'quarter': true,
+    'year': true,
+    'decade' : true,
+    'century' : true,
+    'millennium' : true
+};
+
+/**
+    date_histogram: {
+        type: 'histogram',
+        options: {
+            column: 'date', // column data type: date
+            aggregation: 'day' // MANDATORY
+            offset: -7200 // OPTIONAL (UTC offset in seconds)
+        }
+    }
+*/
+module.exports = class DateHistogram extends BaseHistogram {
+    constructor (query, options, queries) {
+        super(query, options, queries);
+
+        this.aggregation = options.aggregation;
+        this.offset = options.offset;
+    }
+
+    _buildQueryTpl (ctx) {
+        return `
+${offsetNameQueryTpl(ctx)}
+${dataBucketsQuery(ctx)}
+${allBucketsArrayQuery(ctx)}
+SELECT
+    array_position(__wd_all_buckets.bins, __wd_buckets.timestamp) - 1 as bin,
+    date_part('epoch', timezone(__wd_tz.name, __wd_buckets.timestamp)) AS timestamp,
+    __wd_buckets.freq as freq,
+    date_part('epoch', timezone(__wd_tz.name, (__wd_all_buckets.bins)[1])) as timestamp_start,
+    array_length(__wd_all_buckets.bins, 1) as bins_number,
+    date_part('epoch', interval '${ctx.interval}') as bin_width,
+    __wd_buckets.nulls_count as nulls_count
+FROM __wd_buckets, __wd_all_buckets, __wd_tz
+GROUP BY __wd_tz.name, __wd_all_buckets.bins, __wd_buckets.timestamp, __wd_buckets.nulls_count, __wd_buckets.freq
+ORDER BY bin ASC;
+`;
+    }
+
+    _buildQuery (psql, override, callback) {
+        if (!this._isValidAggregation(override)) {
+            return callback(new Error('Invalid aggregation value. Valid ones: ' +
+                Object.keys(DATE_AGGREGATIONS).join(', ')
+            ));
+        }
+
+        if (this._getAggregation(override) === 'auto') {
+            this._getAutomaticAggregation(psql, function (err, aggregation) {
+                if (err || aggregation === 'none') {
+                    this.aggregation = 'day';
+                } else {
+                    this.aggregation = aggregation;
+                }
+                override.aggregation = this.aggregation;
+                this._buildQuery(psql, override, callback);
+            }.bind(this));
+            return null;
+        }
+
+        var interval = this._getAggregation(override) === 'quarter' ?
+            '3 months' : '1 ' + this._getAggregation(override);
+
+        const histogramSql = this._buildQueryTpl({
+            override: override,
+            query: this.query,
+            column: this.column,
+            aggregation: this._getAggregation(override),
+            start: this._getBinStart(override),
+            end: this._getBinEnd(override),
+            offset: this._parseOffset(override),
+            interval: interval
+        });
+
+        debug(histogramSql);
+
+        return callback(null, histogramSql);
+    }
+
+    _isValidAggregation (override) {
+        return DATE_AGGREGATIONS.hasOwnProperty(this._getAggregation(override));
+    }
+
+    _getAutomaticAggregation (psql, callback) {
+        const dateIntervalQuery = dateIntervalQueryTpl({
+            query: this.query,
+            column: this.column
+        });
+
+        psql.query(dateIntervalQuery, function (err, result) {
+            if (err) {
+                return callback(err);
+            }
+
+            const aggregations = result.rows[0];
+            const aggregation = Object.keys(aggregations)
+                .map(key => ({ name: key, value: aggregations[key] }))
+                .reduce((closer, current) => {
+                    if (current.value > MAX_INTERVAL_VALUE) {
+                        return closer;
+                    }
+
+                    const closerDiff = MAX_INTERVAL_VALUE - closer.value;
+                    const currentDiff = MAX_INTERVAL_VALUE - current.value;
+
+                    if (Number.isFinite(current.value) && closerDiff > currentDiff) {
+                        return current;
+                    }
+
+                    return closer;
+                }, { name: 'none', value: -1 });
+
+            callback(null, aggregation.name);
+        });
+    }
+
+    _getSummary (result, override) {
+        const firstRow = result.rows[0] || {};
+
+        return {
+            aggregation: this._getAggregation(override),
+            offset: this._getOffset(override),
+            timestamp_start: firstRow.timestamp_start,
+
+            bin_width: firstRow.bin_width || 0,
+            bins_count: firstRow.bins_number || 0,
+            bins_start: firstRow.timestamp,
+            nulls: firstRow.nulls_count,
+            infinities: firstRow.infinities_count,
+            nans: firstRow.nans_count,
+            avg: firstRow.avg_val
+        };
+    }
+
+    _getBuckets (result) {
+        result.rows.forEach(function(row) {
+            row.min = row.max = row.avg = row.timestamp;
+        });
+
+        return result.rows.map(({ bin, min, max, avg, freq, timestamp }) => ({ bin, min, max, avg, freq, timestamp }));
+    }
+
+    _getAggregation (override = {}) {
+        return override.aggregation ? override.aggregation : this.aggregation;
+    }
+
+    _getOffset (override = {}) {
+        return Number.isFinite(override.offset) ? override.offset : (this.offset || 0);
+    }
+
+    _parseOffset (override) {
+        if (this._shouldIgnoreOffset(override)) {
+            return '0';
+        }
+
+        const offsetInHours = Math.ceil(this._getOffset(override) / 3600);
+
+        return '' + offsetInHours;
+    }
+
+    _shouldIgnoreOffset (override) {
+        return (this._getAggregation(override) === 'hour' || this._getAggregation(override) === 'minute');
+    }
+};

lib/cartodb/models/dataview/histograms/numeric-histogram.js

@@ -0,0 +1,195 @@
+const BaseHistogram = require('./base-histogram');
+const debug = require('debug')('windshaft:dataview:numeric-histogram');
+const utils = require('../../../utils/query-utils');
+
+/** Query to get min and max values from the query */
+const irqQueryTpl = ctx => `
+    __cdb_filtered_source AS (
+        SELECT *
+        FROM (${ctx.query}) __cdb_filtered_source_query
+        WHERE ${utils.handleFloatColumn(ctx)} IS NOT NULL
+    ),
+    __cdb_basics AS (
+        SELECT
+            max(${ctx.column}) AS __cdb_max_val,
+            min(${ctx.column}) AS __cdb_min_val,
+            count(1) AS __cdb_total_rows
+        FROM __cdb_filtered_source
+    )
+`;
+
+/* Query to calculate the number of bins (needs irqQueryTpl before it*/
+const binsQueryTpl = ctx => `
+    __cdb_iqrange AS (
+        SELECT max(quartile_max) - min(quartile_max) AS __cdb_iqr
+        FROM (
+            SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (
+                SELECT ${ctx.column} AS _cdb_iqr_column, ntile(4) over (order by ${ctx.column}
+            ) AS quartile
+            FROM __cdb_filtered_source) _cdb_quartiles
+            WHERE quartile = 1 or quartile = 3
+            GROUP BY 1
+        ) __cdb_iqr
+    ),
+    __cdb_bins AS (
+        SELECT
+            CASE WHEN __cdb_total_rows = 0 OR __cdb_iqr = 0
+            THEN 1
+            ELSE GREATEST(
+                LEAST(${ctx.minBins}, CAST(__cdb_total_rows AS INT)),
+                LEAST(
+                    CAST(((__cdb_max_val - __cdb_min_val) / (2 * __cdb_iqr * power(__cdb_total_rows, 1/3))) AS INT),
+                    ${ctx.maxBins}
+                )
+            )
+            END AS __cdb_bins_number
+        FROM __cdb_basics, __cdb_iqrange, __cdb_filtered_source
+        LIMIT 1
+    )
+`;
+
+const BIN_MIN_NUMBER = 6;
+const BIN_MAX_NUMBER = 48;
+
+/**
+Numeric histogram:
+{
+    type: 'histogram',
+    options: {
+        column: 'name', // column data type: numeric
+        bins: 10 // OPTIONAL
+    }
+}
+*/
+module.exports = class NumericHistogram extends BaseHistogram {
+    constructor (query, options, queries) {
+        super(query, options, queries);
+    }
+
+    _buildQuery (psql, override, callback) {
+        const histogramSql = this._buildQueryTpl({
+            column: this._columnType === 'date' ? utils.columnCastTpl({ column: this.column }) : this.column,
+            isFloatColumn: this._columnType === 'float',
+            query: this.query,
+            start: this._getBinStart(override),
+            end: this._getBinEnd(override),
+            bins: this._getBinsCount(override),
+            minBins: BIN_MIN_NUMBER,
+            maxBins: BIN_MAX_NUMBER
+        });
+
+        debug(histogramSql);
+
+        return callback(null, histogramSql);
+    }
+
+
+/**
+ * ctx: Object with the following values
+ * ctx.column -- Column for the histogram
+ * ctx.isFloatColumn - Whether the column is float or not
+ * ctx.query -- Subquery to extract data
+ * ctx.start -- Start value for the bins. [>= end to force calculation]
+ * ctx.end -- End value for the bins.
+ * ctx.bins -- Numbers of bins to generate [<0 to force calculation]
+ * ctx.minBins - If !full min bins to calculate [Optional]
+ * ctx.maxBins - If !full max bins to calculate [Optional]
+ */
+    _buildQueryTpl (ctx) {
+        var extra_tables = ``;
+        var extra_queries = ``;
+        var extra_groupby = ``;
+
+        if (ctx.start >= ctx.end) {
+            ctx.end = `__cdb_basics.__cdb_max_val`;
+            ctx.start = `__cdb_basics.__cdb_min_val`;
+            extra_groupby = `, __cdb_basics.__cdb_max_val, __cdb_basics.__cdb_min_val`;
+            extra_tables = `, __cdb_basics`;
+            extra_queries = `WITH ${irqQueryTpl(ctx)}`;
+        }
+
+        if (ctx.bins <= 0) {
+            ctx.bins = `__cdb_bins.__cdb_bins_number`;
+            extra_groupby += `, __cdb_bins.__cdb_bins_number`;
+            extra_tables += `, __cdb_bins`;
+            extra_queries = `WITH ${irqQueryTpl(ctx)}, ${binsQueryTpl(ctx)}`;
+        }
+
+        return `
+${extra_queries}
+SELECT
+    (${ctx.end} - ${ctx.start}) / ${ctx.bins}::float AS bin_width,
+    ${ctx.bins} as bins_number,
+    ${utils.countNULLs(ctx)} AS nulls_count,
+    ${utils.countInfinites(ctx)} AS infinities_count,
+    ${utils.countNaNs(ctx)} AS nans_count,
+    min(${utils.handleFloatColumn(ctx)}) AS min,
+    max(${utils.handleFloatColumn(ctx)}) AS max,
+    avg(${utils.handleFloatColumn(ctx)}) AS avg,
+    sum(CASE WHEN (${utils.handleFloatColumn(ctx)} is not NULL) THEN 1 ELSE 0 END) as freq,
+    CASE WHEN ${ctx.start} = ${ctx.end}
+        THEN 0
+        ELSE GREATEST(1, LEAST(
+                    ${ctx.bins},
+                    WIDTH_BUCKET(${utils.handleFloatColumn(ctx)}, ${ctx.start}, ${ctx.end}, ${ctx.bins}))) - 1
+    END AS bin
+FROM
+(
+    ${ctx.query}
+) __cdb_filtered_source_query${extra_tables}
+GROUP BY 10${extra_groupby}
+ORDER BY 10;`;
+    }
+
+    _hasOverridenBins (override) {
+        return override && override.hasOwnProperty('bins');
+    }
+
+    _getSummary (result, override) {
+        const firstRow = result.rows[0] || {};
+
+        var total_nulls = 0;
+        var total_infinities = 0;
+        var total_nans = 0;
+        var total_avg = 0;
+        var total_count = 0;
+
+        result.rows.forEach(function(row) {
+            total_nulls += row.nulls_count;
+            total_infinities += row.infinities_count;
+            total_nans += row.nans_count;
+            total_avg += row.avg * row.freq;
+            total_count += row.freq;
+        });
+        if (total_count !== 0) {
+            total_avg /= total_count;
+        }
+
+        return {
+            bin_width: firstRow.bin_width,
+            bins_count: firstRow.bins_number,
+            bins_start: this._populateBinStart(firstRow, override),
+            nulls: total_nulls,
+            infinities: total_infinities,
+            nans: total_nans,
+            avg: total_avg
+        };
+    }
+
+    _getBuckets (result) {
+        return result.rows.map(({ bin, min, max, avg, freq }) => ({ bin, min, max, avg, freq }));
+    }
+
+    _populateBinStart (firstRow, override = {}) {
+        let binStart;
+
+        if (override.hasOwnProperty('start')) {
+            binStart = this._getBinStart(override);
+        } else {
+            binStart = firstRow.min;
+        }
+
+        return binStart;
+    }
+
+};

lib/cartodb/models/dataview/index.js

@@ -1,6 +1,5 @@
 module.exports = {
     Aggregation: require('./aggregation'),
     Formula: require('./formula'),
-    Histogram: require('./histogram'),
-    List: require('./list')
+    Histogram: require('./histogram')
 };

lib/cartodb/models/dataview/list.js

@@ -1,66 +0,0 @@
-var dot = require('dot');
-dot.templateSettings.strip = false;
-
-var BaseWidget = require('./base');
-
-var TYPE = 'list';
-
-var listSqlTpl = dot.template('select {{=it._columns}} from ({{=it._query}}) as _cdb_list');
-
-/**
-{
-    type: 'list',
-    options: {
-        columns: ['name', 'description']
-    }
-}
-*/
-
-function List(query, options) {
-    options = options || {};
-
-    if (!Array.isArray(options.columns)) {
-        throw new Error('List expects `columns` array in widget options');
-    }
-
-    BaseWidget.apply(this);
-
-    this.query = query;
-    this.columns = options.columns;
-}
-
-List.prototype = new BaseWidget();
-List.prototype.constructor = List;
-
-module.exports = List;
-
-List.prototype.sql = function(psql, filters, override, callback) {
-    if (!callback) {
-        callback = override;
-    }
-
-    var listSql = listSqlTpl({
-        _query: this.query,
-        _columns: this.columns.join(', ')
-    });
-
-    return callback(null, listSql);
-};
-
-List.prototype.format = function(result) {
-    return {
-        rows: result.rows
-    };
-};
-
-List.prototype.getType = function() {
-    return TYPE;
-};
-
-List.prototype.toString = function() {
-    return JSON.stringify({
-        _type: TYPE,
-        _query: this.query,
-        _columns: this.columns.join(', ')
-    });
-};

lib/cartodb/models/dataview/overviews/aggregation.js

@@ -0,0 +1,225 @@
+var BaseOverviewsDataview = require('./base');
+var BaseDataview = require('../aggregation');
+var debug = require('debug')('windshaft:widget:aggregation:overview');
+
+var dot = require('dot');
+dot.templateSettings.strip = false;
+
+var filteredQueryTpl = dot.template([
+    'filtered_source AS (',
+    '  SELECT *',
+    '  FROM ({{=it._query}}) _cdb_filtered_source',
+    '  {{?it._aggregationColumn  && it._isFloatColumn}}WHERE',
+    '    {{=it._aggregationColumn}} != \'infinity\'::float',
+    '  AND',
+    '    {{=it._aggregationColumn}} != \'-infinity\'::float',
+    '  AND',
+    '    {{=it._aggregationColumn}} != \'NaN\'::float{{?}}',
+    ')'
+].join(' \n'));
+
+var summaryQueryTpl = dot.template([
+    'summary AS (',
+    '  SELECT',
+    '  sum(_feature_count) AS count,',
+    '  sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count',
+    '  {{?it._isFloatColumn}},sum(',
+    '    CASE',
+    '      WHEN {{=it._aggregationColumn}} = \'infinity\'::float OR {{=it._aggregationColumn}} = \'-infinity\'::float',
+    '      THEN 1',
+    '      ELSE 0',
+    '    END',
+    '  ) AS infinities_count,',
+    '  sum(CASE WHEN {{=it._aggregationColumn}} = \'NaN\'::float THEN 1 ELSE 0 END) AS nans_count{{?}}',
+    '  FROM ({{=it._query}}) _cdb_aggregation_nulls',
+    ')'
+].join('\n'));
+
+var rankedCategoriesQueryTpl = dot.template([
+    'categories AS(',
+    '  SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,',
+    '    row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank',
+    '  FROM filtered_source',
+    '  {{?it._aggregationColumn!==null}}WHERE {{=it._aggregationColumn}} IS NOT NULL{{?}}',
+    '  GROUP BY {{=it._column}}',
+    '  ORDER BY 2 DESC',
+    ')'
+].join('\n'));
+
+var categoriesSummaryMinMaxQueryTpl = dot.template([
+    'categories_summary_min_max AS(',
+    '  SELECT max(value) max_val, min(value) min_val',
+    '  FROM categories',
+    ')'
+].join('\n'));
+
+var categoriesSummaryCountQueryTpl = dot.template([
+    'categories_summary_count AS(',
+    '  SELECT count(1) AS categories_count',
+    '  FROM (',
+    '    SELECT {{=it._column}} AS category',
+    '    FROM filtered_source',
+    '    GROUP BY {{=it._column}}',
+    '  ) _cdb_categories_count',
+    ')'
+].join('\n'));
+
+var rankedAggregationQueryTpl = dot.template([
+    'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val,',
+    '    count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
+    '  FROM categories, summary, categories_summary_min_max, categories_summary_count',
+    '  WHERE rank < {{=it._limit}}',
+    'UNION ALL',
+    'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val,',
+    '    count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
+    '  FROM categories, summary, categories_summary_min_max, categories_summary_count',
+    '  WHERE rank >= {{=it._limit}}',
+    'GROUP BY nulls_count, min_val, max_val, count,',
+    '  categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}'
+].join('\n'));
+
+var aggregationQueryTpl = dot.template([
+    'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,',
+    '  nulls_count, min_val, max_val, count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
+    'FROM filtered_source, summary, categories_summary_min_max, categories_summary_count',
+    'GROUP BY category, nulls_count, min_val, max_val, count,',
+    '  categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
+    'ORDER BY value DESC'
+].join('\n'));
+
+var CATEGORIES_LIMIT = 6;
+
+ function Aggregation(query, options, queryRewriter, queryRewriteData, params, queries) {
+    BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries);
+
+    this.query = query;
+    this.queries = queries;
+    this.column = options.column;
+    this.aggregation = options.aggregation;
+    this.aggregationColumn = options.aggregationColumn;
+    this._isFloatColumn = null;
+}
+
+Aggregation.prototype = Object.create(BaseOverviewsDataview.prototype);
+Aggregation.prototype.constructor = Aggregation;
+
+module.exports = Aggregation;
+
+Aggregation.prototype.sql = function(psql, override, callback) {
+    var self = this;
+
+    if (!callback) {
+        callback = override;
+        override = {};
+    }
+
+    var _query = this.rewrittenQuery(this.query);
+    var _aggregationColumn = this.aggregation !== 'count' ? this.aggregationColumn : null;
+
+    if (this.aggregationColumn && this._isFloatColumn === null) {
+        this._isFloatColumn = false;
+        this.getColumnType(psql, this.aggregationColumn, this.queries.no_filters, function (err, type) {
+            if (!err && !!type) {
+                self._isFloatColumn = type.float;
+            }
+            self.sql(psql, override, callback);
+        });
+        return null;
+    }
+
+    var aggregationSql;
+    if (!!override.ownFilter) {
+        aggregationSql = [
+            "WITH",
+            [
+                filteredQueryTpl({
+                    _isFloatColumn: this._isFloatColumn,
+                    _query: _query,
+                    _column: this.column,
+                    _aggregationColumn: _aggregationColumn
+                }),
+                summaryQueryTpl({
+                    _isFloatColumn: this._isFloatColumn,
+                    _query: _query,
+                    _column: this.column,
+                    _aggregationColumn: _aggregationColumn
+                }),
+                rankedCategoriesQueryTpl({
+                    _query: _query,
+                    _column: this.column,
+                    _aggregation: this.getAggregationSql(),
+                    _aggregationColumn: _aggregationColumn
+                }),
+                categoriesSummaryMinMaxQueryTpl({
+                    _query: _query,
+                    _column: this.column
+                }),
+                categoriesSummaryCountQueryTpl({
+                    _query: _query,
+                    _column: this.column
+                })
+            ].join(',\n'),
+            aggregationQueryTpl({
+                _isFloatColumn: this._isFloatColumn,
+                _query: _query,
+                _column: this.column,
+                _aggregation: this.getAggregationSql(),
+                _limit: CATEGORIES_LIMIT
+            })
+        ].join('\n');
+    } else {
+        aggregationSql = [
+            "WITH",
+            [
+                filteredQueryTpl({
+                    _isFloatColumn: this._isFloatColumn,
+                    _query: _query,
+                    _column: this.column,
+                    _aggregationColumn: _aggregationColumn
+                }),
+                summaryQueryTpl({
+                    _isFloatColumn: this._isFloatColumn,
+                    _query: _query,
+                    _column: this.column,
+                    _aggregationColumn: _aggregationColumn
+                }),
+                rankedCategoriesQueryTpl({
+                    _query: _query,
+                    _column: this.column,
+                    _aggregation: this.getAggregationSql(),
+                    _aggregationColumn: _aggregationColumn
+                }),
+                categoriesSummaryMinMaxQueryTpl({
+                    _query: _query,
+                    _column: this.column
+                }),
+                categoriesSummaryCountQueryTpl({
+                    _query: _query,
+                    _column: this.column
+                })
+            ].join(',\n'),
+            rankedAggregationQueryTpl({
+                _isFloatColumn: this._isFloatColumn,
+                _query: _query,
+                _column: this.column,
+                _limit: CATEGORIES_LIMIT
+            })
+        ].join('\n');
+    }
+
+    debug(aggregationSql);
+
+    return callback(null, aggregationSql);
+};
+
+var aggregationFnQueryTpl = {
+    count: dot.template('sum(_feature_count)'),
+    sum:   dot.template('sum({{=it._aggregationColumn}}*_feature_count)')
+};
+
+Aggregation.prototype.getAggregationSql = function() {
+    return aggregationFnQueryTpl[this.aggregation]({
+        _aggregationFn: this.aggregation,
+        _aggregationColumn: this.aggregationColumn || 1
+    });
+};

lib/cartodb/models/dataview/overviews/base.js

@@ -0,0 +1,89 @@
+var _ = require('underscore');
+var BaseDataview = require('../base');
+
+function BaseOverviewsDataview(query, queryOptions, BaseDataview, queryRewriter, queryRewriteData, options, queries) {
+  this.BaseDataview = BaseDataview;
+  this.query = query;
+  this.queryOptions = queryOptions;
+  this.queryRewriter = queryRewriter;
+  this.queryRewriteData = queryRewriteData;
+  this.options = options;
+  this.queries = queries;
+  this.baseDataview = new this.BaseDataview(this.query, this.queryOptions, this.queries);
+}
+
+module.exports = BaseOverviewsDataview;
+
+BaseOverviewsDataview.prototype = new BaseDataview();
+BaseOverviewsDataview.prototype.constructor = BaseOverviewsDataview;
+
+// TODO: parameterized these settings
+var SETTINGS = {
+    // use overviews as a default fallback strategy
+    defaultOverviews: false,
+
+    // minimum ratio of bounding box size to grid size
+    // (this would ideally be based on the viewport size in pixels)
+    zoomLevelFactor: 1024.0
+};
+
+// Compute zoom level so that the the resolution grid size of the
+// selected overview is smaller (zoomLevelFactor times smaller at least)
+// than the bounding box size.
+BaseOverviewsDataview.prototype.zoomLevelForBbox = function(bbox) {
+    var pxPerTile = 256.0;
+    var earthWidth = 360.0;
+    // TODO: now we assume overviews are computed for 1-pixel tolerance;
+    // should use extended overviews metadata to compute this properly.
+    if ( bbox ) {
+      var bboxValues = _.map(bbox.split(','), function(v) { return +v; });
+      var w = Math.abs(bboxValues[2]-bboxValues[0]);
+      var h = Math.abs(bboxValues[3]-bboxValues[1]);
+      var maxDim = Math.min(w, h);
+
+      // Find minimum suitable z
+      // note that the QueryRewirter will use the minimum level overview
+      // of level >= z if it exists, and otherwise the base table
+      var z = Math.ceil(-Math.log(maxDim*pxPerTile/earthWidth/SETTINGS.zoomLevelFactor)/Math.log(2.0));
+      return Math.max(z, 0);
+    }
+    return 0;
+};
+
+BaseOverviewsDataview.prototype.rewrittenQuery = function(query) {
+    var zoom_level = this.zoomLevelForBbox(this.options.bbox);
+    return this.queryRewriter.query(query, this.queryRewriteData, { zoom_level: zoom_level });
+};
+
+// Default behaviour
+BaseOverviewsDataview.prototype.defaultSql = function(psql, override, callback) {
+    var query = this.query;
+    var dataview = this.baseDataview;
+    if ( SETTINGS.defaultOverviews ) {
+        query = this.rewrittenQuery(query);
+        dataview = new this.BaseDataview(query, this.queryOptions);
+    }
+    return dataview.sql(psql, override, callback);
+};
+
+// default implementation that can be override in derived classes:
+
+BaseOverviewsDataview.prototype.sql = function(psql, override, callback) {
+    return this.defaultSql(psql, override, callback);
+};
+
+BaseOverviewsDataview.prototype.search = function(psql, userQuery, callback) {
+    return this.baseDataview.search(psql, userQuery, callback);
+};
+
+BaseOverviewsDataview.prototype.format = function(result) {
+    return this.baseDataview.format(result);
+};
+
+BaseOverviewsDataview.prototype.getType = function() {
+    return this.baseDataview.getType();
+};
+
+BaseOverviewsDataview.prototype.toString = function() {
+    return this.baseDataview.toString();
+};

lib/cartodb/models/dataview/overviews/factory.js

@@ -14,7 +14,8 @@ OverviewsDataviewFactory.prototype.getDataview = function(query, dataviewDefinit
         return parentFactory.getDataview(query, dataviewDefinition);
     }
     return new dataviews[type](
-        query, dataviewDefinition.options, this.queryRewriter, this.queryRewriteData, this.options
+        query, dataviewDefinition.options, this.queryRewriter, this.queryRewriteData, this.options,
+        dataviewDefinition.sql
     );
 };
 

lib/cartodb/models/dataview/overviews/formula.js

@@ -1,104 +1,78 @@
-var _ = require('underscore');
-var BaseWidget = require('../base');
+var BaseOverviewsDataview = require('./base');
 var BaseDataview = require('../formula');
-
-var debug = require('debug')('windshaft:widget:formula:overviews');
+var debug = require('debug')('windshaft:widget:formula:overview');
+const utils = require('../../../utils/query-utils');
 
 var dot = require('dot');
 dot.templateSettings.strip = false;
 
-var formulaQueryTpls = {
-  'count': dot.template([
-      'SELECT',
-      'sum(_feature_count) AS result,',
-      '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
-      'FROM ({{=it._query}}) _cdb_formula'
-  ].join('\n')),
-  'sum': dot.template([
-      'SELECT',
-      'sum({{=it._column}}*_feature_count) AS result,',
-      '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
-      'FROM ({{=it._query}}) _cdb_formula'
-  ].join('\n')),
-  'avg': dot.template([
-      'SELECT',
-      'sum({{=it._column}}*_feature_count)/sum(_feature_count) AS result,',
-      '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
-      'FROM ({{=it._query}}) _cdb_formula'
-  ].join('\n')),
+const VALID_OPERATIONS = {
+    count: true,
+    sum: true,
+    avg: true
 };
 
-function Formula(query, options, queryRewriter, queryRewriteData, params) {
-    this.base_dataview = new BaseDataview(query, options);
-    this.query = query;
-    this.column = options.column || '1';
-    this.operation = options.operation;
-    this.queryRewriter = queryRewriter;
-    this.queryRewriteData = queryRewriteData;
-    this.options = params;
+/** Formulae to calculate the end result using _feature_count from the overview table*/
+function dataviewResult(ctx) {
+    switch (ctx.operation) {
+        case 'count':
+            return `sum(_feature_count)`;
+        case 'sum':
+            return `sum(${utils.handleFloatColumn(ctx)}*_feature_count)`;
+        case 'avg':
+            return `sum(${utils.handleFloatColumn(ctx)}*_feature_count)/sum(_feature_count) `;
+    }
+    return `${ctx.operation}(${utils.handleFloatColumn(ctx)})`;
 }
 
-Formula.prototype = new BaseWidget();
+const formulaQueryTpl = ctx =>
+`SELECT
+    ${dataviewResult(ctx)} AS result,
+    ${utils.countNULLs(ctx)} AS nulls_count
+    ${ctx.isFloatColumn ? `,${utils.countInfinites(ctx)} AS infinities_count,` : ``}
+    ${ctx.isFloatColumn ? `${utils.countNaNs(ctx)} AS nans_count` : ``}
+FROM (${ctx.query}) __cdb_formula`;
+
+function Formula(query, options, queryRewriter, queryRewriteData, params, queries) {
+    BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries);
+    this.column = options.column || '1';
+    this.operation = options.operation;
+    this._isFloatColumn = null;
+    this.queries = queries;
+}
+
+Formula.prototype = Object.create(BaseOverviewsDataview.prototype);
 Formula.prototype.constructor = Formula;
 
 module.exports = Formula;
 
-var zoom_level_factor = 100.0;
-
-// Compute zoom level so that the the resolution grid size of the
-// selected overview is smaller (zoom_level_factor times smaller at least)
-// than the bounding box size.
-function zoom_level_for_bbox(bbox) {
-    var px_per_tile = 256.0;
-    var earth_width = 360.0;
-    // TODO: now we assume overviews are computed for 1-pixel tolerance;
-    // should use extended overviews metadata to compute this properly.
-    if ( bbox ) {
-      var bbox_values = _.map(bbox.split(','), function(v) { return +v; });
-      var w = Math.abs(bbox_values[2]-bbox_values[0]);
-      var h = Math.abs(bbox_values[3]-bbox_values[1]);
-      var max_dim = Math.min(w, h);
-
-      // Find minimum suitable z
-      // note that the QueryRewirter will use the minimum level overview
-      // of level >= z if it exists, and otherwise the base table
-      var z = Math.ceil(-Math.log(max_dim*px_per_tile/earth_width/zoom_level_factor)/Math.log(2.0));
-      return Math.max(z, 0);
+Formula.prototype.sql = function (psql, override, callback) {
+    var self = this;
+    if (!VALID_OPERATIONS[this.operation]) {
+        return this.defaultSql(psql, override, callback);
     }
-    return 0;
-}
 
-Formula.prototype.sql = function(psql, filters, override, callback) {
-    var _query = this.query;
-    var formulaQueryTpl = formulaQueryTpls[this.operation];
-
-    if ( formulaQueryTpl ) {
-        // supported formula for use with overviews
-        var zoom_level = zoom_level_for_bbox(this.options.bbox);
-        _query = this.queryRewriter.query(_query, this.queryRewriteData, { zoom_level: zoom_level });
-        var formulaSql = formulaQueryTpl({
-            _query: _query,
-            _operation: this.operation,
-            _column: this.column
+    if (this._isFloatColumn === null) {
+        this._isFloatColumn = false;
+        this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) {
+            if (!err && !!type) {
+                self._isFloatColumn = type.float;
+            }
+            self.sql(psql, override, callback);
         });
-        debug(formulaSql);
-        callback = callback || override;
-
-        return callback(null, formulaSql);
+        return null;
     }
 
-    // For non supported operations (min, max) we're not using overviews.
-    return this.base_dataview.sql(psql, filters, override, callback);
-};
+    var formulaSql = formulaQueryTpl({
+        isFloatColumn: this._isFloatColumn,
+        query: this.rewrittenQuery(this.query),
+        operation: this.operation,
+        column: this.column
+    });
 
-Formula.prototype.format = function(result) {
-    return this.base_dataview.format(result);
-};
+    callback = callback || override;
 
-Formula.prototype.getType = function() {
-    return this.base_dataview.getType();
-};
+    debug(formulaSql);
 
-Formula.prototype.toString = function() {
-    return this.base_dataview.toString();
+    return callback(null, formulaSql);
 };

lib/cartodb/models/dataview/overviews/histogram.js

@@ -0,0 +1,282 @@
+var _ = require('underscore');
+var BaseOverviewsDataview = require('./base');
+var BaseDataview = require('../histogram');
+var debug = require('debug')('windshaft:dataview:histogram:overview');
+
+var dot = require('dot');
+dot.templateSettings.strip = false;
+
+var BIN_MIN_NUMBER = 6;
+var BIN_MAX_NUMBER = 48;
+
+var filteredQueryTpl = dot.template([
+    'filtered_source AS (',
+    '  SELECT *',
+    '  FROM ({{=it._query}}) _cdb_filtered_source',
+    '  WHERE',
+    '    {{=it._column}} IS NOT NULL',
+    '  {{?it._isFloatColumn}}AND',
+    '    {{=it._column}} != \'infinity\'::float',
+    '  AND',
+    '    {{=it._column}} != \'-infinity\'::float',
+    '  AND',
+    '    {{=it._column}} != \'NaN\'::float{{?}}',
+    ')'
+].join(' \n'));
+
+var basicsQueryTpl = dot.template([
+    'basics AS (',
+    '  SELECT',
+    '    max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,',
+    '    sum({{=it._column}}*_feature_count)/sum(_feature_count) AS avg_val, sum(_feature_count) AS total_rows',
+    '  FROM filtered_source',
+    ')'
+].join(' \n'));
+
+var overrideBasicsQueryTpl = dot.template([
+    'basics AS (',
+    '  SELECT',
+    '    max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,',
+    '    sum({{=it._column}}*_feature_count)/sum(_feature_count) AS avg_val, sum(_feature_count) AS total_rows',
+    '  FROM filtered_source',
+    ')'
+].join('\n'));
+
+var iqrQueryTpl = dot.template([
+    'iqrange AS (',
+    '  SELECT max(quartile_max) - min(quartile_max) AS iqr',
+    '  FROM (',
+    '    SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (',
+    '      SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}',
+    '    ) AS quartile',
+    '    FROM filtered_source) _cdb_quartiles',
+    '    WHERE quartile = 1 or quartile = 3',
+    '    GROUP BY quartile',
+    '  ) _cdb_iqr',
+    ')'
+].join('\n'));
+
+var binsQueryTpl = dot.template([
+    'bins AS (',
+    '  SELECT CASE WHEN total_rows = 0 OR iqr = 0',
+    '      THEN 1',
+    '      ELSE GREATEST(',
+    '        LEAST({{=it._minBins}}, CAST(total_rows AS INT)),',
+    '        LEAST(',
+    '          CAST(((max_val - min_val) / (2 * iqr * power(total_rows, 1/3))) AS INT),',
+    '          {{=it._maxBins}}',
+    '        )',
+    '      )',
+    '    END AS bins_number',
+    '  FROM basics, iqrange, filtered_source',
+    '  LIMIT 1',
+    ')'
+].join('\n'));
+
+var overrideBinsQueryTpl = dot.template([
+    'bins AS (',
+    '  SELECT {{=it._bins}} AS bins_number',
+    ')'
+].join('\n'));
+
+var nullsQueryTpl = dot.template([
+    'nulls AS (',
+    '  SELECT',
+    '  count(*) AS nulls_count',
+    '  FROM ({{=it._query}}) _cdb_histogram_nulls',
+    '  WHERE {{=it._column}} IS NULL',
+    ')'
+].join('\n'));
+
+var infinitiesQueryTpl = dot.template([
+    'infinities AS (',
+    '  SELECT',
+    '    count(*) AS infinities_count',
+    '  FROM ({{=it._query}}) _cdb_histogram_infinities',
+    '  WHERE',
+    '    {{=it._column}} = \'infinity\'::float',
+    '  OR',
+    '    {{=it._column}} = \'-infinity\'::float',
+    ')'
+].join('\n'));
+
+var nansQueryTpl = dot.template([
+    'nans AS (',
+    '  SELECT',
+    '    count(*) AS nans_count',
+    '  FROM ({{=it._query}}) _cdb_histogram_infinities',
+    '  WHERE {{=it._column}} = \'NaN\'::float',
+    ')'
+].join('\n'));
+
+var histogramQueryTpl = dot.template([
+    'SELECT',
+    '    (max_val - min_val) / cast(bins_number as float) AS bin_width,',
+    '    bins_number,',
+    '    nulls_count,',
+    '    {{?it._isFloatColumn}}infinities_count,',
+    '    nans_count,{{?}}',
+    '    avg_val,',
+    '    CASE WHEN min_val = max_val',
+    '      THEN 0',
+    '      ELSE GREATEST(1, LEAST(WIDTH_BUCKET({{=it._column}}, min_val, max_val, bins_number), bins_number)) - 1',
+    '    END AS bin,',
+    '    min({{=it._column}})::numeric AS min,',
+    '    max({{=it._column}})::numeric AS max,',
+    '    sum({{=it._column}}*_feature_count)/sum(_feature_count)::numeric AS avg,',
+    '    sum(_feature_count) AS freq',
+    'FROM filtered_source, basics, nulls, bins{{?it._isFloatColumn}},infinities, nans{{?}}',
+    'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val',
+    '   {{?it._isFloatColumn}}, infinities_count, nans_count{{?}}',
+    'ORDER BY bin'
+].join('\n'));
+
+function Histogram(query, options, queryRewriter, queryRewriteData, params, queries) {
+    BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries);
+
+    this.query = query;
+    this.queries = queries;
+    this.column = options.column;
+    this.bins = options.bins;
+
+    this._columnType = null;
+}
+
+Histogram.prototype = Object.create(BaseOverviewsDataview.prototype);
+Histogram.prototype.constructor = Histogram;
+
+module.exports = Histogram;
+
+Histogram.prototype.sql = function(psql, override, callback) {
+    var self = this;
+
+    if (!callback) {
+        callback = override;
+        override = {};
+    }
+
+
+    if (this._columnType === null) {
+        this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) {
+            // assume numeric, will fail later
+            self._columnType = 'numeric';
+            if (!err && !!type) {
+                self._columnType = Object.keys(type).find(function (key) {
+                    return type[key];
+                });
+            }
+            self.sql(psql, override, callback);
+        }, true); // use read-only transaction
+        return null;
+    }
+
+    if (this._columnType === 'date') {
+        // overviews currently aggregate dates to NULL
+        // to avoid problem we don't use overviews for histograms of date columns
+        return this.defaultSql(psql, override, callback);
+    }
+
+    var histogramSql = this._buildQuery(override);
+
+    return callback(null, histogramSql);
+};
+
+Histogram.prototype._buildQuery = function (override) {
+    var filteredQuery, basicsQuery, binsQuery;
+    var _column = this.column;
+    var _query = this.rewrittenQuery(this.query);
+
+    filteredQuery = filteredQueryTpl({
+        _isFloatColumn: this._columnType === 'float',
+        _query: _query,
+        _column: _column
+    });
+
+    if (this._shouldOverride(override)) {
+        debug('overriding with %j', override);
+        basicsQuery = overrideBasicsQueryTpl({
+            _query: _query,
+            _column: _column,
+            _start: override.start,
+            _end: override.end
+        });
+
+        binsQuery = [
+            overrideBinsQueryTpl({
+                _bins: override.bins
+            })
+        ].join(',\n');
+    } else {
+        basicsQuery = basicsQueryTpl({
+            _query: _query,
+            _column: _column
+        });
+
+        if (this._shouldOverrideBins(override)) {
+            binsQuery = [
+                overrideBinsQueryTpl({
+                    _bins: override.bins
+                })
+            ].join(',\n');
+        } else {
+            binsQuery = [
+                iqrQueryTpl({
+                    _query: _query,
+                    _column: _column
+                }),
+                binsQueryTpl({
+                    _query: _query,
+                    _minBins: BIN_MIN_NUMBER,
+                    _maxBins: BIN_MAX_NUMBER
+                })
+            ].join(',\n');
+        }
+    }
+
+    var cteSql = [
+        filteredQuery,
+        basicsQuery,
+        binsQuery,
+        nullsQueryTpl({
+            _query: _query,
+            _column: _column
+        })
+    ];
+
+    if (this._columnType === 'float') {
+        cteSql.push(
+            infinitiesQueryTpl({
+                _query: _query,
+                _column: _column
+            }),
+            nansQueryTpl({
+                _query: _query,
+                _column: _column
+            })
+        );
+    }
+
+    var histogramSql = [
+        "WITH",
+        cteSql.join(',\n'),
+        histogramQueryTpl({
+            _isFloatColumn: this._columnType === 'float',
+            _query: _query,
+            _column: _column
+        })
+    ].join('\n');
+
+    debug(histogramSql);
+
+    return histogramSql;
+};
+
+Histogram.prototype._shouldOverride = function (override) {
+    return override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins');
+};
+
+Histogram.prototype._shouldOverrideBins = function (override) {
+    return override && _.has(override, 'bins');
+};
+
+

lib/cartodb/models/dataview/overviews/index.js

@@ -1,3 +1,5 @@
 module.exports = {
-    Formula: require('./formula')
+  Aggregation: require('./aggregation'),
+  Formula: require('./formula'),
+  Histogram: require('./histogram')
 };

lib/cartodb/models/filter/analysis.js

@@ -1,6 +1,6 @@
 var filters = {
-    category: require('./camshaft/category'),
-    range: require('./camshaft/range')
+    category: require('./analysis/category'),
+    range: require('./analysis/range')
 };
 
 function createFilter(filterDefinition) {
@@ -11,11 +11,11 @@ function createFilter(filterDefinition) {
     return new filters[filterType](filterDefinition.column, filterDefinition.params);
 }
 
-function CamshaftFilters(filters) {
+function AnalysisFilters(filters) {
     this.filters = filters;
 }
 
-CamshaftFilters.prototype.sql = function(rawSql) {
+AnalysisFilters.prototype.sql = function(rawSql) {
   var filters = this.filters || {};
   var applyFilters = {};
 
@@ -32,4 +32,4 @@ CamshaftFilters.prototype.sql = function(rawSql) {
       }, rawSql);
 };
 
-module.exports = CamshaftFilters;
+module.exports = AnalysisFilters;

lib/cartodb/models/filter/analysis/category.js

@@ -6,7 +6,7 @@ dot.templateSettings.strip = false;
 
 var filterQueryTpl = dot.template([
     'SELECT *',
-    'FROM ({{=it._sql}}) _camshaft_category_filter',
+    'FROM ({{=it._sql}}) _analysis_category_filter',
     'WHERE {{=it._filters}}'
 ].join('\n'));
 var escapeStringTpl = dot.template('$escape_{{=it._i}}${{=it._value}}$escape_{{=it._i}}$');

lib/cartodb/models/filter/analysis/range.js

@@ -6,7 +6,7 @@ dot.templateSettings.strip = false;
 var betweenFilterTpl = dot.template('{{=it._column}} BETWEEN {{=it._min}} AND {{=it._max}}');
 var minFilterTpl = dot.template('{{=it._column}} >= {{=it._min}}');
 var maxFilterTpl = dot.template('{{=it._column}} <= {{=it._max}}');
-var filterQueryTpl = dot.template('SELECT * FROM ({{=it._sql}}) _camshaft_range_filter WHERE {{=it._filter}}');
+var filterQueryTpl = dot.template('SELECT * FROM ({{=it._sql}}) _analysis_range_filter WHERE {{=it._filter}}');
 
 function Range(column, filterParams) {
     this.column = column;

lib/cartodb/models/filter/bbox.js

@@ -8,7 +8,7 @@ var filterQueryTpl = dot.template([
 ].join('\n'));
 
 var bboxFilterTpl = dot.template(
-    '{{=it._column}} && ST_Transform(ST_MakeEnvelope({{=it._bbox}}, 4326), {{=it._srid}})'
+    'ST_Intersects({{=it._column}}, ST_Transform(ST_MakeEnvelope({{=it._bbox}}, 4326), {{=it._srid}}))'
 );
 
 var LATITUDE_MAX_VALUE = 85.0511287798066;
@@ -66,7 +66,8 @@ function getBoundingBoxes(west, south, east, north) {
         bboxes.push([west, south, east, north]);
     } else {
         bboxes.push([west, south, 180, north]);
-        bboxes.push([-180, south, east % 180, north]);
+        // here we assume west,east have been adjusted => west >= -180 => east > 180
+        bboxes.push([-180, south, east - 360, north]);
     }
 
     return bboxes;

lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js

@@ -0,0 +1,172 @@
+const AggregationMapConfig = require('../../aggregation/aggregation-mapconfig');
+const queryUtils = require('../../../utils/query-utils');
+
+const unsupportedGeometryTypeErrorMessage = ctx =>
+`Unsupported geometry type: ${ctx.geometryType}. ` +
+`Aggregation is available only for geometry type: ${AggregationMapConfig.SUPPORTED_GEOMETRY_TYPES}`;
+
+const invalidAggregationParamValueErrorMessage = ctx =>
+`Invalid value for 'aggregation' query param: ${ctx.value}. Valid ones are 'true' or 'false'`;
+
+module.exports = class AggregationMapConfigAdapter {
+    constructor (pgConnection) {
+        this.pgConnection = pgConnection;
+    }
+
+    getMapConfig (user, requestMapConfig, params, context, callback) {
+        if (!this._isValidAggregationQueryParam(params)) {
+            return callback(new Error(invalidAggregationParamValueErrorMessage({ value: params.aggregation })));
+        }
+
+        let mapConfig;
+        try {
+            mapConfig = new AggregationMapConfig(user, requestMapConfig, this.pgConnection);
+        } catch (err) {
+            return callback(err);
+        }
+
+
+        if (!this._shouldAdapt(mapConfig, params)) {
+            return callback(null, requestMapConfig);
+        }
+
+        this.pgConnection.getConnection(user, (err, connection) => {
+            if (err) {
+                return callback(err);
+            }
+
+            this._adaptLayers(connection, mapConfig, requestMapConfig, context, callback);
+        });
+    }
+
+    _isValidAggregationQueryParam (params) {
+        const { aggregation } = params;
+        return aggregation === undefined || aggregation === 'true' || aggregation === 'false';
+    }
+
+    _shouldAdapt (mapConfig, params) {
+        const { aggregation } = params;
+
+        if (aggregation === 'false') {
+            return false;
+        }
+
+        if (aggregation === 'true' || mapConfig.isAggregationMapConfig()) {
+            return true;
+        }
+
+        return false;
+    }
+
+    _adaptLayers (connection, mapConfig, requestMapConfig, context, callback) {
+        const adaptLayerPromises = requestMapConfig.layers.map((layer, index) => {
+            return this._adaptLayer(connection, mapConfig, layer, index);
+        });
+
+        Promise.all(adaptLayerPromises)
+            .then(results => {
+                context.aggregation = {
+                    layers: []
+                };
+
+                results.forEach(({ layer, index, adapted }) => {
+                    if (adapted) {
+                        requestMapConfig.layers[index] = layer;
+                    }
+                    const aggregatedFormats = this._getAggregationMetadata(mapConfig, layer, adapted);
+                    context.aggregation.layers.push(aggregatedFormats);
+                });
+
+                callback(null, requestMapConfig);
+            })
+            .catch(err => callback(err));
+    }
+
+    _adaptLayer (connection, mapConfig, layer, index) {
+        return new Promise((resolve, reject) => {
+            this._shouldAdaptLayer(connection, mapConfig, layer, index, (err, shouldAdapt) => {
+                if (err) {
+                    return reject(err);
+                }
+
+                if (!shouldAdapt) {
+                    return resolve({ layer, index, adapted: shouldAdapt });
+                }
+
+                const sqlQueryWrap = layer.options.sql_wrap;
+
+                let aggregationSql = mapConfig.getAggregatedQuery(index);
+
+                if (sqlQueryWrap) {
+                    aggregationSql = sqlQueryWrap.replace(/<%=\s*sql\s*%>/g, aggregationSql);
+                }
+
+                layer.options.sql = aggregationSql;
+
+                mapConfig.getLayerAggregationColumns(index, (err, columns) => {
+                    if (err) {
+                        return reject(err);
+                    }
+
+                    layer.options.columns = columns;
+
+                    return resolve({ layer, index, adapted: shouldAdapt });
+                });
+            });
+        });
+    }
+
+    _shouldAdaptLayer (connection, mapConfig, layer, index, callback) {
+        if (!mapConfig.isAggregationLayer(index)) {
+            return callback(null, false);
+        }
+
+        const aggregationMetadata = queryUtils.getAggregationMetadata({
+            query: layer.options.sql_raw ? layer.options.sql_raw : layer.options.sql,
+            geometryColumn: AggregationMapConfig.getAggregationGeometryColumn()
+        });
+
+        connection.query(aggregationMetadata, (err, res) => {
+            if (err) {
+                return callback(null, false);
+            }
+
+            const result = res.rows[0] || {};
+
+            if (!mapConfig.isVectorOnlyMapConfig() && !AggregationMapConfig.supportsGeometryType(result.type)) {
+                const message = unsupportedGeometryTypeErrorMessage({ geometryType: result.type });
+                const error = new Error(message);
+                error.type = 'layer';
+                error.layer = {
+                    id: mapConfig.getLayerId(index),
+                    index: index,
+                    type: mapConfig.layerType(index)
+                };
+
+                return callback(error);
+            }
+
+            if (mapConfig.isVectorOnlyMapConfig() && !AggregationMapConfig.supportsGeometryType(result.type)) {
+                return callback(null, false);
+            }
+
+            if (!mapConfig.doesLayerReachThreshold(index, result.count)) {
+                return callback(null, false);
+            }
+
+            callback(null, true);
+        });
+    }
+
+    _getAggregationMetadata (mapConfig, layer, adapted) {
+        if (!adapted) {
+            return { png: false, mvt: false };
+        }
+
+        if (mapConfig.isVectorOnlyMapConfig()) {
+            return { png: false, mvt: true };
+        }
+
+        return { png: true, mvt: true };
+    }
+};

lib/cartodb/models/mapconfig/adapter/analysis-mapconfig-adapter.js

@@ -11,73 +11,25 @@ function AnalysisMapConfigAdapter(analysisBackend) {
 
 module.exports = AnalysisMapConfigAdapter;
 
-var SKIP_COLUMNS = {
-    'the_geom': true,
-    'the_geom_webmercator': true
-};
-
-function skipColumns(columnNames) {
-    return columnNames
-        .filter(function(columnName) { return !SKIP_COLUMNS[columnName]; });
-}
-
-var layerQueryTemplate = dot.template([
-    'SELECT {{=it._columns}}',
-    'FROM ({{=it._query}}) _cdb_analysis_query'
-].join('\n'));
-
-function layerQuery(node) {
-    if (node.type === 'source') {
-        return node.getQuery();
-    }
-    var _columns = ['ST_Transform(the_geom, 3857) the_geom_webmercator'].concat(skipColumns(node.getColumns()));
-    return layerQueryTemplate({ _query: node.getQuery(), _columns: _columns.join(', ') });
-}
-
-function appendFiltersToNodes(requestMapConfig, dataviewsFiltersBySourceId) {
-    var analyses = requestMapConfig.analyses || [];
-
-    requestMapConfig.analyses = analyses.map(function(analysisDefinition) {
-        var analysisGraph = new camshaft.reference.AnalysisGraph(analysisDefinition);
-        var definition = analysisDefinition;
-        Object.keys(dataviewsFiltersBySourceId).forEach(function(sourceId) {
-            definition = analysisGraph.getDefinitionWith(sourceId, {filters: dataviewsFiltersBySourceId[sourceId] });
-        });
-
-        return definition;
-    });
-
-    return requestMapConfig;
-}
-
-function shouldAdaptLayers(requestMapConfig) {
-    return Array.isArray(requestMapConfig.layers) &&
-        Array.isArray(requestMapConfig.analyses) && requestMapConfig.analyses.length > 0;
-}
-
-var DATAVIEW_TYPE_2_FILTER_TYPE = {
-    aggregation: 'category',
-    histogram: 'range'
-};
-function getFilter(dataview, params) {
-    var type = dataview.type;
-
-    return {
-        type: DATAVIEW_TYPE_2_FILTER_TYPE[type],
-        column: dataview.options.column,
-        params: params
-    };
-}
-
-AnalysisMapConfigAdapter.prototype.getMapConfig = function(analysisConfiguration, requestMapConfig, filters, callback) {
+AnalysisMapConfigAdapter.prototype.getMapConfig = function(user, requestMapConfig, params, context, callback) {
     // jshint maxcomplexity:7
     var self = this;
-    filters = filters || {};
 
     if (!shouldAdaptLayers(requestMapConfig)) {
         return callback(null, requestMapConfig);
     }
 
+    var analysisConfiguration = context.analysisConfiguration;
+
+    var filters = {};
+    if (params.filters) {
+        try {
+            filters = JSON.parse(params.filters);
+        } catch (e) {
+            // ignore
+        }
+    }
+
     var dataviewsFilters = filters.dataviews || {};
     debug(dataviewsFilters);
     var dataviews = requestMapConfig.dataviews || {};
@@ -106,11 +58,31 @@ AnalysisMapConfigAdapter.prototype.getMapConfig = function(analysisConfiguration
 
     requestMapConfig = appendFiltersToNodes(requestMapConfig, dataviewsFiltersBySourceId);
 
+    // Expected format for analyses filters
+    // filters = {analyses: {
+    //     a1: [{min, max}, {accept, reject}],
+    //     b1: [{range, column, min, max}, {category, column, accept, reject}]
+    // }}
+    requestMapConfig = appendFiltersToNodes(requestMapConfig, filters.analyses);
+
     function createAnalysis(analysisDefinition, done) {
-        self.analysisBackend.create(analysisConfiguration, analysisDefinition, done);
+        self.analysisBackend.create(analysisConfiguration, analysisDefinition, function (err, analysis) {
+            if (err) {
+                var error = new Error(err.message);
+                error.type = 'analysis';
+                error.analysis = {
+                    id: analysisDefinition.id,
+                    node_id: err.node_id,
+                    type: analysisDefinition.type
+                };
+                return done(error);
+            }
+
+            done(null, analysis);
+        });
     }
 
-    var analysesQueue = queue(requestMapConfig.analyses.length);
+    var analysesQueue = queue(1);
     requestMapConfig.analyses.forEach(function(analysis) {
         analysesQueue.defer(createAnalysis, analysis);
     });
@@ -126,7 +98,7 @@ AnalysisMapConfigAdapter.prototype.getMapConfig = function(analysisConfiguration
                 sourceId2Query[rootNode.params.id] = rootNode;
             }
 
-            analysis.getSortedNodes().forEach(function(node) {
+            analysis.getNodes().forEach(function(node) {
                 if (node.params && node.params.id) {
                     sourceId2Query[node.params.id] = node;
                 }
@@ -145,13 +117,12 @@ AnalysisMapConfigAdapter.prototype.getMapConfig = function(analysisConfiguration
                     var analysisSql = layerQuery(layerNode);
                     var sqlQueryWrap = layer.options.sql_wrap;
                     if (sqlQueryWrap) {
+                        layer.options.sql_raw = analysisSql;
                         analysisSql = sqlQueryWrap.replace(/<%=\s*sql\s*%>/g, analysisSql);
                     }
                     layer.options.sql = analysisSql;
-                    var layerDataviews = getLayerDataviews(layer, dataviews);
-                    layer.options.columns = layerDataviews.reduce(function(columns, dataview) {
-                        return columns.concat(getDataviewColumns(dataview));
-                    }, []);
+                    layer.options.columns = getDataviewsColumns(getLayerDataviews(layer, dataviews));
+                    layer.options.affected_tables = getAllAffectedTablesFromSourceNodes(layerNode);
                 } else {
                     missingNodesErrors.push(
                         new Error('Missing analysis node.id="' + layerSourceId +'" for layer='+layerIndex)
@@ -161,16 +132,116 @@ AnalysisMapConfigAdapter.prototype.getMapConfig = function(analysisConfiguration
             return layer;
         });
 
-        debug('mapconfig output', JSON.stringify(requestMapConfig, null, 4));
-
-        if (missingNodesErrors.length > 0) {
-            return callback(missingNodesErrors);
+        var missingDataviewsNodesErrors = getMissingDataviewsSourceIds(dataviews, sourceId2Node);
+        if (missingNodesErrors.length > 0 || missingDataviewsNodesErrors.length > 0) {
+            return callback(missingNodesErrors.concat(missingDataviewsNodesErrors));
         }
 
-        return callback(null, requestMapConfig, analysesResults);
+        // Augment dataviews with sql from analyses
+        Object.keys(dataviews).forEach(function(dataviewName) {
+            var dataview = requestMapConfig.dataviews[dataviewName];
+            var dataviewSourceId = dataview.source.id;
+            var dataviewNode = sourceId2Node[dataviewSourceId];
+            dataview.node = {
+                type: dataviewNode.type,
+                filters: dataviewNode.getFilters()
+            };
+            dataview.sql = {
+                own_filter_on: dataviewQuery(dataviewNode, dataviewName, true),
+                own_filter_off: dataviewQuery(dataviewNode, dataviewName, false),
+                no_filters: dataviewNode.getQuery(Object.keys(dataviewNode.getFilters())
+                    .reduce(function(applyFilters, filterId) {
+                        applyFilters[filterId] = false;
+                        return applyFilters;
+                    }, {})
+                )
+            };
+        });
+        if (Object.keys(dataviews).length > 0) {
+            requestMapConfig.dataviews = dataviews;
+        }
+
+        debug('mapconfig output', JSON.stringify(requestMapConfig, null, 4));
+
+        context.analysesResults = analysesResults;
+
+        return callback(null, requestMapConfig);
     });
 };
 
+var SKIP_COLUMNS = {
+    'the_geom': true,
+    'the_geom_webmercator': true
+};
+
+function skipColumns(columnNames) {
+    return columnNames
+        .filter(function(columnName) { return !SKIP_COLUMNS[columnName]; });
+}
+
+var wrappedQueryTpl = dot.template([
+    'SELECT {{=it._columns}}',
+    'FROM ({{=it._query}}) _cdb_analysis_query'
+].join('\n'));
+
+function layerQuery(node) {
+    if (node.type === 'source') {
+        return node.getQuery();
+    }
+    var _columns = ['ST_Transform(the_geom, 3857) the_geom_webmercator'].concat(skipColumns(node.getColumns()));
+    return wrappedQueryTpl({ _query: node.getQuery(), _columns: _columns.join(', ') });
+}
+
+function dataviewQuery(node, dataviewName, ownFilter) {
+    var applyFilters = {};
+    if (!ownFilter) {
+        applyFilters[dataviewName] = false;
+    }
+
+    if (node.type === 'source') {
+        return node.getQuery(applyFilters);
+    }
+    var _columns = ['ST_Transform(the_geom, 3857) the_geom_webmercator'].concat(skipColumns(node.getColumns()));
+    return wrappedQueryTpl({ _query: node.getQuery(applyFilters), _columns: _columns.join(', ') });
+}
+
+function appendFiltersToNodes(requestMapConfig, dataviewsFiltersBySourceId) {
+    var analyses = requestMapConfig.analyses || [];
+    dataviewsFiltersBySourceId = dataviewsFiltersBySourceId || {};
+
+    requestMapConfig.analyses = analyses.map(function(analysisDefinition) {
+        var analysisGraph = new camshaft.reference.AnalysisGraph(analysisDefinition);
+        var definition = analysisDefinition;
+        Object.keys(dataviewsFiltersBySourceId).forEach(function(sourceId) {
+            definition = analysisGraph.getDefinitionWith(sourceId, {filters: dataviewsFiltersBySourceId[sourceId] });
+        });
+
+        return definition;
+    });
+
+    return requestMapConfig;
+}
+
+function shouldAdaptLayers(requestMapConfig) {
+    return Array.isArray(requestMapConfig.layers) && requestMapConfig.layers.some(getLayerSourceId) ||
+        (Array.isArray(requestMapConfig.analyses) && requestMapConfig.analyses.length > 0) ||
+        requestMapConfig.dataviews;
+}
+
+var DATAVIEW_TYPE_2_FILTER_TYPE = {
+    aggregation: 'category',
+    histogram: 'range'
+};
+function getFilter(dataview, params) {
+    var type = dataview.type;
+
+    return {
+        type: DATAVIEW_TYPE_2_FILTER_TYPE[type],
+        column: dataview.options.column,
+        params: params
+    };
+}
+
 function getLayerSourceId(layer) {
     return layer.options.source && layer.options.source.id;
 }
@@ -195,11 +266,22 @@ function getLayerDataviews(layer, dataviews) {
     return layerDataviews;
 }
 
+function getDataviewsColumns(dataviews) {
+    return Object.keys(dataviews.reduce(function(columnsDict, dataview) {
+        getDataviewColumns(dataview).forEach(function(columnName) {
+            if (!!columnName) {
+                columnsDict[columnName] = true;
+            }
+        });
+        return columnsDict;
+    }, {}));
+}
+
 function getDataviewColumns(dataview) {
     var columns = [];
     var options = dataview.options;
     ['column', 'aggregationColumn'].forEach(function(opt) {
-        if (options.hasOwnProperty(opt)) {
+        if (options.hasOwnProperty(opt) && !!options[opt]) {
             columns.push(options[opt]);
         }
     });
@@ -211,6 +293,15 @@ function getDataviewsList(dataviews) {
 }
 
 function getDataviewsErrors(dataviews) {
+    var dataviewType = typeof dataviews;
+    if (dataviewType !== 'object') {
+        return [new Error('"dataviews" must be a valid JSON object: "' + dataviewType + '" type found')];
+    }
+
+    if (Array.isArray(dataviews)) {
+        return [new Error('"dataviews" must be a valid JSON object: "array" type found')];
+    }
+
     var errors = [];
 
     Object.keys(dataviews).forEach(function(dataviewName) {
@@ -226,3 +317,35 @@ function getDataviewsErrors(dataviews) {
 
     return errors;
 }
+
+function getMissingDataviewsSourceIds(dataviews, sourceId2Node) {
+    var missingDataviewsSourceIds = [];
+    Object.keys(dataviews).forEach(function(dataviewName) {
+        var dataview = dataviews[dataviewName];
+        var dataviewSourceId = getDataviewSourceId(dataview);
+        if (!sourceId2Node.hasOwnProperty(dataviewSourceId)) {
+            missingDataviewsSourceIds.push(new AnalysisError('Node with `source.id="' + dataviewSourceId +'"`' +
+                ' not found in analyses for dataview "' + dataviewName + '"'));
+        }
+    });
+
+    return missingDataviewsSourceIds;
+}
+
+function AnalysisError(message) {
+    Error.captureStackTrace(this, this.constructor);
+    this.name = this.constructor.name;
+    this.type = 'analysis';
+    this.message = message;
+}
+
+function getAllAffectedTablesFromSourceNodes(node) {
+    var affectedTables = node.getAllInputNodes(function (node) {
+        return node.getType() === 'source';
+    }).reduce(function(list, node) {
+        return list.concat(node.getAffectedTables());
+    },[]);
+    return affectedTables;
+}
+
+require('util').inherits(AnalysisError, Error);

lib/cartodb/models/mapconfig/adapter/dataviews-widgets-adapter.js

@@ -0,0 +1,98 @@
+function DataviewsWidgetsMapConfigAdapter() {
+}
+
+module.exports = DataviewsWidgetsMapConfigAdapter;
+
+
+DataviewsWidgetsMapConfigAdapter.prototype.getMapConfig = function(user, requestMapConfig, params, context, callback) {
+    if (!shouldAdapt(requestMapConfig)) {
+        return callback(null, requestMapConfig);
+    }
+
+    // prepare placeholders for new dataviews created from widgets
+    requestMapConfig.analyses = requestMapConfig.analyses || [];
+    requestMapConfig.dataviews = requestMapConfig.dataviews || {};
+
+    requestMapConfig.layers.forEach(function(layer, index) {
+        var layerSourceId = getLayerSourceId(layer);
+
+        if (!layer.options.widgets) {
+            return;
+        }
+
+        if (!layerSourceId && !layer.options.sql) {
+            return;
+        }
+
+        var dataviewSourceId = layerSourceId || 'cdb-layer-source-' + index;
+        // Append a new analysis if layer has no source id but sql.
+        if (!layerSourceId) {
+            requestMapConfig.analyses.push(
+                {
+                    id: dataviewSourceId,
+                    type: 'source',
+                    params: {
+                        query: layer.options.sql
+                    }
+                }
+            );
+        }
+        var source = { id: dataviewSourceId };
+        var layerWidgets = layer.options.widgets || {};
+        Object.keys(layerWidgets).forEach(function(widgetId) {
+            var dataview = layerWidgets[widgetId];
+            requestMapConfig.dataviews[widgetId] = {
+                source: source,
+                type: dataview.type,
+                options: dataview.options
+            };
+        });
+
+        layer.options.source = source;
+
+        delete layer.options.sql;
+        // don't delete widgets for now as it might be useful for old clients
+        //delete layer.options.widgets;
+    });
+
+    // filters have to be rewritten also
+    var filters = getFilters(params);
+    var layersFilters = filters.layers || [];
+    filters.dataviews = filters.dataviews || {};
+
+    layersFilters.forEach(function(layerFilters) {
+        Object.keys(layerFilters).forEach(function(filterName) {
+            if (!filters.dataviews.hasOwnProperty(filterName)) {
+                filters.dataviews[filterName] = layerFilters[filterName];
+            }
+        });
+    });
+
+    delete filters.layers;
+
+    params.filters = JSON.stringify(filters);
+
+    return callback(null, requestMapConfig);
+};
+
+function shouldAdapt(requestMapConfig) {
+    return Array.isArray(requestMapConfig.layers) && requestMapConfig.layers.some(function hasWidgets(layer) {
+        return layer.options && layer.options.widgets && Object.keys(layer.options.widgets).length > 0;
+    });
+}
+
+function getLayerSourceId(layer) {
+    return layer.options.source && layer.options.source.id;
+}
+
+function getFilters(params) {
+    var filters = {};
+    if (params.filters) {
+        try {
+            filters = JSON.parse(params.filters);
+        } catch (e) {
+            // ignore
+        }
+    }
+    return filters;
+}

lib/cartodb/models/mapconfig/adapter/index.js

@@ -0,0 +1,26 @@
+'use strict';
+
+function MapConfigAdapter(adapters) {
+    this.adapters = Array.isArray(adapters) ? adapters : Array.apply(null, arguments);
+}
+
+module.exports = MapConfigAdapter;
+
+MapConfigAdapter.prototype.getMapConfig = function(user, requestMapConfig, params, context, callback) {
+    var self = this;
+    var i = 0;
+    var tasksLeft = this.adapters.length;
+
+    function next(err, _requestMapConfig) {
+        if (err) {
+            return callback(err);
+        }
+        if (tasksLeft-- === 0) {
+            return callback(null, _requestMapConfig);
+        }
+        var nextAdapter = self.adapters[i++];
+        nextAdapter.getMapConfig(user, _requestMapConfig, params, context, next);
+    }
+
+    next(null, requestMapConfig);
+};

lib/cartodb/models/mapconfig/adapter/mapconfig-buffer-size-adapter.js

@@ -0,0 +1,25 @@
+function MapConfigBufferSizeAdapter() {
+    this.formats = ['png', 'png32', 'mvt', 'grid.json'];
+}
+
+module.exports = MapConfigBufferSizeAdapter;
+
+MapConfigBufferSizeAdapter.prototype.getMapConfig = function (user, requestMapConfig, params, context, callback) {
+    if (!context.templateParams || !context.templateParams.buffersize) {
+        return callback(null, requestMapConfig);
+    }
+
+    this.formats.forEach(function (format) {
+        if (Number.isFinite(context.templateParams.buffersize[format])) {
+            if (requestMapConfig.buffersize === undefined) {
+                requestMapConfig.buffersize = {};
+            }
+
+            requestMapConfig.buffersize[format] = context.templateParams.buffersize[format];
+        }
+    });
+
+    setImmediate(function () {
+        callback(null, requestMapConfig);
+    });
+};

lib/cartodb/models/mapconfig/adapter/mapconfig-named-layers-adapter.js

@@ -2,17 +2,20 @@ var queue = require('queue-async');
 var _ = require('underscore');
 var Datasource = require('windshaft').model.Datasource;
 
-function MapConfigNamedLayersAdapter(templateMaps) {
+function MapConfigNamedLayersAdapter(templateMaps, pgConnection) {
     this.templateMaps = templateMaps;
+    this.pgConnection = pgConnection;
 }
 
 module.exports = MapConfigNamedLayersAdapter;
 
-MapConfigNamedLayersAdapter.prototype.getLayers = function(username, layers, dbMetadata, callback) {
+MapConfigNamedLayersAdapter.prototype.getMapConfig = function (user, requestMapConfig, params, context, callback) {
     var self = this;
 
+    var layers = requestMapConfig.layers;
+
     if (!layers) {
-        return callback(null);
+        return callback(null, requestMapConfig);
     }
 
     var adaptLayersQueue = queue(layers.length);
@@ -28,9 +31,9 @@ MapConfigNamedLayersAdapter.prototype.getLayers = function(username, layers, dbM
             var templateConfigParams = layer.options.config || {};
             var templateAuthTokens = layer.options.auth_tokens;
 
-            self.templateMaps.getTemplate(username, templateName, function(err,  template) {
+            self.templateMaps.getTemplate(user, templateName, function(err,  template) {
                 if (err || !template) {
-                    return done(new Error("Template '" + templateName + "' of user '" + username + "' not found"));
+                    return done(new Error("Template '" + templateName + "' of user '" + user + "' not found"));
                 }
 
                 if (self.templateMaps.isAuthorized(template, templateAuthTokens)) {
@@ -40,7 +43,6 @@ MapConfigNamedLayersAdapter.prototype.getLayers = function(username, layers, dbM
 
                     if (nestedNamedLayers.length > 0) {
                         var nestedNamedMapsError = new Error('Nested named layers are not allowed');
-                        // nestedNamedMapsError.http_status = 400;
                         return done(nestedNamedMapsError);
                     }
 
@@ -96,7 +98,10 @@ MapConfigNamedLayersAdapter.prototype.getLayers = function(username, layers, dbM
 
         });
 
-        return callback(null, layers, datasourceBuilder.build());
+        requestMapConfig.layers = layers;
+        context.datasource = datasourceBuilder.build();
+
+        return callback(null, requestMapConfig);
     }
 
 
@@ -104,7 +109,7 @@ MapConfigNamedLayersAdapter.prototype.getLayers = function(username, layers, dbM
 
     if (_.some(layers, isNamedTypeLayer)) {
         // Lazy load dbAuth
-        dbMetadata.setDBAuth(username, dbAuth, function(err) {
+        this.pgConnection.setDBAuth(user, dbAuth, function(err) {
             if (err) {
                 return callback(err);
             }
@@ -114,7 +119,8 @@ MapConfigNamedLayersAdapter.prototype.getLayers = function(username, layers, dbM
             adaptLayersQueue.awaitAll(layersAdaptQueueFinish);
         });
     } else {
-        return callback(null, layers, datasourceBuilder.build());
+        context.datasource = datasourceBuilder.build();
+        return callback(null, requestMapConfig);
     }
 
 };

lib/cartodb/models/mapconfig/adapter/mapconfig-overviews-adapter.js

@@ -9,11 +9,14 @@ function MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi) {
 
 module.exports = MapConfigOverviewsAdapter;
 
-MapConfigOverviewsAdapter.prototype.getLayers = function(username, layers, analysesResults, callback) {
+MapConfigOverviewsAdapter.prototype.getMapConfig = function(user, requestMapConfig, params, context, callback) {
   var self = this;
 
+  var layers = requestMapConfig.layers;
+  var analysesResults = context.analysesResults;
+
   if (!layers || layers.length === 0) {
-      return callback(null, layers);
+      return callback(null, requestMapConfig);
   }
 
   var augmentLayersQueue = queue(layers.length);
@@ -22,7 +25,7 @@ MapConfigOverviewsAdapter.prototype.getLayers = function(username, layers, analy
       if ( layer.type !== 'mapnik' && layer.type !== 'cartodb' ) {
           return done(null, layer);
       }
-      self.overviewsMetadataApi.getOverviewsMetadata(username, layer.options.sql, function(err, metadata){
+      self.overviewsMetadataApi.getOverviewsMetadata(user, layer.options.sql, function(err, metadata){
            if (err) {
                done(err, layer);
            } else {
@@ -30,12 +33,12 @@ MapConfigOverviewsAdapter.prototype.getLayers = function(username, layers, analy
                step(
                    function collectFiltersData() {
                        var filters, unfiltered_query;
-                       if ( layer.options.source && analysesResults ) {
+                       if ( layer.options.source && analysesResults && !layer.options.sql_wrap) {
                            var sourceId = layer.options.source.id;
                            var node = _.find(analysesResults, function(a){ return a.rootNode.params.id === sourceId; });
                            if ( node ) {
                                node = node.rootNode;
-                               filters = node.filters; // TODO: node.getFilters() when available in camshaft
+                               filters = node.getFilters();
                                var filters_disabler = Object.keys(filters).reduce(
                                    function(disabler, filter_id){ disabler[filter_id] = false; return disabler; },
                                    {}
@@ -51,7 +54,7 @@ MapConfigOverviewsAdapter.prototype.getLayers = function(username, layers, analy
                        var next_step = this;
                        if ( filters ) {
                            self.filterStatsApi.getFilterStats(
-                               username,
+                               user,
                                unfiltered_query, filters,
                                function(err, stats) {
                                  if ( !err ) {
@@ -69,7 +72,7 @@ MapConfigOverviewsAdapter.prototype.getLayers = function(username, layers, analy
                            layer = _.extend({}, layer);
                            layer.options = _.extend({}, layer.options, { query_rewrite_data: query_rewrite_data });
                        }
-                       done(err, layer);
+                       done(null, layer);
                    }
                );
            }
@@ -85,7 +88,9 @@ MapConfigOverviewsAdapter.prototype.getLayers = function(username, layers, analy
         return callback(new Error('Missing layers array from layergroup config'));
     }
 
-    return callback(null, layers);
+    requestMapConfig.layers = layers;
+
+    return callback(null, requestMapConfig);
   }
 
   layers.forEach(function(layer) {

lib/cartodb/models/mapconfig/adapter/sql-wrap-mapconfig-adapter.js

@@ -0,0 +1,25 @@
+function SqlWrapMapConfigAdapter() {
+}
+
+module.exports = SqlWrapMapConfigAdapter;
+
+
+SqlWrapMapConfigAdapter.prototype.getMapConfig = function(user, requestMapConfig, params, context, callback) {
+    if (requestMapConfig && Array.isArray(requestMapConfig.layers)) {
+        requestMapConfig.layers = requestMapConfig.layers.map(function(layer) {
+            if (layer.options) {
+                var sqlQueryWrap = layer.options.sql_wrap;
+                if (sqlQueryWrap) {
+                    var layerSql = layer.options.sql;
+                    if (layerSql) {
+                        layer.options.sql_raw = layerSql;
+                        layer.options.sql = sqlQueryWrap.replace(/<%=\s*sql\s*%>/g, layerSql);
+                    }
+                }
+            }
+            return layer;
+        });
+    }
+
+    return callback(null, requestMapConfig);
+};

lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js

@@ -0,0 +1,180 @@
+'use strict';
+
+var dot = require('dot');
+dot.templateSettings.strip = false;
+var queue = require('queue-async');
+var PSQL = require('cartodb-psql');
+var turboCarto = require('turbo-carto');
+
+var SubstitutionTokens = require('../../../utils/substitution-tokens');
+var PostgresDatasource = require('../../../backends/turbo-carto-postgres-datasource');
+
+var MapConfig = require('windshaft').model.MapConfig;
+
+function TurboCartoAdapter() {
+}
+
+module.exports = TurboCartoAdapter;
+
+TurboCartoAdapter.prototype.getMapConfig = function (user, requestMapConfig, params, context, callback) {
+    var self = this;
+
+    var layers = requestMapConfig.layers;
+
+    if (!layers || layers.length === 0) {
+        return callback(null, requestMapConfig);
+    }
+
+    var parseCartoQueue = queue(layers.length);
+
+    layers.forEach(function(layer, index) {
+        var layerId = MapConfig.getLayerId(requestMapConfig, index);
+        parseCartoQueue.defer(self._parseCartoCss.bind(self), user, params, layer, index, layerId);
+    });
+
+    parseCartoQueue.awaitAll(function (err, results) {
+        if (err) {
+            return callback(err);
+        }
+
+        var errors = results.reduce(function(errors, result) {
+            if (result.error) {
+                errors.push(result.error);
+            }
+            return errors;
+        }, []);
+        if (errors.length > 0) {
+            return callback(errors);
+        }
+
+        requestMapConfig.layers = results.map(function(result) { return result.layer; });
+        context.turboCarto = {
+            layers: results.map(function(result) {
+                return result.meta;
+            })
+        };
+
+        return callback(null, requestMapConfig);
+    });
+};
+
+var tokensQueryTpl = dot.template([
+    'WITH input_query AS (',
+    '  {{=it._sql}}',
+    '),',
+    'bbox_query AS (',
+    '  SELECT ST_SetSRID(ST_Extent(the_geom_webmercator), 3857) as bbox from input_query',
+    '),',
+    'zoom_query as (',
+    '  SELECT GREATEST(',
+    '  ceil(log(40075017000 / 256 / GREATEST(ST_XMax(bbox) - ST_XMin(bbox), ST_YMax(bbox) - ST_YMin(bbox)))/log(2)),',
+    '  0) as zoom',
+    '  FROM bbox_query',
+    '),',
+    'pixel_size_query as (',
+    '  SELECT 40075017 * cos(radians(ST_Y(ST_Transform(ST_Centroid(bbox), 4326)))) / 2 ^ ((zoom) + 8) as pixel_size',
+    '  FROM bbox_query, zoom_query',
+    '),',
+    'scale_denominator_query as (',
+    '  SELECT (pixel_size / 0.00028)::numeric as scale_denominator',
+    '  FROM pixel_size_query',
+    ')',
+    'select ST_AsText(bbox) bbox, pixel_size, scale_denominator, zoom',
+    'from bbox_query, pixel_size_query, scale_denominator_query, zoom_query'
+].join('\n'));
+
+TurboCartoAdapter.prototype._parseCartoCss = function (username, params, layer, layerIndex, layerId, callback) {
+    if (!shouldParseLayerCartocss(layer)) {
+        return callback(null, { layer: layer });
+    }
+
+    var pg = new PSQL(dbParamsFromReqParams(params));
+    function processCallback(err, cartocss, meta) {
+        // Only return turbo-carto errors
+        if (err && err.name === 'TurboCartoError') {
+            var error = new Error(err.message);
+            error.http_status = 400;
+            error.type = 'layer';
+            error.subtype = 'turbo-carto';
+            error.layer = {
+                id: layerId,
+                index: layerIndex,
+                type: layer.type,
+                context: err.context
+            };
+
+            return callback(null, { error: error });
+        }
+
+        // Try to continue in the rest of the cases
+        if (cartocss) {
+            layer.options.cartocss = cartocss;
+        }
+        return callback(null, { layer: layer, meta: meta });
+    }
+
+    var layerSql = layer.options.sql;
+    var layerRawSql = layer.options.sql_raw;
+    if (SubstitutionTokens.hasTokens(layerSql) && layerRawSql) {
+        var self = this;
+        var tokensQuery = tokensQueryTpl({_sql: layerRawSql});
+        return pg.query(tokensQuery, function(err, resultSet) {
+            if (err) {
+                return processCallback(err);
+            }
+
+            resultSet = resultSet || {};
+            var rows = resultSet.rows || [];
+            var result = rows[0] || {};
+
+            var tokens = {
+                bbox: 'ST_SetSRID(ST_GeomFromText(\'' + result.bbox + '\'), 3857)',
+                scale_denominator: result.scale_denominator,
+                pixel_width: result.pixel_size,
+                pixel_height: result.pixel_size
+            };
+
+            var sql = SubstitutionTokens.replace(layerSql, tokens);
+            self.process(pg, layer.options.cartocss, sql, processCallback);
+        }, true); // use read-only transaction
+    }
+
+    var tokens = {
+        bbox: 'ST_MakeEnvelope(-20037508.34,-20037508.34,20037508.34,20037508.34,3857)',
+        scale_denominator: '500000001',
+        pixel_width: '156412',
+        pixel_height: '156412'
+    };
+
+    var sql = SubstitutionTokens.replace(layerSql, tokens);
+    this.process(pg, layer.options.cartocss, sql, processCallback);
+};
+
+TurboCartoAdapter.prototype.process = function (psql, cartocss, sql, callback) {
+    var datasource = new PostgresDatasource(psql, sql);
+    turboCarto(cartocss, datasource, callback);
+};
+
+function shouldParseLayerCartocss(layer) {
+    return layer && layer.options && layer.options.cartocss && layer.options.sql;
+}
+
+function dbParamsFromReqParams(params) {
+    var dbParams = {};
+    if ( params.dbuser ) {
+        dbParams.user = params.dbuser;
+    }
+    if ( params.dbpassword ) {
+        dbParams.pass = params.dbpassword;
+    }
+    if ( params.dbhost ) {
+        dbParams.host = params.dbhost;
+    }
+    if ( params.dbport ) {
+        dbParams.port = params.dbport;
+    }
+    if ( params.dbname ) {
+        dbParams.dbname = params.dbname;
+    }
+    return dbParams;
+}

lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js

@@ -1,7 +1,7 @@
 var assert = require('assert');
 var step = require('step');
 
-var MapStoreMapConfigProvider = require('./map_store_provider');
+var MapStoreMapConfigProvider = require('./map-store-provider');
 
 /**
  * @param {MapConfig} mapConfig
@@ -26,7 +26,7 @@ CreateLayergroupMapConfigProvider.prototype.getMapConfig = function(callback) {
     var context = {};
     step(
         function prepareContextLimits() {
-            self.userLimitsApi.getRenderLimits(self.user, this);
+            self.userLimitsApi.getRenderLimits(self.user, self.params.api_key, this);
         },
         function handleRenderLimits(err, renderLimits) {
             assert.ifError(err);

lib/cartodb/models/mapconfig/provider/map-store-provider.js

@@ -27,7 +27,7 @@ MapStoreMapConfigProvider.prototype.getMapConfig = function(callback) {
     var context = {};
     step(
         function prepareContextLimits() {
-            self.userLimitsApi.getRenderLimits(self.user, this);
+            self.userLimitsApi.getRenderLimits(self.user, self.params.api_key, this);
         },
         function handleRenderLimits(err, renderLimits) {
             assert.ifError(err);

lib/cartodb/models/mapconfig/provider/named-map-provider.js

@@ -4,24 +4,20 @@ var crypto = require('crypto');
 var dot = require('dot');
 var step = require('step');
 var MapConfig = require('windshaft').model.MapConfig;
-var templateName = require('../../backends/template_maps').templateName;
+var templateName = require('../../../backends/template_maps').templateName;
 var QueryTables = require('cartodb-query-tables');
 
 /**
  * @constructor
  * @type {NamedMapMapConfigProvider}
  */
-function NamedMapMapConfigProvider(templateMaps, pgConnection, metadataBackend, userLimitsApi,
-                                   namedLayersAdapter, overviewsAdapter, turboCartoAdapter, analysisMapConfigAdapter,
+function NamedMapMapConfigProvider(templateMaps, pgConnection, metadataBackend, userLimitsApi, mapConfigAdapter,
                                    owner, templateId, config, authToken, params) {
     this.templateMaps = templateMaps;
     this.pgConnection = pgConnection;
     this.metadataBackend = metadataBackend;
     this.userLimitsApi = userLimitsApi;
-    this.namedLayersAdapter = namedLayersAdapter;
-    this.turboCartoAdapter = turboCartoAdapter;
-    this.analysisMapConfigAdapter = analysisMapConfigAdapter;
-    this.overviewsAdapter = overviewsAdapter;
+    this.mapConfigAdapter = mapConfigAdapter;
 
     this.owner = owner;
     this.templateName = templateName(templateId);
@@ -54,10 +50,11 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) {
     var self = this;
 
     var mapConfig = null;
-    var datasource = null;
     var rendererParams;
     var apiKey;
 
+    var context = {};
+
     step(
         function getTemplate() {
             self.getTemplate(this);
@@ -93,11 +90,13 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) {
         },
         function instantiateTemplate(err, templateParams) {
             assert.ifError(err);
+            context.templateParams = templateParams;
             return self.templateMaps.instance(self.template, templateParams);
         },
-        function prepareAnalysisLayers(err, requestMapConfig) {
+        function prepareAdapterMapConfig(err, requestMapConfig) {
             assert.ifError(err);
-            var analysisConfiguration = {
+            context.analysisConfiguration = {
+                user: self.owner,
                 db: {
                     host: rendererParams.dbhost,
                     port: rendererParams.dbport,
@@ -110,76 +109,19 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) {
                     apiKey: apiKey
                 }
             };
-
-            var filters = {};
-            if (self.params.filters) {
-                try {
-                    filters = JSON.parse(self.params.filters);
-                } catch (e) {
-                    // ignore
-                }
-            }
-            self.analysisMapConfigAdapter.getMapConfig(analysisConfiguration, requestMapConfig, filters, this);
+            self.mapConfigAdapter.getMapConfig(self.owner, requestMapConfig, rendererParams, context, this);
         },
-        function prepareLayergroup(err, _mapConfig, analysesResults) {
-            assert.ifError(err);
-            var next = this;
-            self.analysesResults = analysesResults || [];
-            self.namedLayersAdapter.getLayers(self.owner, _mapConfig.layers, self.pgConnection,
-                function(err, layers, datasource) {
-                    if (err) {
-                        return next(err);
-                    }
-
-                    if (layers) {
-                        _mapConfig.layers = layers;
-                    }
-                    return next(null, _mapConfig, datasource);
-                }
-            );
-        },
-        function addOverviewsInformation(err, _mapConfig, datasource) {
-            assert.ifError(err);
-            var next = this;
-
-            self.overviewsAdapter.getLayers(self.owner, _mapConfig.layers, self.analysesResults, function(err, layers) {
-                if (err) {
-                    return next(err);
-                }
-
-                if (layers) {
-                    _mapConfig.layers = layers;
-                }
-
-                return next(null, _mapConfig, datasource);
-            });
-        },
-        function parseTurboCarto(err, _mapConfig, datasource) {
-            assert.ifError(err);
-            var next = this;
-
-            self.turboCartoAdapter.getLayers(self.owner, _mapConfig.layers, function (err, layers) {
-                if (err) {
-                    return next(err);
-                }
-
-                if (layers) {
-                    _mapConfig.layers = layers;
-                }
-
-                return next(null, _mapConfig, datasource);
-            });
-        },
-        function prepareContextLimits(err, _mapConfig, _datasource) {
+        function prepareContextLimits(err, _mapConfig) {
             assert.ifError(err);
             mapConfig = _mapConfig;
-            datasource = _datasource;
-            self.userLimitsApi.getRenderLimits(self.owner, this);
+            self.userLimitsApi.getRenderLimits(self.owner, self.params.api_key, this);
         },
         function cacheAndReturnMapConfig(err, renderLimits) {
             self.err = err;
-            self.mapConfig = (mapConfig === null) ? null : new MapConfig(mapConfig, datasource);
+            self.mapConfig = (mapConfig === null) ? null : new MapConfig(mapConfig, context.datasource);
+            self.analysesResults = context.analysesResults || [];
             self.rendererParams = rendererParams;
+            self.context = context;
             self.context.limits = renderLimits || {};
             return callback(self.err, self.mapConfig, self.rendererParams, self.context);
         }

lib/cartodb/models/resource-locator.js

@@ -0,0 +1,258 @@
+var dot = require('dot');
+dot.templateSettings.strip = false;
+
+function ResourceLocator(environment) {
+    this.environment = environment;
+
+    this.resourcesUrlTemplates = null;
+    if (this.environment.resources_url_templates) {
+        var templates = environment.resources_url_templates;
+
+        if (templates.http) {
+            this.resourcesUrlTemplates = this.resourcesUrlTemplates || {};
+            this.resourcesUrlTemplates.http = dot.template(templates.http + '/{{=it.resource}}');
+        }
+        if (templates.https) {
+            this.resourcesUrlTemplates = this.resourcesUrlTemplates || {};
+            this.resourcesUrlTemplates.https = dot.template(templates.https + '/{{=it.resource}}');
+        }
+    }
+}
+
+module.exports = ResourceLocator;
+
+ResourceLocator.prototype.getTileUrls = function(username, resourcePath) {
+    if (this.resourcesUrlTemplates) {
+        const urls = this.getUrlsFromTemplate(username, new TileResource(resourcePath));
+        return {
+            http: Array.isArray(urls.http) ? urls.http : [urls.http],
+            https: Array.isArray(urls.https) ? urls.https : [urls.https]
+        };
+    }
+    var cdnUrls = getCdnUrls(this.environment.serverMetadata, username, new TileResource(resourcePath));
+    if (cdnUrls) {
+        return cdnUrls;
+    } else {
+        var port = this.environment.port;
+        return {
+            http: [`http://${username}.localhost.lan:${port}/api/v1/map/${resourcePath}`]
+        };
+    }
+};
+
+ResourceLocator.prototype.getTemplateUrls = function(username, resourcePath) {
+    if (this.resourcesUrlTemplates) {
+        return this.getUrlsFromTemplate(username, new TemplateResource(resourcePath), true);
+    }
+    var cdnUrls = getCdnUrls(this.environment.serverMetadata, username, new TemplateResource(resourcePath));
+    if (cdnUrls) {
+        return cdnUrls;
+    } else {
+        var port = this.environment.port;
+        return {
+            http: {
+                urlTemplate: `http://${username}.localhost.lan:${port}/api/v1/map/${resourcePath}`,
+                subdomains: []
+            }
+        };
+    }
+};
+
+ResourceLocator.prototype.getUrls = function(username, resourcePath) {
+    if (this.resourcesUrlTemplates) {
+        return this.getUrlsFromTemplate(username, new Resource(resourcePath));
+    }
+    var cdnUrls = getCdnUrls(this.environment.serverMetadata, username, new Resource(resourcePath));
+    if (cdnUrls) {
+        return cdnUrls;
+    } else {
+        var port = this.environment.port;
+        return {
+            http: `http://${username}.localhost.lan:${port}/api/v1/map/${resourcePath}`
+        };
+    }
+};
+
+function urlForTemplate(tpl, username, cdnDomain, resource, templated) {
+    cdnDomain = cdnDomain || {};
+    if (templated) {
+        return {
+            urlTemplate: tpl({
+                cdn_url: (cdnDomain.hasOwnProperty('urlTemplate') ? cdnDomain.urlTemplate : cdnDomain),
+                user: username,
+                port: this.environment.port,
+                resource: resource.getPath()
+            }),
+            subdomains: cdnDomain.subdomains || []
+        };
+    }
+    if (Array.isArray(cdnDomain)) {
+        return cdnDomain.map(d => tpl({
+            cdn_url: d,
+            user: username,
+            port: this.environment.port,
+            resource: resource.getPath()
+        }));
+    } else {
+        return tpl({
+            cdn_url: cdnDomain,
+            user: username,
+            port: this.environment.port,
+            resource: resource.getPath()
+        });
+    }
+}
+
+ResourceLocator.prototype.getUrlsFromTemplate = function(username, resource, templated) {
+    var urls = {};
+    var cdnDomain = getCdnDomain(this.environment.serverMetadata, resource) || {};
+    if (this.resourcesUrlTemplates.http) {
+        urls.http = urlForTemplate(this.resourcesUrlTemplates.http, username, cdnDomain.http, resource, templated);
+    }
+    if (this.resourcesUrlTemplates.https) {
+        urls.https = urlForTemplate(this.resourcesUrlTemplates.https, username, cdnDomain.https, resource, templated);
+    }
+
+    return urls;
+};
+
+class Resource {
+    constructor (resourcePath) {
+        this.resourcePath = resourcePath;
+    }
+
+    getPath () {
+        return this.resourcePath;
+    }
+
+    getDomain (domain, subdomains) {
+        if (!subdomains) {
+            return domain;
+        }
+        return domain.replace('{s}', subdomain(subdomains, this.resourcePath));
+    }
+
+    getUrl (baseUrl, username, subdomains) {
+        let urls = getUrl(baseUrl, username, this.resourcePath);
+        if (subdomains) {
+            urls = urls.replace('{s}', subdomain(subdomains, this.resourcePath));
+        }
+        return urls;
+    }
+}
+
+class TileResource extends Resource {
+    constructor (resourcePath) {
+        super(resourcePath);
+    }
+
+    getDomain (domain, subdomains) {
+        if (!subdomains) {
+            return domain;
+        }
+        return subdomains.map(s => domain.replace('{s}', s));
+    }
+
+    getUrl (baseUrl, username, subdomains) {
+        if (!subdomains) {
+            return [super.getUrl(baseUrl, username)];
+        }
+        return subdomains.map(subdomain => {
+            return getUrl(baseUrl, username, this.resourcePath)
+                .replace('{s}', subdomain);
+        });
+    }
+}
+
+class TemplateResource extends Resource {
+    constructor (resourcePath) {
+        super(resourcePath);
+    }
+
+    getDomain (domain, subdomains) {
+        return {
+            urlTemplate: domain,
+            subdomains: subdomains || []
+        };
+    }
+
+    getUrl (baseUrl, username, subdomains) {
+        return {
+            urlTemplate: getUrl(baseUrl, username, this.resourcePath),
+            subdomains: subdomains || []
+        };
+    }
+}
+
+function getUrl(baseUrl, username, path) {
+    return `${baseUrl}/${username}/api/v1/map/${path}`;
+}
+
+function getCdnUrls(serverMetadata, username, resource) {
+    if (serverMetadata && serverMetadata.cdn_url) {
+        var cdnUrl = serverMetadata.cdn_url;
+        var httpUrls = resource.getUrl(`http://${cdnUrl.http}`, username);
+        var httpsUrls = resource.getUrl(`https://${cdnUrl.https}`, username);
+        if (cdnUrl.templates) {
+            var templates = cdnUrl.templates;
+            httpUrls = resource.getUrl(templates.http.url, username, templates.http.subdomains);
+            httpsUrls = resource.getUrl(templates.https.url, username, templates.https.subdomains);
+        }
+        return {
+            http: httpUrls,
+            https: httpsUrls,
+        };
+    }
+    return null;
+}
+
+function getCdnDomain(serverMetadata, resource) {
+    if (serverMetadata && serverMetadata.cdn_url) {
+        var cdnUrl = serverMetadata.cdn_url;
+        var httpDomain = resource.getDomain(cdnUrl.http);
+        var httpsDomain = resource.getDomain(cdnUrl.https);
+        if (cdnUrl.templates) {
+            var templates = cdnUrl.templates;
+            var httpUrlTemplate = templates.http.url;
+            var httpsUrlTemplate = templates.https.url;
+            httpDomain = httpUrlTemplate.replace(/^(http[s]*:\/\/)/, '');
+            httpDomain = resource.getDomain(httpDomain, templates.http.subdomains);
+            httpsDomain = httpsUrlTemplate.replace(/^(http[s]*:\/\/)/, '');
+            httpsDomain = resource.getDomain(httpsDomain, templates.https.subdomains);
+        }
+        return {
+            http: httpDomain,
+            https: httpsDomain,
+        };
+    }
+    return null;
+}
+
+// ref https://jsperf.com/js-crc32
+function crcTable() {
+    var c;
+    var table = [];
+    for (var n = 0; n < 256; n++) {
+      c = n;
+      for (var k = 0; k < 8; k++) {
+        c = ((c & 1) ? (0xEDB88320 ^ (c >>> 1)) : (c >>> 1));
+      }
+      table[n] = c;
+    }
+    return table;
+}
+var CRC_TABLE = crcTable();
+
+function crc32(str) {
+    var crc = 0 ^ (-1);
+    for (var i = 0; i < str.length; i++) {
+      crc = (crc >>> 8) ^ CRC_TABLE[(crc ^ str.charCodeAt(i)) & 0xFF];
+    }
+    return (crc ^ (-1)) >>> 0;
+}
+
+function subdomain(subdomains, resource) {
+    var index = crc32(resource) % subdomains.length;
+    return subdomains[index];
+}
+module.exports.subdomain = subdomain;

lib/cartodb/server.js

@@ -3,7 +3,6 @@ var bodyParser = require('body-parser');
 var RedisPool = require('redis-mpool');
 var cartodbRedis = require('cartodb-redis');
 var _ = require('underscore');
-var debug = require('debug')('windshaft:cartodb');
 
 var controller = require('./controllers');
 
@@ -13,7 +12,8 @@ var VarnishHttpCacheBackend = require('./cache/backend/varnish_http');
 var FastlyCacheBackend = require('./cache/backend/fastly');
 
 var StatsClient = require('./stats/client');
-var Profiler = require('./stats/profiler_proxy');
+const stats = require('./middleware/stats');
+
 var RendererStatsReporter = require('./stats/reporter/renderer');
 
 var windshaft = require('windshaft');
@@ -34,10 +34,22 @@ var AnalysisBackend = require('./backends/analysis');
 var timeoutErrorTilePath = __dirname + '/../../assets/render-timeout-fallback.png';
 var timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, {encoding: null});
 
-var MapConfigOverviewsAdapter = require('./models/mapconfig_overviews_adapter');
+var SqlWrapMapConfigAdapter = require('./models/mapconfig/adapter/sql-wrap-mapconfig-adapter');
+var MapConfigNamedLayersAdapter = require('./models/mapconfig/adapter/mapconfig-named-layers-adapter');
+var MapConfigBufferSizeAdapter = require('./models/mapconfig/adapter/mapconfig-buffer-size-adapter');
+var AnalysisMapConfigAdapter = require('./models/mapconfig/adapter/analysis-mapconfig-adapter');
+var MapConfigOverviewsAdapter = require('./models/mapconfig/adapter/mapconfig-overviews-adapter');
+var TurboCartoAdapter = require('./models/mapconfig/adapter/turbo-carto-adapter');
+var DataviewsWidgetsAdapter = require('./models/mapconfig/adapter/dataviews-widgets-adapter');
+var AggregationMapConfigAdapter = require('./models/mapconfig/adapter/aggregation-mapconfig-adapter');
+var MapConfigAdapter = require('./models/mapconfig/adapter');
 
-var TurboCartoParser = require('./utils/style/turbo-carto-parser');
-var TurboCartoAdapter = require('./utils/style/turbo-carto-adapter');
+var StatsBackend = require('./backends/stats');
+
+const lzmaMiddleware = require('./middleware/lzma');
+const errorMiddleware = require('./middleware/error-middleware');
+
+const prepareContextMiddleware = require('./middleware/context');
 
 module.exports = function(serverOptions) {
     // Make stats client globally accessible
@@ -113,8 +125,27 @@ module.exports = function(serverOptions) {
     var onTileErrorStrategy;
     if (global.environment.enabledFeatures.onTileErrorStrategy !== false) {
         onTileErrorStrategy = function onTileErrorStrategy$TimeoutTile(err, tile, headers, stats, format, callback) {
-            if (err && err.message === 'Render timed out' && format === 'png') {
-                return callback(null, timeoutErrorTile, { 'Content-Type': 'image/png' }, {});
+
+            function isRenderTimeoutError (err) {
+                return err.message === 'Render timed out';
+            }
+
+            function isDatasourceTimeoutError (err) {
+                return err.message && err.message.match(/canceling statement due to statement timeout/i);
+            }
+
+            function isTimeoutError (err) {
+                return isRenderTimeoutError(err) || isDatasourceTimeoutError(err);
+            }
+
+            function isRasterFormat (format) {
+                return format === 'png' || format === 'jpg';
+            }
+
+            if (isTimeoutError(err) && isRasterFormat(format)) {
+                return callback(null, timeoutErrorTile, {
+                    'Content-Type': 'image/png',
+                }, {});
             } else {
                 return callback(err, tile, headers, stats);
             }
@@ -128,7 +159,8 @@ module.exports = function(serverOptions) {
             grainstore: serverOptions.grainstore,
             mapnik: serverOptions.renderer.mapnik
         },
-        http: serverOptions.renderer.http
+        http: serverOptions.renderer.http,
+        mvt: serverOptions.renderer.mvt
     });
 
     // initialize render cache
@@ -145,24 +177,31 @@ module.exports = function(serverOptions) {
     var tileBackend = new windshaft.backend.Tile(rendererCache);
     var mapValidatorBackend = new windshaft.backend.MapValidator(tileBackend, attributesBackend);
     var mapBackend = new windshaft.backend.Map(rendererCache, mapStore, mapValidatorBackend);
-    var analysisBackend = new AnalysisBackend(serverOptions.analysis);
+
+    var analysisBackend = new AnalysisBackend(metadataBackend, serverOptions.analysis);
+
+    var statsBackend = new StatsBackend();
 
     var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache();
     app.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
 
-    var overviewsAdapter = new MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi);
-
-    var turboCartoParser = new TurboCartoParser(pgQueryRunner);
-    var turboCartoAdapter = new TurboCartoAdapter(turboCartoParser);
+    var mapConfigAdapter = new MapConfigAdapter(
+        new MapConfigNamedLayersAdapter(templateMaps, pgConnection),
+        new MapConfigBufferSizeAdapter(),
+        new SqlWrapMapConfigAdapter(),
+        new DataviewsWidgetsAdapter(),
+        new AnalysisMapConfigAdapter(analysisBackend),
+        new AggregationMapConfigAdapter(pgConnection),
+        new MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi),
+        new TurboCartoAdapter()
+    );
 
     var namedMapProviderCache = new NamedMapProviderCache(
         templateMaps,
         pgConnection,
         metadataBackend,
-        analysisBackend,
         userLimitsApi,
-        overviewsAdapter,
-        turboCartoAdapter
+        mapConfigAdapter
     );
 
     ['update', 'delete'].forEach(function(eventType) {
@@ -174,18 +213,23 @@ module.exports = function(serverOptions) {
     var TablesExtentApi = require('./api/tables_extent_api');
     var tablesExtentApi = new TablesExtentApi(pgQueryRunner);
 
+    var versions = getAndValidateVersions(serverOptions);
+
+    const prepareContext = typeof serverOptions.req2params === 'function' ?
+        serverOptions.req2params :
+        prepareContextMiddleware(authApi, pgConnection);
+
     /*******************************************************************************************************************
      * Routing
      ******************************************************************************************************************/
 
     new controller.Layergroup(
-        authApi,
+        prepareContext,
         pgConnection,
         mapStore,
         tileBackend,
         previewBackend,
         attributesBackend,
-        new windshaft.backend.Widget(),
         surrogateKeysCache,
         userLimitsApi,
         layergroupAffectedTablesCache,
@@ -193,7 +237,7 @@ module.exports = function(serverOptions) {
     ).register(app);
 
     new controller.Map(
-        authApi,
+        prepareContext,
         pgConnection,
         templateMaps,
         mapBackend,
@@ -201,14 +245,12 @@ module.exports = function(serverOptions) {
         surrogateKeysCache,
         userLimitsApi,
         layergroupAffectedTablesCache,
-        overviewsAdapter,
-        turboCartoAdapter,
-        analysisBackend
+        mapConfigAdapter,
+        statsBackend
     ).register(app);
 
     new controller.NamedMaps(
-        authApi,
-        pgConnection,
+        prepareContext,
         namedMapProviderCache,
         tileBackend,
         previewBackend,
@@ -217,14 +259,18 @@ module.exports = function(serverOptions) {
         metadataBackend
     ).register(app);
 
-    new controller.NamedMapsAdmin(authApi, pgConnection, templateMaps).register(app);
+    new controller.NamedMapsAdmin(authApi, templateMaps).register(app);
 
-    new controller.ServerInfo().register(app);
+    new controller.Analyses(prepareContext).register(app);
+
+    new controller.ServerInfo(versions).register(app);
 
     /*******************************************************************************************************************
      * END Routing
      ******************************************************************************************************************/
 
+    app.use(errorMiddleware());
+
     return app;
 };
 
@@ -232,12 +278,45 @@ function validateOptions(opts) {
     if (!_.isString(opts.base_url) || !_.isString(opts.base_url_mapconfig) || !_.isString(opts.base_url_templated)) {
         throw new Error("Must initialise server with: 'base_url'/'base_url_mapconfig'/'base_url_templated' URLs");
     }
+}
 
-    // Be nice and warn if configured mapnik version is != instaled mapnik version
-    if (mapnik.versions.mapnik !== opts.grainstore.mapnik_version) {
-        debug('WARNING: detected mapnik version (' + mapnik.versions.mapnik + ')' +
-            ' != configured mapnik version (' + opts.grainstore.mapnik_version + ')');
+function getAndValidateVersions(options) {
+    // jshint undef:false
+    var warn = console.warn.bind(console);
+    // jshint undef:true
+
+    var packageDefinition = require('../../package.json');
+
+    var declaredDependencies = packageDefinition.dependencies || {};
+    var installedDependenciesVersions = {
+        camshaft: require('camshaft').version,
+        grainstore: windshaft.grainstore.version(),
+        mapnik: windshaft.mapnik.versions.mapnik,
+        node_mapnik: windshaft.mapnik.version,
+        'turbo-carto': require('turbo-carto').version,
+        windshaft: windshaft.version,
+        windshaft_cartodb: packageDefinition.version
+    };
+
+    var dependenciesToValidate = ['camshaft', 'turbo-carto', 'windshaft'];
+    dependenciesToValidate.forEach(function(depName) {
+        var declaredDependencyVersion = declaredDependencies[depName];
+        var installedDependencyVersion = installedDependenciesVersions[depName];
+        if (declaredDependencyVersion !== installedDependencyVersion) {
+            warn(
+                'Dependency="%s" installed version="%s" does not match declared version="%s". Check your installation.',
+                depName, installedDependencyVersion, declaredDependencyVersion
+            );
+        }
+    });
+
+    // Be nice and warn if configured mapnik version is != installed mapnik version
+    if (mapnik.versions.mapnik !== options.grainstore.mapnik_version) {
+        warn('WARNING: detected mapnik version (' + mapnik.versions.mapnik + ')' +
+            ' != configured mapnik version (' + options.grainstore.mapnik_version + ')');
     }
+
+    return installedDependenciesVersions;
 }
 
 function bootstrapFonts(opts) {
@@ -264,15 +343,28 @@ function bootstrap(opts) {
     app.enable('jsonp callback');
     app.disable('x-powered-by');
     app.disable('etag');
+
+    // Fix: https://github.com/CartoDB/Windshaft-cartodb/issues/705
+    // See: http://expressjs.com/en/4x/api.html#app.set
+    app.set('json replacer', function (key, value) {
+        if (value !== value) {
+            return 'NaN';
+        }
+
+        if (value === Infinity) {
+            return 'Infinity';
+        }
+
+        if (value === -Infinity) {
+            return '-Infinity';
+        }
+
+        return value;
+    });
+
     app.use(bodyParser.json());
 
     app.use(function bootstrap$prepareRequestResponse(req, res, next) {
-        req.context = req.context || {};
-        req.profiler = new Profiler({
-            statsd_client: global.statsClient,
-            profile: opts.useProfiler
-        });
-
         if (global.environment && global.environment.api_hostname) {
             res.set('X-Served-By-Host', global.environment.api_hostname);
         }
@@ -280,6 +372,13 @@ function bootstrap(opts) {
         next();
     });
 
+    app.use(stats({
+        enabled: opts.useProfiler,
+        statsClient: global.statsClient
+    }));
+
+    app.use(lzmaMiddleware);
+
     // temporary measure until we upgrade to newer version expressjs so we can check err.status
     app.use(function(err, req, res, next) {
         if (err) {