Release 5.2.0

Add Mapnik 'cache-features' option

.jshintignore

@@ -1,4 +1,3 @@
 test/results/
 test/monkey/
 test/benchmark.js
-test/support/

.travis.yml

@@ -1,27 +1,14 @@
+sudo: required
 dist: trusty
-addons:
-  postgresql: "9.5"
-  apt:
-    sources:
-      - ubuntu-toolchain-r-test
-    packages:
-      - postgresql-9.5-postgis-2.3
-      - postgresql-plpython-9.5
-      - pkg-config
-      - libcairo2-dev
-      - libjpeg8-dev
-      - libgif-dev
-      - libpango1.0-dev
-      - g++-4.9
+
+services:
+  - docker
 
 before_install:
-  - createdb template_postgis
-  - createuser publicuser
-  - psql -c "CREATE EXTENSION postgis" template_postgis
+  - docker pull cartoimages/windshaft-testing
 
-env:
-  - NPROCS=1 JOBS=1 PGUSER=postgres CXX=g++-4.9
+script:
+  - docker run -e POSTGIS_VERSION=2.4 -v `pwd`:/srv cartoimages/windshaft-testing bash docker-test.sh
+
+language: generic
 
-language: node_js
-node_js:
-  - "6"

INSTALL.md

@@ -5,7 +5,7 @@ Make sure that you have the requirements needed. These are
 
 - Core
   - Node.js >=6.9.x
-  - yarn >=0.21.3
+  - yarn >=0.27.5 <1.0.0
   - PostgreSQL >8.3.x, PostGIS >1.5.x
   - Redis >2.4.0 (http://www.redis.io)
   - Mapnik >3.x. See [Installing Mapnik](https://github.com/CartoDB/Windshaft#installing-mapnik).

Makefile

@@ -16,7 +16,7 @@ config.status--test:
 	./configure --environment=test
 
 config/environments/test.js: config.status--test
-	./config.status--test 
+	./config.status--test
 
 TEST_SUITE := $(shell find test/{acceptance,integration,unit} -name "*.js")
 TEST_SUITE_UNIT := $(shell find test/unit -name "*.js")

NEWS.md

@@ -1,5 +1,204 @@
 # Changelog
 
+## 5.2.0
+Released 2018-02-01
+
+Announcements:
+ - Upgrade windshaft to [4.3.3](https://github.com/CartoDB/windshaft/releases/tag/4.3.2) adding support for cache-features' in Mapnik/CartoDB layers.
+
+## 5.1.0
+Released 2018-01-30
+New features:
+ - Now mapnik has support for fine-grained metrics.
+ - Variables can be passed for later substitution in postgis datasource.
+
+Announcements:
+ - Upgrade windshaft to [4.3.1](https://github.com/CartoDB/windshaft/releases/tag/4.3.1). Underneath it upgrades mapnik and all the related dependencies.
+
+## 5.0.1
+Released 2018-01-29
+
+Bug Fixes:
+- Allow aggregation for queries with no the_geom (only the_geom_webmercator) #856
+
+## 5.0.0
+Released 2018-01-29
+
+Backward incompatible changes:
+- Aggregation dataview returns categories with the same type as the database type. For example, if we are aggretating by a numeric field, the resulting JSON will contain a number instead of a stringified number.
+
+## 4.8.0
+Released 2018-01-04
+
+New features:
+ - Return url template in metadata #838.
+
+Bux fixes:
+ - Tests: Order torque objects before comparison
+
+## 4.7.0
+Released 2018-01-03
+
+New features:
+ - Return tilejson in metadata #837.
+
+Bug fixes:
+ - Allow to create vector map-config for layers that doesn't have points. Layers with lines or polygons won't be aggregated by default.
+
+
+## 4.6.0
+Released 2018-01-02
+
+Announcements:
+ - Upgrades windshaft to [4.2.0](https://github.com/CartoDB/windshaft/releases/tag/4.2.0).
+ - Validate aggregation input params.
+ - Fix column names collisions in histograms [#828](https://github.com/CartoDB/Windshaft-cartodb/pull/828).
+ - Add full-sample aggregation support for vector map-config.
+
+## 4.5.0
+Released 2017-12-19
+
+Announcements:
+ - Date histograms: Add second, decade, century and millenium aggregations
+ - Date histograms: Switch the auto threshold from 366 buckets to 100.
+ - Logging all errors.
+ - Add support for aggregated visualizations.
+ - Allow vector-only map-config creation.
+ - Histograms: Now they accept a `no_filters` parameter.
+
+
+## 4.4.0
+Released 2017-12-12
+
+Announcements:
+ - Upgrades camshaft to [0.60.0](https://github.com/CartoDB/camshaft/releases/tag/0.60.0).
+
+
+## 4.3.1
+Released 2017-12-12
+
+Bug fix:
+  - Fixed bug introduced in version 4.0.1 that brokes the static map generation using JPG as format #808
+
+## 4.3.0
+Released 2017-12-11
+
+Announcements:
+- Optimize Formula queries.
+- Optimize Formula queries in overviews.
+- Optimize Numeric Histogram queries.
+- Optimize Date Histogram queries.
+- Date Histograms: Now returns the same value for max/min/avg/timestamp per bin.
+- Date Histograms: Now it should return the same no matter the DB/Client time zone.
+
+## 4.2.0
+Released 2017-12-04
+
+Announcements:
+ - Allow to request MVT tiles without CartoCSS
+ - Upgrades windshaft to [4.1.0](https://github.com/CartoDB/windshaft/releases/tag/4.1.0).
+
+
+## 4.1.1
+Released 2017-11-29
+
+Announcements:
+ - Upgrades turbo-carto to [0.20.2](https://github.com/CartoDB/turbo-carto/releases/tag/0.20.2).
+
+
+## 4.1.0
+Released 2017-mm-dd
+
+Announcements:
+ - Upgrades windshaft to [4.0.1](https://github.com/CartoDB/windshaft/releases/tag/4.0.1).
+ - Add `categories` query param to define the number of categories to be ranked for aggregation dataviews.
+
+
+## 4.0.1
+Released 2017-10-18
+
+Announcements:
+ - Upgrades camshaft to [0.59.4](https://github.com/CartoDB/camshaft/releases/tag/0.59.4).
+ - Upgrades windshaft to [4.0.0](https://github.com/CartoDB/windshaft/releases/tag/4.0.0).
+ - Split and move `req2params` method to multiple middlewares.
+ - Use express error handler middleware to respond in case of something went wrong.
+ - Use `res.locals` object to share info between middlewares and leave `req.params` as an object containing properties mapped to the named route params.
+ - Move `LZMA` decompression to its own middleware.
+ - Implement stats middleware removing some duplicated code while sending response.
+
+
+## 4.0.0
+Released 2017-10-04
+
+Backward incompatible changes:
+ - Removes `list` dataview type.
+
+Announcements:
+ - Upgrades body-parser to 1.18.2.
+ - Upgrades express to 4.16.0.
+ - Upgrades debug to 3.1.0.
+ - Upgrades request to 2.83.0.
+ - Upgrades turbo-carto to [0.20.1](https://github.com/CartoDB/turbo-carto/releases/tag/0.20.1)
+ - Upgrades cartodb-psql to [0.10.2](https://github.com/CartoDB/node-cartodb-psql/releases/tag/0.10.2).
+ - Upgrades camshaft to [0.59.2](https://github.com/CartoDB/camshaft/releases/tag/0.59.2).
+ - Upgrades windshaft to [3.3.3](https://github.com/CartoDB/windshaft/releases/tag/3.3.3).
+ - Upgrades yarn minimum version requirement to v0.27.5
+
+
+## 3.13.0
+Released 2017-10-02
+ - Upgrades camshaft, cartodb-query-tables, and turbo-carto: better support for query variables.
+
+Bugfixes:
+ - Bounding box parameter ignored in static named maps #735.
+ - camhaft 0.59.1 fixes duplicate columns in aggregate-intersection analysis
+
+## 3.12.10
+Released 2017-09-18
+ - Upgrades windshaft to [3.3.2](https://github.com/CartoDB/windshaft/releases/tag/3.3.2).
+
+## 3.12.9
+Released 2017-09-07
+
+Bug fixes:
+- Do not use distinct when calculating quantiles. #743
+
+## 3.12.8
+Released 2017-09-07
+
+Bug fixes:
+- Integer out of range in date histograms. (https://github.com/CartoDB/support/issues/962)
+
+## 3.12.7
+Released 2017-09-01
+
+ - Upgrades camshaft to [0.58.1](https://github.com/CartoDB/camshaft/releases/tag/0.58.1).
+
+
+## 3.12.6
+Released 2017-08-31
+
+ - Upgrades camshaft to [0.58.0](https://github.com/CartoDB/camshaft/releases/tag/0.58.0).
+
+
+## 3.12.5
+Released 2017-08-24
+
+ - Upgrades camshaft to [0.57.0](https://github.com/CartoDB/camshaft/releases/tag/0.57.0).
+
+
+## 3.12.4
+Released 2017-08-23
+
+Announcements:
+ - Upgrades camshaft to [0.56.0](https://github.com/CartoDB/camshaft/releases/tag/0.56.0).
+
+## 3.12.3
+Released 2017-08-22
+
+Announcements:
+ - Upgrades camshaft to [0.55.8](https://github.com/CartoDB/camshaft/releases/tag/0.55.8).
+
 ## 3.12.2
 Released 2017-08-16
 

app.js

@@ -2,14 +2,24 @@ var http = require('http');
 var https = require('https');
 var path = require('path');
 var fs = require('fs');
-
 var _ = require('underscore');
+var semver = require('semver');
+const setICUEnvVariable = require('./lib/cartodb/utils/icu_data_env_setter');
 
 // jshint undef:false
 var log = console.log.bind(console);
 var logError = console.error.bind(console);
 // jshint undef:true
 
+var nodejsVersion = process.versions.node;
+if (!semver.satisfies(nodejsVersion, '>=6.9.0')) {
+    logError(`Node version ${nodejsVersion} is not supported, please use Node.js 6.9 or higher.`);
+    process.exit(1);
+}
+
+// This function should be called before the require('yargs').
+setICUEnvVariable();
+
 var argv = require('yargs')
     .usage('Usage: $0 <environment> [options]')
     .help('h')

config/environments/development.js.example

@@ -55,7 +55,7 @@ var config = {
     ,socket_timeout: 600000
     ,enable_cors: true
     ,cache_enabled: false
-    ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
+    ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler]) (:res[X-Tiler-Errors])'
     // If log_filename is given logs will be written
     // there, in append mode. Otherwise stdout is used (default).
     // Log file will be re-opened on receiving the HUP signal
@@ -107,6 +107,20 @@ var config = {
       // Milliseconds since last access before renderer cache item expires
       cache_ttl: 60000,
       statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
+      mvt: {
+        //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik,
+        //PostGIS 2.4 is required for this to work
+        //If disabled it will use Mapnik MVT generation
+        usePostGIS: false,
+        dbPoolParams: {
+              // maximum number of resources to create at any given time
+              size: 16,
+              // max milliseconds a resource can go unused before it should be destroyed
+              idleTimeout: 3000,
+              // frequency to check for idle resources
+              reapInterval: 1000
+          }
+      },
       mapnik: {
           // The size of the pool of internal mapnik backend
           // This pool size is per mapnik renderer created in Windshaft's RendererFactory
@@ -190,7 +204,11 @@ var config = {
               // which cost is no more expensive than snapping and results are
               // much closer to the original geometry
               removeRepeatedPoints: false // this requires postgis >=2.2
-          }
+          },
+
+          // If enabled Mapnik will reuse the features retrieved from the database
+          // instead of requesting them once per style inside a layer
+          'cache-features': true
 
       },
       http: {

config/environments/production.js.example

@@ -55,7 +55,7 @@ var config = {
     ,socket_timeout: 600000
     ,enable_cors: true
     ,cache_enabled: true
-    ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
+    ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler]) (:res[X-Tiler-Errors])'
     // If log_filename is given logs will be written
     // there, in append mode. Otherwise stdout is used (default).
     // Log file will be re-opened on receiving the HUP signal
@@ -101,6 +101,20 @@ var config = {
       // Milliseconds since last access before renderer cache item expires
       cache_ttl: 60000,
       statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
+      mvt: {
+        //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik,
+        //PostGIS 2.4 is required for this to work
+        //If disabled it will use Mapnik MVT generation
+        usePostGIS: false,
+        dbPoolParams: {
+              // maximum number of resources to create at any given time
+              size: 16,
+              // max milliseconds a resource can go unused before it should be destroyed
+              idleTimeout: 3000,
+              // frequency to check for idle resources
+              reapInterval: 1000
+          }
+      },
       mapnik: {
           // The size of the pool of internal mapnik backend
           // This pool size is per mapnik renderer created in Windshaft's RendererFactory
@@ -184,7 +198,11 @@ var config = {
               // which cost is no more expensive than snapping and results are
               // much closer to the original geometry
               removeRepeatedPoints: false // this requires postgis >=2.2
-          }
+          },
+
+          // If enabled Mapnik will reuse the features retrieved from the database
+          // instead of requesting them once per style inside a layer
+          'cache-features': true
 
       },
       http: {

config/environments/staging.js.example

@@ -55,7 +55,7 @@ var config = {
     ,socket_timeout: 600000
     ,enable_cors: true
     ,cache_enabled: true
-    ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms (:res[X-Tiler-Profiler]) -> :res[Content-Type]'
+    ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms (:res[X-Tiler-Profiler]) -> :res[Content-Type] (:res[X-Tiler-Errors])'
     // If log_filename is given logs will be written
     // there, in append mode. Otherwise stdout is used (default).
     // Log file will be re-opened on receiving the HUP signal
@@ -101,6 +101,20 @@ var config = {
       // Milliseconds since last access before renderer cache item expires
       cache_ttl: 60000,
       statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
+      mvt: {
+        //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik,
+        //PostGIS 2.4 is required for this to work
+        //If disabled it will use Mapnik MVT generation
+        usePostGIS: false,
+        dbPoolParams: {
+              // maximum number of resources to create at any given time
+              size: 16,
+              // max milliseconds a resource can go unused before it should be destroyed
+              idleTimeout: 3000,
+              // frequency to check for idle resources
+              reapInterval: 1000
+          }
+      },
       mapnik: {
           // The size of the pool of internal mapnik backend
           // This pool size is per mapnik renderer created in Windshaft's RendererFactory
@@ -184,7 +198,11 @@ var config = {
               // which cost is no more expensive than snapping and results are
               // much closer to the original geometry
               removeRepeatedPoints: false // this requires postgis >=2.2
-          }
+          },
+
+          // If enabled Mapnik will reuse the features retrieved from the database
+          // instead of requesting them once per style inside a layer
+          'cache-features': true
 
       },
       http: {

config/environments/test.js.example

@@ -54,7 +54,7 @@ var config = {
     ,socket_timeout: 600000
     ,enable_cors: true
     ,cache_enabled: false
-    ,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
+    ,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler]) (:res[X-Tiler-Errors])'
     // If log_filename is given logs will be written
     // there, in append mode. Otherwise stdout is used (default).
     // Log file will be re-opened on receiving the HUP signal
@@ -100,6 +100,20 @@ var config = {
       // Milliseconds since last access before renderer cache item expires
       cache_ttl: 60000,
       statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
+      mvt: {
+        //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik,
+        //PostGIS 2.4 is required for this to work
+        //If disabled it will use Mapnik MVT generation
+        usePostGIS: false,
+        dbPoolParams: {
+              // maximum number of resources to create at any given time
+              size: 16,
+              // max milliseconds a resource can go unused before it should be destroyed
+              idleTimeout: 3000,
+              // frequency to check for idle resources
+              reapInterval: 1000
+          }
+      },
       mapnik: {
           // The size of the pool of internal mapnik backend
           // This pool size is per mapnik renderer created in Windshaft's RendererFactory
@@ -183,7 +197,12 @@ var config = {
               // which cost is no more expensive than snapping and results are
               // much closer to the original geometry
               removeRepeatedPoints: false // this requires postgis >=2.2
-          }
+          },
+
+          // If enabled Mapnik will reuse the features retrieved from the database
+          // instead of requesting them once per style inside a layer
+          'cache-features': true
+
       },
       http: {
           timeout: 2000, // the timeout in ms for a http tile request

docker-test.sh

@@ -0,0 +1,11 @@
+export NPROCS=1 && export JOBS=1 && export CXX=g++-4.9 && export PGUSER=postgres
+
+npm install -g yarn@0.27.5
+yarn
+
+ /etc/init.d/postgresql start
+
+createdb template_postgis && createuser publicuser
+psql -c "CREATE EXTENSION postgis" template_postgis
+
+POSTGIS_VERSION=2.4 npm test

docs/Map-API.md

@@ -17,3 +17,4 @@ You can create two types of maps with the Maps API:
 * [Anonymous Maps](anonymous_maps.md)
 * [Named Maps](named_maps.md)
 * [Static Maps API](static_maps_api.md)
+* [MapConfig File Format]([local file in the docs repo](https://github.com/CartoDB/docs/blob/master/_app/_mapsapi/06-mapconfig.md))

docs/MapConfig-Aggregation-extension.md

@@ -0,0 +1,62 @@
+# 1. Purpose
+
+This specification describes an extension for
+[MapConfig 1.7.0](https://github.com/CartoDB/Windshaft/blob/master/doc/MapConfig-1.7.0.md) version.
+
+
+# 2. Changes over specification
+
+This extension introduces a new layer options for aggregated data tile generation.
+
+## 2.1 Aggregation options
+
+The layer options attribute is extended with a new optional `aggregation` attribute.
+The value of this attribute can be `false` to explicitly disable aggregation for the layer.
+
+```javascript
+{
+    aggregation: {
+
+        // OPTIONAL
+        // string, defines the placement of aggregated geometries. Can be one of:
+        // * "point-sample", the default places geometries at a sample point (one of the aggregated geometries)
+        // * "point-grid" places geometries at the center of the aggregation grid cells
+        // * "centroid" places geometriea at the average position of the aggregated points
+        // See https://github.com/CartoDB/Windshaft-cartodb/blob/master/docs/aggregation.md#placement for more details
+        placement: "point-sample",
+
+        // OPTIONAL
+        // object, defines the columns of the aggregated datasets. Each property corresponds to a columns name and
+        // should contain an object with two properties: "aggregate_function" (one of "sum", "max", "min", "avg", "mode" or "count"),
+        // and "aggregated_column" (the name of a column of the original layer query or "*")
+        // A column defined as `"_cdb_features_count": {"aggregate_function": "count", aggregated_column: "*"}`
+        // is always generated in addition to the defined columns.
+        // The column names `cartodb_id`, `the_geom`, `the_geom_webmercator` and `_cdb_feature_count` cannot be used
+        // for aggregated columns, as they correspond to columns always present in the result.
+        // See https://github.com/CartoDB/Windshaft-cartodb/blob/master/docs/aggregation.md#columns for more details
+        columns: {
+            "aggregated_column_1": {
+                "aggregate_function": "sum",
+                "aggregated_column": "original_column_1"
+            }
+        },
+
+        // OPTIONAL
+        // Number, defines the cell-size of the spatial aggregation grid as a pixel resolution power of two (1/4, 1/2,... 2, 4, 16)
+        // to scale from 256x256 pixels; the default is 1 corresponding to 256x256 cells per tile.
+        // See https://github.com/CartoDB/Windshaft-cartodb/blob/master/docs/aggregation.md#resolution for more details
+        resolution: 1,
+
+        // OPTIONAL
+        // Number, the minimum number of (estimated) rows in the dataset (query results) for aggregation to be applied.
+        // See https://github.com/CartoDB/Windshaft-cartodb/blob/master/docs/aggregation.md#threshold for more details
+        threshold: 500000
+    }
+}
+```
+
+# History
+
+## 1.0.0
+
+ - Initial version

docs/MapConfig-Dataviews-extension.md

@@ -8,50 +8,13 @@ This specification describes an extension for
 
 This extension depends on Analyses extension. It extends MapConfig with a new attribute: `dataviews`.
 
-It makes possible to get tabular data from analysis nodes: lists, aggregated lists, aggregations, and histograms.
+It makes possible to get tabular data from analysis nodes: aggregated lists, aggregations, and histograms.
 
 ## 2.1. Dataview types
 
-### List
-
-A list is a simple result set per row where is possible to retrieve several columns from the original layer query.
-
-Definition
-```
-{
-    // REQUIRED
-    // string, `type` the list type
-    “type”: “list”,
-    // REQUIRED
-    // object, `options` dataview params
-    “options”: {
-        // REQUIRED
-        // array, `columns` to select for the list
-        “columns”: [“name”, “description”]
-    }
-}
-```
-
-Expected output
-```
-{
-  "type": "list",
-  "rows": [
-    {
-      "{columnName1}": "val1",
-      "{columnName2}": 100
-    },
-    {
-      "{columnName1}": "val2",
-      "{columnName2}": 200
-    }
-  ]
-}
-```
-
 ### Aggregation
 
-An aggregation is very similar to a list but results are aggregated by a column and a given aggregation function.
+An aggregation is a list with aggregated results by a column and a given aggregation function.
 
 Definition
 ```

docs/aggregation.md

@@ -0,0 +1,187 @@
+# Tile Aggregation
+
+To be able to represent a large amount of data (say, hundred of thousands to millions of points) in a tile. This can be useful both for raster tiles (where the aggregation reduces the number of features to be rendered) and vector tiles (the tile contais less features).
+
+Aggregation is available only for point geometries. During aggregation the points are grouped using a grid; all the points laying in the same cell of the grid are summarized in a single aggregated result point.
+ - The position of the aggregated point is controlled by the `placement` parameter.
+ - The aggregated rows always contain at least a column, named `_cdb_feature_count`, which contains the number of the original points that the aggregated point represents.
+
+### Special default aggregation
+
+When no placement or columns are specified a special default aggregation is performed.
+
+This special mode performs only spatial aggregation (using a grid defined by the requested tile and the resolution, parameter, as all the other cases), and returns a _random_ record from each group (grid cell) with all its columns and an additional `_cdb_features_count` with the number of features in the group.
+
+Regarding the randomness of the sample: currently we use the row with the minimum `cartodb_id` value in each group.
+
+The rationale behind having this special aggregation with all the original columns is to provide a mostly transparent way to handle large datasets without having to provide special map configurations for those cases (i.e. preserving the logic used to produce the maps with smaller datasets). [Overviews have been used so far with this intent](https://carto.com/docs/tips-and-tricks/back-end-data-performance/), but they are inflexible.
+
+### User defined aggregations
+
+When either a explicit placement or columns are requested we no longer use the special, query; we use one determined by the placement (which will default to "centroid"), and it will have as columns only the aggregated columns specified, in addition to `_cdb_features_count`, which is always present.
+
+We might decide in the future to allow sampling column values for any of the different placement modes.
+
+### Behaviour for raster and vector tiles
+
+The vector tiles from a vector-only map will be aggregated by default.
+However, Raster tiles (or vector tiles from a map which defines CartoCSS styles) will be aggregated only upon request.
+
+Aggregation that would otherwise occur can be disabled by passing an `aggregation=false` parameter to the map instantiation HTTP call.
+
+To control how aggregation is performed, an aggregation option can be added to the layer:
+
+```json
+{
+    "layers": [
+        {
+            "options": {
+                "sql": "SELECT * FROM data",
+                "aggregation": {
+                    "placement": "centroid",
+                    "columns": {
+                        "value": {
+                            "aggregate_function": "sum",
+                            "aggregated_column": "value"
+                        }
+                    }
+                }
+            }
+        }
+    ]
+}
+```
+
+Even if aggregation is explicitly requested it may not be activated, e.g., if the geometries are not points
+or the whole dataset is too small. The map instantiation response contains metadata that informs if any particular
+layer will be aggregated when tiles are requested, both for vector (mvt) and raster (png) tiles.
+
+```json
+{
+  "layergroupid": "7b97b6e76590fef889b63edd2efb1c79:1513608333045",
+  "metadata": {
+    "layers": [
+      {
+        "type": "mapnik",
+        "id": "layer0",
+        "meta": {
+          "stats": {
+            "estimatedFeatureCount": 6232136
+          },
+          "aggregation": {
+            "png": true,
+            "mvt": true
+          }
+        }
+      }
+    ]
+  }
+}
+```
+
+## Aggregation parameters
+
+The aggregation parameters for a layer are defined inside an `aggregation` option of the layer:
+
+```json
+{
+    "layers": [
+        {
+            "options": {
+                "sql": "SELECT * FROM data",
+                "aggregation": {"...": "..."}
+            }
+        }
+    ]
+}
+```
+
+### `placement`
+
+Determines the kind of aggregated geometry generated:
+
+#### `point-sample`
+
+This is the default placement. It will place the aggregated point at a random sample of the grouped points,
+like the default aggregation does. No other attribute is sampled, though, the point will contain the aggregated attributes determined by the `columns` parameter.
+
+#### `point-grid`
+
+Generates points at the center of the aggregation grid cells (squares).
+
+#### `centroid`
+
+Generates points with the averaged coordinated of the grouped points (i.e. the points inside each grid cell).
+
+### `columns`
+
+The aggregated attributes defined by `columns` are computed by a applying an _aggregate function_ to all the points in each group.
+Valid aggregate functions are `sum`, `avg` (average), `min` (minimum), `max` (maximum) and `mode` (the most frequent value in the group).
+The values to be aggregated are defined by the _aggregated column_ of the source data. The column keys define the name of the resulting column in the aggregated dataset.
+
+For example here we define three aggregate attributes named `total`, `max_price` and `price` which are all computed with the same column, `price`,
+of the original dataset applying three different aggregate functions.
+
+```json
+{
+    "columns": {
+        "total": { "aggregate_function": "sum", "aggregated_column": "price" },
+        "max_price": { "aggregate_function": "max", "aggregated_column": "price" },
+        "price": { "aggregate_function": "avg", "aggregated_column": "price" }
+    }
+}
+```
+
+> Note that you can use the original column names as names of the result, but all the result column names must be unique.  In particular, the names `cartodb_id`, `the_geom`, `the_geom_webmercator` and `_cdb_feature_count` cannot be used for aggregated columns, as they correspond to columns always present in the result.
+
+### `resolution`
+
+Defines the cell-size of the spatial aggregation grid. This is equivalent to the [CartoCSS `-torque-resolution`](https://carto.com/docs/carto-engine/cartocss/properties-for-torque/#-torque-resolution-float) property of Torque maps.
+
+The aggregation cells are `resolution`×`resolution` pixels in size, where pixels here are defined to be 1/256 of the (linear) size of a tile.
+The default value is 1, so that aggregation coincides with raster pixels. A value of 2 would make each cell to be 4 (2×2) pixels, and a value of
+0.5 would yield 4 cells per pixel. In teneral values less than 1 produce sub-pixel precision.
+
+> Note that is independent of the number of pixels for raster tile or the coordinate resolution (mvt_extent) of vector tiles.
+
+
+### `threshold`
+
+This is the minimum number of (estimated) rows in the dataset (query results) for aggregation to be applied. If the number of rows estimate is less than the threshold aggregation will be disabled for the layer; the instantiation response will reflect that and tiles will be generated without aggregation.
+
+### Example
+
+```json
+{
+    "version": "1.7.0",
+    "extent": [-20037508.5, -20037508.5, 20037508.5, 20037508.5],
+    "srid": 3857,
+    "maxzoom": 18,
+    "minzoom": 3,
+    "layers": [
+        {
+            "type": "mapnik",
+            "options": {
+                "sql": "select * from table",
+                "cartocss": "#table { marker-width: [total]; marker-fill: ramp(value, (red, green, blue), jenks); }",
+                "cartocss_version": "2.3.0",
+                "aggregation": {
+                    "placement": "centroid",
+                    "columns": {
+                        "value": {
+                            "aggregate_function": "avg",
+                            "aggregated_column": "value"
+                        },
+                        "total": {
+                            "aggregate_function": "sum",
+                            "aggregated_column": "value"
+                        }
+                    },
+                    "resolution": 2,
+                    "threshold": 500000
+                }
+            }
+        }
+    ]
+}
+```

docs/anonymous_maps.md

@@ -1,6 +1,7 @@
 # Anonymous Maps
 
-Anonymous Maps allows you to instantiate a map given SQL and CartoCSS. It also allows you to add interaction capabilities using [UTF Grid.](https://github.com/mapbox/utfgrid-spec)
+Anonymous Maps allows you to instantiate a map given SQL and CartoCSS. It also allows you to add interaction capabilities using [UTF Grid.](https://github.com/mapbox/utfgrid-spec).
+Alternatively, you can get the data for the map (geometry and attributes for each layer) using vector tiles (in which case CartoCSS is not required).
 
 
 ## Instantiate
@@ -41,6 +42,13 @@ updated_at | The ISO date of the last time the data involved in the query was up
 metadata | Includes information about the layers.
 cdn_url | URLs to fetch the data using the best CDN for your zone.
 
+**Improved response metadata**
+
+Originally, you needed to concantenate the `layergroupid` with the correct domain and the path for the tiles.
+Now, for convenience, the layergroup includes the final URLs in two formats:
+1. Leaflet's urlTemplate alike: useful when working with raster tiles or with libraries with an API similar to Leaflet's one.
+1. [TileJSON spec](https://github.com/mapbox/tilejson-spec): useful when working with Mapbox GL or any other library that supports TileJSON.
+
 ### Example
 
 #### Call
@@ -61,30 +69,231 @@ curl 'https://{username}.carto.com/api/v1/map' -H 'Content-Type: application/jso
         "type": "mapnik",
         "meta": {}
       }
-    ]
+    ],
+    "tilejson": {
+        "raster": {
+            "tilejson": "2.2.0",
+            "tiles": [
+                "http://a.cdb.com/c01a54877c62831bb51720263f91fb33/{z}/{x}/{y}.png",
+                "http://b.cdb.com/c01a54877c62831bb51720263f91fb33/{z}/{x}/{y}.png"
+            ]
+        }
+    },
+    "url": {
+        "raster": {
+            "urlTemplate": "http://{s}.cdb.com/c01a54877c62831bb51720263f91fb33/{z}/{x}/{y}.png",
+            "subdomains": ["a", "b"]
+        }
+    }
   },
   "cdn_url": {
     "http": "http://cdb.com",
-    "https": "https://cdb.com"
+    "https": "https://cdb.com",
+    "templates": {
+         "http": { "subdomains": ["a","b"], "url": "http://{s}.cdb.com" },
+         "https": { "subdomains": ["a","b"], "url": "https://{s}.example.com" },
+    }
   }
 }
 ```
 
-### Retrieve resources from the layergroup
+## Map Tile Rendering
+
+Map tiles are used to create the graphic representation of your map in a web browser. Tiles can be requested either as pre-rendered *raster* tiles (images) or as *vector* map data to be rendered by the client (browser).
+
+- **Raster**: If a tile is requested as a raster image format, like PNG, the map will be rendered on the server, using the CartoCSS styles defined in the layers of the map. It is necessary that all the layers of a map define CartoCSS styles in order to obtain raster tiles. Raster tiles are made up of 256x256 pixels; to avoid graphic quality issues tiles should be used unscaled to represent the zoom level (Z) for which they are requested. In order to render tiles, data will be retrieved from the database (in vector format) on the server-side.
+
+- **Vector**: Tiles can also be requested as MVT (Mapbox Vector Tiles). In this case, only the geospatial vector data, without any styling, is returned. These tiles should be processed in the client-side to render the map.  In this case layers do not need to define CartoCSS, as any rendering and styling will be performed on the client side. The vector data of a tile represents real-world geometries by defining the vertices of points, lines or polygons in a tile-specific coordinate system.
+
+## Retrieve resources from the layergroup
 
 When you have a layergroup, there are several resources for retrieving layergoup details such as, accessing Mapnik tiles, getting individual layers, accessing defined Attributes, and blending and layer selection.
 
-#### Mapnik tiles
+### Raster tiles
 
-These tiles will get just the Mapnik layers. To get individual layers, see the following section.
+These raster tiles are PNG images that represent only the Mapnik layers of a map. See [individual layers](#individual-layers) for details about how to retrieve other layers.
 
 ```bash
 https://{username}.carto.com/api/v1/map/{layergroupid}/{z}/{x}/{y}.png
 ```
 
-#### Individual layers
+### Mapbox Vector Tiles (MVT)
 
-The MapConfig specification holds the layers definition in a 0-based index. Layers can be requested individually in different formats depending on the layer type.
+[Mapbox Vector Tiles (MVT)](https://www.mapbox.com/vector-tiles/specification/) are map tiles that transfer geographic vector data to the client-side. Browser performance is fast since you can pan and zoom without having to query the server.
+
+CARTO uses Web Graphics Library (WebGL) to process MVT files on the browser. This is useful since WebGL is compatible with most web browsers, include support for multiple client-side mapping engines, and do not require additional information from the server; which makes it more efficient for rendering map tiles. However, you can use any implementation tool for processing MVT files.
+
+The following examples describe how to fetch MVT tiles with a cURL request.
+
+#### MVT and Windshaft
+
+CARTO uses Windshaft as the map tiler library to render multilayer maps with the Maps API. You can use Windshaft to request MVT using the same layer type that is used for requesting raster tiles (Mapnik layer). Simply change the file format `.mvt` in the URL.
+
+
+```bash
+https://{username}.cartodb.com/api/v1/map/HASH/:layer/{z}/{x}/{y}.mvt
+```
+
+The following example instantiates an anonymous map with layer options:
+
+```bash
+{
+  user_name: 'mycartodbuser',
+  sublayers: [{
+    sql: "SELECT * FROM table_name";
+    cartocss: '#layer { marker-fill: #F0F0F0; }'
+  }],
+  maps_api_template: 'https://{user}.cartodb.com' // Optional
+}
+```
+
+**Note**: If no layer type is specified, Mapnik tiles are used by default. To access MVT tiles, specify `https://{username}.cartodb.com/api/v1/map/HASH/{z}/{x}/{y}.mvt` as the `maps_api_template` variable.
+
+**Tip:** If you are using [Named Maps](https://carto.com/docs/carto-engine/maps-api/named-maps/) to instantiate a layer, indicate the MVT file format and layer in the response:
+
+```bash
+https://{username}.cartodb.com/api/v1/map/named/:templateId/:layer/{z}/{x}/{y}.mvt
+```
+
+For all layers in a Named Map, you must indicate Mapnik as the layer filter:
+
+```bash
+https://{username}.cartodb.com/api/v1/map/named/:templateId/mapnik/{z}/{x}/{y}.mvt
+```
+
+#### Layergroup Filter for MVT Tiles
+
+To filter layers using Windshaft, use the following request where layers are numbered:
+
+```bash
+https://{username}.cartodb.com/api/v1/map/HASH/0,1,2/{z}/{x}/{y}.mvt
+```
+
+To request all layers, remove the layergroup filter parameter:
+
+```bash
+https://{username}.cartodb.com/api/v1/map/HASH/{z}/{x}/{y}.mvt
+```
+
+To filter a specific layer:
+
+```bash
+https://{username}.cartodb.com/api/v1/map/HASH/2/{z}/{x}/{y}.mvt
+```
+
+#### Example 1: MVT Tiles with Windshaft, CARTO.js, and MapboxGL
+
+1) Import the required libraries:
+
+```bash
+<script src='https://api.tiles.mapbox.com/mapbox-gl-js/v0.9.0/mapbox-gl.js'></script>
+<link href='https://api.tiles.mapbox.com/mapbox-gl-js/v0.9.0/mapbox-gl.css' rel='stylesheet' />
+<script src="http://libs.cartocdn.com/cartodb.js/v3/3.15/cartodb.core.js"></script>
+```
+
+2) Configure Map Client:
+
+```bash
+mapboxgl.accessToken = '{yourMapboxToken}';
+```
+
+3) Create Map Object (Mapbox):
+
+```bash
+var map = new mapboxgl.Map({
+container: 'map',
+zoom: 1,
+minZoom: 0,
+maxZoom: 18,
+center: [30, 0]
+});
+```
+
+4) Define Layer Options (CARTO):
+
+```bash
+var layerOptions = {
+user_name: "{username}",
+sublayers: [{
+sql: "SELECT * FROM {table_name}",
+cartocss: "...",
+     }]
+};
+```
+
+5) Request Tiles (from CARTO) and Set to Map Object (Mapbox):
+
+**Note:** By default, [CARTO core functions](https://carto.com/docs/carto-engine/carto-js/core-api/) retrieve URLs for fully rendered tiles. You must replace the default format (.png) with the MVT format (.mvt).
+
+
+```bash
+cartodb.Tiles.getTiles(layerOptions, function(result, err) {
+var tiles = result.tiles.map(function(tileUrl) {
+return tileUrl
+.replace('{s}', 'a')
+.replace(/\.png/, '.mvt');
+});
+map.setStyle(simpleStyle(tiles));
+});
+```
+
+#### Example 2: MVT Libraries with Windshaft and MapboxGL
+
+When you are not including CARTO.js to implement MVT tiles, you must use the `map.setStyle` parameter to specify vector map rendering.
+
+1) Import the required libraries:
+
+```bash
+<script src='https://api.tiles.mapbox.com/mapbox-gl-js/v0.9.0/mapbox-gl.js'></script>
+<link href='https://api.tiles.mapbox.com/mapbox-gl-js/v0.9.0/mapbox-gl.css' rel='stylesheet'/>
+```
+
+2) Configure Map Client:
+
+```bash
+mapboxgl.accessToken = '{yourMapboxToken}';
+```
+
+3) Create Map Object (Mapbox):
+
+```bash
+var map = new mapboxgl.Map({
+container: 'map',
+zoom: 1,
+minZoom: 0,
+maxZoom: 18,
+center: [30, 0]
+});
+```
+
+4) Set the Style
+
+```bash
+map.setStyle({
+    "version": 7,
+    "glyphs": "...",
+    "constants": {...},
+    "sources": {
+      "cartodb": {
+        "type": "vector",
+        "tiles": [ "http://{username}.cartodb.com/api/v1/map/named/templateId/mapnik/{z}/{x}/{y}.mvt"
+        ],
+        "maxzoom": 18
+      }
+    },
+    "layers": [{...}]
+});
+```
+
+**Tip:** If you are using MapboxGL, see the following resource for additional information.
+
+- [MapboxGL API Reference](https://www.mapbox.com/mapbox-gl-js/api/)
+- [MapboxGL Style Specifications](https://www.mapbox.com/mapbox-gl-js/style-spec/)
+- [Example of MapboxGL Implementation](https://www.mapbox.com/mapbox-gl-js/examples/)
+
+### Individual layers
+
+The MapConfig specification holds the layers definition in a 0-based index. Layers can be requested individually, in different formats, depending on the layer type.
 
 Individual layers can be accessed using that 0-based index. For UTF grid tiles:
 
@@ -100,19 +309,19 @@ If the MapConfig had a Torque layer at index 1 it could be possible to request i
 https://{username}.carto.com/api/v1/map/{layergroupid}/1/{z}/{x}/{y}.torque.json
 ```
 
-#### Attributes defined in `attributes` section
+### Attributes defined in `attributes` section
 
 ```bash
 https://{username}.carto.com/api/v1/map/{layergroupid}/{layer}/attributes/{feature_id}
 ```
 
-Which returns JSON with the attributes defined, like:
+Which returns JSON with the attributes defined, such as:
 
 ```javascript
 { "c": 1, "d": 2 }
 ```
 
-#### Blending and layer selection
+### Blending and layer selection
 
 ```bash
 https://{username}.carto.com/api/v1/map/{layergroupid}/{layer_filter}/{z}/{x}/{y}.png
@@ -141,10 +350,7 @@ https://{username}.carto.com/api/v1/map/{layergroupid}/0,3,4/{z}/{x}/{y}.png
 Some notes about filtering:
 
   - Invalid index values or out of bounds indexes will end in `Invalid layer filtering` errors.
-  - Ordering is not considered. So right now filtering layers 0,3,4 is the very same thing as filtering 3,4,0. As this
-  may change in the future **it is recommended** to always select the layers in ascending order so you will get a
-  consistent behavior in the future.
-
+  - Ordering is not considered. So right now filtering layers 0,3,4 is the very same thing as filtering 3,4,0. As this may change in the future, **it is recommended** to always select the layers in ascending order so that you will always get consistent behavior.
 
 ## Create JSONP
 
@@ -185,7 +391,6 @@ callback({
 })
 ```
 
-
 ## Remove
 
 Anonymous Maps cannot be removed by an API call. They will expire after about five minutes, or sometimes longer. If an Anonymous Map expires and tiles are requested from it, an error will be raised. This could happen if a user leaves a map open and after time, returns to the map and attempts to interact with it in a way that requires new tiles (e.g. zoom). The client will need to go through the steps of creating the map again to fix the problem.

docs/general_concepts.md

@@ -22,6 +22,6 @@ Errors are reported using standard HTTP codes and extended information encoded i
 
 If you use JSONP, the 200 HTTP code is always returned so the JavaScript client can receive errors from the JSON object.
 
-## CORS support
+## CORS Support
 
 All the endpoints, which might be accessed using a web browser, add CORS headers and allow OPTIONS method.

docs/static_maps_api.md

@@ -152,7 +152,8 @@ It is important to note that generated images are cached from the live data refe
 * Timeout limits for generating static maps are the same across CARTO Builder and CARTO Engine. It is important to ensure timely processing of queries.
 * If you are publishing your map as a static image with the API, you must manually add [attributions](https://carto.com/attribution) for your static map image. For example, add the following attribution code:
 
-{% highlight javascript %}attribution: '&copy; <a href="http://www.openstreetmap.org/copyright">OpenStreetMap</a> contributors, &copy; <a href="https://carto.com/attributions">CARTO</a>
+{% highlight javascript %}
+attribution: '&copy; <a href="http://www.openstreetmap.org/copyright">OpenStreetMap</a> contributors, &copy; <a href="https://carto.com/attributions">CARTO</a>
 {% endhighlight %}
 
 ## Examples

lib/cartodb/api/auth_api.js

@@ -19,22 +19,22 @@ function AuthApi(pgConnection, metadataBackend, mapStore, templateMaps) {
 
 module.exports = AuthApi;
 
-// Check if a request is authorized by a signer
+// Check if the user is authorized by a signer
 //
-// @param req express request object
+// @param res express response object
 // @param callback function(err, signed_by) signed_by will be
 //                 null if the request is not signed by anyone
 //                 or will be a string cartodb username otherwise.
 //
-AuthApi.prototype.authorizedBySigner = function(req, callback) {
-    if ( ! req.params.token || ! req.params.signer ) {
+AuthApi.prototype.authorizedBySigner = function(res, callback) {
+    if ( ! res.locals.token || ! res.locals.signer ) {
         return callback(null, false); // no signer requested
     }
 
     var self = this;
 
-    var layergroup_id = req.params.token;
-    var auth_token = req.params.auth_token;
+    var layergroup_id = res.locals.token;
+    var auth_token = res.locals.auth_token;
 
     this.mapStore.load(layergroup_id, function(err, mapConfig) {
         if (err) {
@@ -84,11 +84,12 @@ AuthApi.prototype.authorizedByAPIKey = function(user, req, callback) {
  * Check access authorization
  *
  * @param req - standard req object. Importantly contains table and host information
+ * @param res - standard res object. Contains the auth parameters in locals
  * @param callback function(err, allowed) is access allowed not?
  */
-AuthApi.prototype.authorize = function(req, callback) {
+AuthApi.prototype.authorize = function(req, res, callback) {
     var self = this;
-    var user = req.context.user;
+    var user = res.locals.user;
 
     step(
         function () {
@@ -101,11 +102,11 @@ AuthApi.prototype.authorize = function(req, callback) {
             // if not authorized by api_key, continue
             if (!authorized)  {
                 // not authorized by api_key, check if authorized by signer
-                return self.authorizedBySigner(req, this);
+                return self.authorizedBySigner(res, this);
             }
 
             // authorized by api key, login as the given username and stop
-            self.pgConnection.setDBAuth(user, req.params, function(err) {
+            self.pgConnection.setDBAuth(user, res.locals, function(err) {
                 callback(err, true); // authorized (or error)
             });
         },
@@ -120,7 +121,7 @@ AuthApi.prototype.authorize = function(req, callback) {
                 // if no signer name was given, let dbparams and
                 // PostgreSQL do the rest.
                 //
-                if ( ! req.params.signer ) {
+                if ( ! res.locals.signer ) {
                     return callback(null, true); // authorized so far
                 }
 
@@ -128,7 +129,7 @@ AuthApi.prototype.authorize = function(req, callback) {
                 return callback(null, false);
             }
 
-            self.pgConnection.setDBAuth(user, req.params, function(err) {
+            self.pgConnection.setDBAuth(user, res.locals, function(err) {
                 req.profiler.done('setDBAuth');
                 callback(err, true); // authorized (or error)
             });

lib/cartodb/backends/dataview.js

@@ -37,12 +37,19 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param
                 throw new Error("Dataview '" + dataviewName + "' does not exists");
             }
 
-            var pg = new PSQL(dbParamsFromReqParams(params));
-
             var ownFilter = +params.own_filter;
-            ownFilter = !!ownFilter;
+            var noFilters = +params.no_filters;
+            if (Number.isFinite(ownFilter) && Number.isFinite(noFilters)) {
+                err = new Error();
+                err.message = 'Both own_filter and no_filters cannot be sent in the same request';
+                err.type = 'dataview';
+                err.http_status = 400;
+                return callback(err);
+            }
 
-            var query = (ownFilter) ? dataviewDefinition.sql.own_filter_on : dataviewDefinition.sql.own_filter_off;
+            var pg = new PSQL(dbParamsFromReqParams(params));
+            
+            var query = getDataviewQuery(dataviewDefinition, ownFilter, noFilters);
             if (params.bbox) {
                 var bboxFilter = new BBoxFilter({column: 'the_geom_webmercator', srid: 3857}, {bbox: params.bbox});
                 query = bboxFilter.sql(query);
@@ -55,7 +62,7 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param
             );
 
             var dataview = dataviewFactory.getDataview(query, dataviewDefinition);
-            dataview.getResult(pg, getOverrideParams(params, ownFilter), this);
+            dataview.getResult(pg, getOverrideParams(params, !!ownFilter), this);
         },
         function returnCallback(err, result) {
             return callback(err, result);
@@ -63,6 +70,16 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param
     );
 };
 
+function getDataviewQuery(dataviewDefinition, ownFilter, noFilters) {
+    if (noFilters) {
+        return dataviewDefinition.sql.no_filters;
+    } else if (ownFilter === 1) {
+        return dataviewDefinition.sql.own_filter_on;
+    } else {
+        return dataviewDefinition.sql.own_filter_off;
+    }
+}
+
 function getQueryRewriteData(mapConfig, dataviewDefinition, params) {
     var sourceId = dataviewDefinition.source.id; // node.id
     var layer = _.find(mapConfig.obj().layers, function(l) {
@@ -94,7 +111,7 @@ function getQueryRewriteData(mapConfig, dataviewDefinition, params) {
 }
 
 function getOverrideParams(params, ownFilter) {
-    var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins', 'offset'),
+    var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins', 'offset', 'categories'),
         function castNumbers(overrides, val, k) {
             if (!Number.isFinite(+val)) {
                 throw new Error('Invalid number format for parameter \'' + k + '\'');
@@ -113,9 +130,7 @@ function getOverrideParams(params, ownFilter) {
     return overrideParams;
 }
 
-DataviewBackend.prototype.search = function (mapConfigProvider, user, params, callback) {
-    var dataviewName = params.dataviewName;
-
+DataviewBackend.prototype.search = function (mapConfigProvider, user, dataviewName, params, callback) {
     step(
         function getMapConfig() {
             mapConfigProvider.getMapConfig(this);

lib/cartodb/backends/turbo-carto-postgres-datasource.js

@@ -21,10 +21,10 @@ function createTemplate(method) {
 }
 
 var methods = {
-    quantiles: 'CDB_QuantileBins(array_agg(distinct({{=it._column}}::numeric)), {{=it._buckets}}) as quantiles',
+    quantiles: 'CDB_QuantileBins(array_agg({{=it._column}}::numeric), {{=it._buckets}}) as quantiles',
     equal: 'CDB_EqualIntervalBins(array_agg({{=it._column}}::numeric), {{=it._buckets}}) as equal',
-    jenks: 'CDB_JenksBins(array_agg(distinct({{=it._column}}::numeric)), {{=it._buckets}}) as jenks',
-    headtails: 'CDB_HeadsTailsBins(array_agg(distinct({{=it._column}}::numeric)), {{=it._buckets}}) as headtails'
+    jenks: 'CDB_JenksBins(array_agg({{=it._column}}::numeric), {{=it._buckets}}) as jenks',
+    headtails: 'CDB_HeadsTailsBins(array_agg({{=it._column}}::numeric), {{=it._buckets}}) as headtails'
 };
 
 var methodTemplates = Object.keys(methods).reduce(function(methodTemplates, methodName) {

lib/cartodb/controllers/analyses.js

@@ -1,152 +1,150 @@
-var step = require('step');
-var assert = require('assert');
-var dot = require('dot');
-dot.templateSettings.strip = false;
 var PSQL = require('cartodb-psql');
-
-var util = require('util');
-var BaseController = require('./base');
-
 var cors = require('../middleware/cors');
 var userMiddleware = require('../middleware/user');
 
-function AnalysesController(authApi, pgConnection) {
-    BaseController.call(this, authApi, pgConnection);
+function AnalysesController(prepareContext) {
+    this.prepareContext = prepareContext;
 }
 
-util.inherits(AnalysesController, BaseController);
-
 module.exports = AnalysesController;
 
-AnalysesController.prototype.register = function(app) {
-    app.get(app.base_url_mapconfig + '/analyses/catalog', cors(), userMiddleware, this.catalog.bind(this));
-};
-
-AnalysesController.prototype.sendResponse = function(req, res, resource) {
-    res.set('Cache-Control', 'public,max-age=10,must-revalidate');
-    this.send(req, res, resource, 200);
-};
-
-AnalysesController.prototype.catalog = function(req, res) {
-    var self = this;
-    var username = req.context.user;
-
-    step(
-        function reqParams() {
-            self.req2params(req, this);
-        },
-        function catalogQuery(err) {
-            assert.ifError(err);
-            var pg = new PSQL(dbParamsFromReqParams(req.params));
-            getMetadata(username, pg, this);
-        },
-        function prepareResponse(err, results) {
-            assert.ifError(err);
-
-            var analysisIdToTable = results.tables.reduce(function(analysisIdToTable, table) {
-                var analysisId = table.relname.split('_')[2];
-                if (analysisId && analysisId.length === 40) {
-                    analysisIdToTable[analysisId] = table;
-                }
-                return analysisIdToTable;
-            }, {});
-
-            var catalogWithTables = results.catalog.map(function(analysis) {
-                if (analysisIdToTable.hasOwnProperty(analysis.node_id)) {
-                    analysis.table = analysisIdToTable[analysis.node_id];
-                }
-                return analysis;
-            });
-
-            return catalogWithTables.sort(function(analysisA, analysisB) {
-                if (!!analysisA.table && !!analysisB.table) {
-                    return analysisB.table.size - analysisA.table.size;
-                }
-
-                if (!!analysisA.table) {
-                    return -1;
-                }
-
-                if (!!analysisB.table) {
-                    return 1;
-                }
-
-                return -1;
-            });
-        },
-        function sendResponse(err, catalogWithTables) {
-            if (err) {
-                if (err.message.match(/permission\sdenied/)) {
-                    err = new Error('Unauthorized');
-                    err.http_status = 401;
-                }
-                self.sendError(req, res, err);
-            } else {
-                self.sendResponse(req, res, { catalog: catalogWithTables });
-            }
-        }
+AnalysesController.prototype.register = function (app) {
+    app.get(
+        `${app.base_url_mapconfig}/analyses/catalog`,
+        cors(),
+        userMiddleware,
+        this.prepareContext,
+        this.createPGClient(),
+        this.getDataFromQuery({ queryTemplate: catalogQueryTpl, key: 'catalog' }),
+        this.getDataFromQuery({ queryTemplate: tablesQueryTpl, key: 'tables' }),
+        this.prepareResponse(),
+        this.setCacheControlHeader(),
+        this.sendResponse(),
+        this.unathorizedError()
     );
 };
 
-var catalogQueryTpl = dot.template(
-    'SELECT analysis_def->>\'type\' as type, * FROM cartodb.cdb_analysis_catalog WHERE username = \'{{=it._username}}\''
-);
-
-var tablesQueryTpl = dot.template([
-    "WITH analysis_tables AS (",
-    "    SELECT",
-    "        n.nspname AS nspname,",
-    "        c.relname AS relname,",
-    "        pg_total_relation_size(",
-    "            format('%s.%s', pg_catalog.quote_ident(n.nspname), pg_catalog.quote_ident(c.relname))",
-    "        ) AS size,",
-    "        format('%s.%s', pg_catalog.quote_ident(nspname), pg_catalog.quote_ident(relname)) AS fully_qualified_name",
-    "    FROM pg_catalog.pg_class c, pg_catalog.pg_namespace n",
-    "    WHERE c.relnamespace = n.oid",
-    "    AND pg_catalog.quote_ident(c.relname) ~ '^analysis_[a-z0-9]{10}_[a-z0-9]{40}$'",
-    "    AND n.nspname IN ('{{=it._username}}', 'public')",
-    ")",
-    "SELECT *, pg_size_pretty(size) as size_pretty",
-    "FROM analysis_tables",
-    "ORDER BY size DESC"
-].join('\n'));
-
-
-function getMetadata(username, pg, callback) {
-    var results = {
-        catalog: [],
-        tables: []
+AnalysesController.prototype.createPGClient = function () {
+    return function createPGClientMiddleware (req, res, next) {
+        res.locals.pg = new PSQL(dbParamsFromReqParams(res.locals));
+        next();
     };
-    step(
-        function getCatalog() {
-            pg.query(catalogQueryTpl({_username: username}), this, true); // use read-only transaction
-        },
-        function handleCatalog(err, resultSet) {
-            assert.ifError(err);
-            resultSet = resultSet || {};
-            results.catalog = resultSet.rows || [];
-            this();
-        },
-        function getTables(err) {
-            assert.ifError(err);
-            pg.query(tablesQueryTpl({_username: username}), this, true); // use read-only transaction
-        },
-        function handleTables(err, resultSet) {
-            assert.ifError(err);
-            resultSet = resultSet || {};
-            results.tables = resultSet.rows || [];
-            this();
-        },
-        function finish(err) {
+};
+
+AnalysesController.prototype.getDataFromQuery = function ({ queryTemplate, key }) {
+    const readOnlyTransactionOn = true;
+
+    return function getCatalogMiddleware(req, res, next) {
+        const { pg, user } = res.locals;
+        const sql = queryTemplate({ _username: user });
+
+        pg.query(sql, (err, resultSet = {}) => {
             if (err) {
-                return callback(err);
+                return next(err);
             }
 
-            return callback(null, results);
-        }
-    );
-}
+            res.locals[key] = resultSet.rows || [];
 
+            next();
+        }, readOnlyTransactionOn);
+    };
+};
+
+AnalysesController.prototype.prepareResponse = function () {
+    return function prepareResponseMiddleware (req, res, next) {
+        const { catalog, tables } = res.locals;
+
+        const analysisIdToTable = tables.reduce((analysisIdToTable, table) => {
+            const analysisId = table.relname.split('_')[2];
+
+            if (analysisId && analysisId.length === 40) {
+                analysisIdToTable[analysisId] = table;
+            }
+
+            return analysisIdToTable;
+        }, {});
+
+        const analysisCatalog = catalog.map(analysis => {
+            if (analysisIdToTable.hasOwnProperty(analysis.node_id)) {
+                analysis.table = analysisIdToTable[analysis.node_id];
+            }
+
+            return analysis;
+        })
+        .sort((analysisA, analysisB) => {
+            if (!!analysisA.table && !!analysisB.table) {
+                return analysisB.table.size - analysisA.table.size;
+            }
+
+            if (!!analysisA.table) {
+                return -1;
+            }
+
+            if (!!analysisB.table) {
+                return 1;
+            }
+
+            return -1;
+        });
+
+        res.body = { catalog: analysisCatalog };
+
+        next();
+    };
+};
+
+AnalysesController.prototype.setCacheControlHeader = function () {
+    return function setCacheControlHeaderMiddleware (req, res, next) {
+        res.set('Cache-Control', 'public,max-age=10,must-revalidate');
+        next();
+    };
+};
+
+AnalysesController.prototype.sendResponse = function() {
+    return function sendResponseMiddleware (req, res) {
+        res.status(200);
+
+        if (req.query && req.query.callback) {
+            res.jsonp(res.body);
+        } else {
+            res.json(res.body);
+        }
+    };
+};
+
+AnalysesController.prototype.unathorizedError = function () {
+    return function unathorizedErrorMiddleware(err, req, res, next) {
+        if (err.message.match(/permission\sdenied/)) {
+            err = new Error('Unauthorized');
+            err.http_status = 401;
+        }
+
+        next(err);
+    };
+};
+
+const catalogQueryTpl = ctx => `
+    SELECT analysis_def->>'type' as type, * FROM cdb_analysis_catalog WHERE username = '${ctx._username}'
+`;
+
+var tablesQueryTpl = ctx => `
+    WITH analysis_tables AS (
+        SELECT
+            n.nspname AS nspname,
+            c.relname AS relname,
+            pg_total_relation_size(
+                format('%s.%s', pg_catalog.quote_ident(n.nspname), pg_catalog.quote_ident(c.relname))
+            ) AS size,
+            format('%s.%s', pg_catalog.quote_ident(nspname), pg_catalog.quote_ident(relname)) AS fully_qualified_name
+        FROM pg_catalog.pg_class c, pg_catalog.pg_namespace n
+        WHERE c.relnamespace = n.oid
+        AND pg_catalog.quote_ident(c.relname) ~ '^analysis_[a-z0-9]{10}_[a-z0-9]{40}$'
+        AND n.nspname IN ('${ctx._username}', 'public')
+    )
+    SELECT *, pg_size_pretty(size) as size_pretty
+    FROM analysis_tables
+    ORDER BY size DESC
+`;
 
 function dbParamsFromReqParams(params) {
     var dbParams = {};

lib/cartodb/controllers/base.js

@@ -1,333 +0,0 @@
-var assert = require('assert');
-
-var _ = require('underscore');
-var step = require('step');
-var debug = require('debug')('windshaft:cartodb');
-
-var LZMA = require('lzma').LZMA;
-var lzmaWorker = new LZMA();
-
-// Whitelist query parameters and attach format
-var REQUEST_QUERY_PARAMS_WHITELIST = [
-    'config',
-    'map_key',
-    'api_key',
-    'auth_token',
-    'callback',
-    'zoom',
-    'lon',
-    'lat',
-    // analysis
-    'filters' // json
-];
-
-function BaseController(authApi, pgConnection) {
-    this.authApi = authApi;
-    this.pgConnection = pgConnection;
-}
-
-module.exports = BaseController;
-
-// jshint maxcomplexity:10
-/**
- * Whitelist input and get database name & default geometry type from
- * subdomain/user metadata held in CartoDB Redis
- * @param req - standard express request obj. Should have host & table
- * @param callback
- */
-BaseController.prototype.req2params = function(req, callback){
-    var self = this;
-
-    if ( req.query.lzma ) {
-
-        // Decode (from base64)
-        var lzma = new Buffer(req.query.lzma, 'base64')
-            .toString('binary')
-            .split('')
-            .map(function(c) {
-                return c.charCodeAt(0) - 128;
-            });
-
-
-        // Decompress
-        lzmaWorker.decompress(
-            lzma,
-            function(result) {
-                req.profiler.done('lzma');
-                try {
-                    delete req.query.lzma;
-                    _.extend(req.query, JSON.parse(result));
-                    self.req2params(req, callback);
-                } catch (err) {
-                    req.profiler.done('req2params');
-                    callback(new Error('Error parsing lzma as JSON: ' + err));
-                }
-            }
-        );
-        return;
-    }
-
-    var allowedQueryParams = REQUEST_QUERY_PARAMS_WHITELIST;
-    if (Array.isArray(req.context.allowedQueryParams)) {
-        allowedQueryParams = allowedQueryParams.concat(req.context.allowedQueryParams);
-    }
-    req.query = _.pick(req.query, allowedQueryParams);
-    req.params = _.extend({}, req.params); // shuffle things as request is a strange array/object
-
-    var user = req.context.user;
-
-    if ( req.params.token ) {
-        // Token might match the following patterns:
-        // - {user}@{tpl_id}@{token}:{cache_buster}
-        var tksplit = req.params.token.split(':');
-        req.params.token = tksplit[0];
-        if ( tksplit.length > 1 ) {
-            req.params.cache_buster= tksplit[1];
-        }
-        tksplit = req.params.token.split('@');
-        if ( tksplit.length > 1 ) {
-            req.params.signer = tksplit.shift();
-            if ( ! req.params.signer ) {
-                req.params.signer = user;
-            }
-            else if ( req.params.signer !== user ) {
-                var err = new Error(
-                        'Cannot use map signature of user "' + req.params.signer + '" on db of user "' + user + '"'
-                );
-                err.http_status = 403;
-                req.profiler.done('req2params');
-                callback(err);
-                return;
-            }
-            if ( tksplit.length > 1 ) {
-                /*var template_hash = */tksplit.shift(); // unused
-            }
-            req.params.token = tksplit.shift();
-        }
-    }
-
-    // bring all query values onto req.params object
-    _.extend(req.params, req.query);
-
-    req.profiler.done('req2params.setup');
-
-    step(
-        function getPrivacy(){
-            self.authApi.authorize(req, this);
-        },
-        function validateAuthorization(err, authorized) {
-            req.profiler.done('authorize');
-            assert.ifError(err);
-            if(!authorized) {
-                err = new Error("Sorry, you are unauthorized (permission denied)");
-                err.http_status = 403;
-                throw err;
-            }
-            return null;
-        },
-        function getDatabase(err){
-            assert.ifError(err);
-            self.pgConnection.setDBConn(user, req.params, this);
-        },
-        function finishSetup(err) {
-            if ( err ) {
-                req.profiler.done('req2params');
-                return callback(err, req);
-            }
-
-            // Add default database connection parameters
-            // if none given
-            _.defaults(req.params, {
-                dbuser: global.environment.postgres.user,
-                dbpassword: global.environment.postgres.password,
-                dbhost: global.environment.postgres.host,
-                dbport: global.environment.postgres.port
-            });
-
-            req.profiler.done('req2params');
-            callback(null, req);
-        }
-    );
-};
-// jshint maxcomplexity:6
-
-// jshint maxcomplexity:9
-BaseController.prototype.send = function(req, res, body, status, headers) {
-    if (req.params.dbhost) {
-        res.set('X-Served-By-DB-Host', req.params.dbhost);
-    }
-
-    res.set('X-Tiler-Profiler', req.profiler.toJSONString());
-
-    if (headers) {
-        res.set(headers);
-    }
-
-    res.status(status);
-
-    if (!Buffer.isBuffer(body) && typeof body === 'object') {
-        if (req.query && req.query.callback) {
-            res.jsonp(body);
-        } else {
-            res.json(body);
-        }
-    } else {
-        res.send(body);
-    }
-
-    try {
-        // May throw due to dns, see
-        // See http://github.com/CartoDB/Windshaft/issues/166
-        req.profiler.sendStats();
-    } catch (err) {
-        debug("error sending profiling stats: " + err);
-    }
-};
-// jshint maxcomplexity:6
-
-BaseController.prototype.sendError = function(req, res, err, label) {
-    var allErrors = Array.isArray(err) ? err : [err];
-
-    allErrors = populateTimeoutErrors(allErrors);
-
-    label = label || 'UNKNOWN';
-    err = allErrors[0] || new Error(label);
-    allErrors[0] = err;
-
-    var statusCode = findStatusCode(err);
-
-    if (err.message === 'Tile does not exist' && req.params.format === 'mvt') {
-        statusCode = 204;
-    }
-
-    debug('[%s ERROR] -- %d: %s, %s', label, statusCode, err, err.stack);
-
-    // If a callback was requested, force status to 200
-    if (req.query && req.query.callback) {
-        statusCode = 200;
-    }
-
-    var errorResponseBody = {
-        errors: allErrors.map(errorMessage),
-        errors_with_context: allErrors.map(errorMessageWithContext)
-    };
-
-    this.send(req, res, errorResponseBody, statusCode);
-};
-
-function stripConnectionInfo(message) {
-    // Strip connection info, if any
-    return message
-        // See https://github.com/CartoDB/Windshaft/issues/173
-        .replace(/Connection string: '[^']*'\n\s/im, '')
-        // See https://travis-ci.org/CartoDB/Windshaft/jobs/20703062#L1644
-        .replace(/is the server.*encountered/im, 'encountered');
-}
-
-var ERROR_INFO_TO_EXPOSE = {
-    message: true,
-    layer: true,
-    type: true,
-    analysis: true,
-    subtype: true
-};
-
-function shouldBeExposed (prop) {
-    return !!ERROR_INFO_TO_EXPOSE[prop];
-}
-
-function errorMessage(err) {
-    // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
-    var message = (_.isString(err) ? err : err.message) || 'Unknown error';
-
-    return stripConnectionInfo(message);
-}
-
-function errorMessageWithContext(err) {
-    // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
-    var message = (_.isString(err) ? err : err.message) || 'Unknown error';
-
-    var error = {
-        type: err.type || 'unknown',
-        message: stripConnectionInfo(message),
-    };
-
-    for (var prop in err) {
-        // type & message are properties from Error's prototype and will be skipped
-        if (err.hasOwnProperty(prop) && shouldBeExposed(prop)) {
-            error[prop] = err[prop];
-        }
-    }
-
-    return error;
-}
-module.exports.errorMessage = errorMessage;
-
-function findStatusCode(err) {
-    var statusCode;
-    if ( err.http_status ) {
-        statusCode = err.http_status;
-    } else {
-        statusCode = statusFromErrorMessage('' + err);
-    }
-    return statusCode;
-}
-module.exports.findStatusCode = findStatusCode;
-
-function statusFromErrorMessage(errMsg) {
-    // Find an appropriate statusCode based on message
-    // jshint maxcomplexity:7
-    var statusCode = 400;
-    if ( -1 !== errMsg.indexOf('permission denied') ) {
-        statusCode = 403;
-    }
-    else if ( -1 !== errMsg.indexOf('authentication failed') ) {
-        statusCode = 403;
-    }
-    else if (errMsg.match(/Postgis Plugin.*[\s|\n].*column.*does not exist/)) {
-        statusCode = 400;
-    }
-    else if ( -1 !== errMsg.indexOf('does not exist') ) {
-        if ( -1 !== errMsg.indexOf(' role ') ) {
-            statusCode = 403; // role 'xxx' does not exist
-        } else if ( errMsg.match(/function .* does not exist/) ) {
-            statusCode = 400; // invalid SQL (SQL function does not exist)
-        } else {
-            statusCode = 404;
-        }
-    }
-
-    return statusCode;
-}
-
-function isRenderTimeoutError (err) {
-    return err.message === 'Render timed out';
-}
-
-function isDatasourceTimeoutError (err) {
-    return err.message && err.message.match(/canceling statement due to statement timeout/i);
-}
-
-function isTimeoutError (err) {
-    return isRenderTimeoutError(err) || isDatasourceTimeoutError(err);
-}
-
-function populateTimeoutErrors (errors) {
-    return errors.map(function (error) {
-        if (isRenderTimeoutError(error)) {
-            error.subtype = 'render';
-        }
-
-        if (isDatasourceTimeoutError(error)) {
-            error.subtype = 'datasource';
-        }
-
-        if (isTimeoutError(error)) {
-            error.message = 'You are over platform\'s limits. Please contact us to know more details';
-            error.type = 'limit';
-            error.http_status = 429;
-        }
-
-        return error;
-    });
-}

lib/cartodb/controllers/layergroup.js

@@ -1,12 +1,10 @@
 var assert = require('assert');
 var step = require('step');
 
-var util = require('util');
-var BaseController = require('./base');
-
 var cors = require('../middleware/cors');
 var userMiddleware = require('../middleware/user');
 var allowQueryParams = require('../middleware/allow-query-params');
+var vectorError = require('../middleware/vector-error');
 
 var DataviewBackend = require('../backends/dataview');
 var AnalysisStatusBackend = require('../backends/analysis-status');
@@ -28,10 +26,8 @@ var QueryTables = require('cartodb-query-tables');
  * @param {AnalysisBackend} analysisBackend
  * @constructor
  */
-function LayergroupController(authApi, pgConnection, mapStore, tileBackend, previewBackend, attributesBackend,
+function LayergroupController(prepareContext, pgConnection, mapStore, tileBackend, previewBackend, attributesBackend,
                               surrogateKeysCache, userLimitsApi, layergroupAffectedTables, analysisBackend) {
-    BaseController.call(this, authApi, pgConnection);
-
     this.pgConnection = pgConnection;
     this.mapStore = mapStore;
     this.tileBackend = tileBackend;
@@ -43,39 +39,66 @@ function LayergroupController(authApi, pgConnection, mapStore, tileBackend, prev
 
     this.dataviewBackend = new DataviewBackend(analysisBackend);
     this.analysisStatusBackend = new AnalysisStatusBackend();
-}
 
-util.inherits(LayergroupController, BaseController);
+    this.prepareContext = prepareContext;
+}
 
 module.exports = LayergroupController;
 
-
 LayergroupController.prototype.register = function(app) {
-    app.get(app.base_url_mapconfig +
-        '/:token/:z/:x/:y@:scale_factor?x.:format', cors(), userMiddleware,
-        this.tile.bind(this));
+    app.get(
+        app.base_url_mapconfig + '/:token/:z/:x/:y@:scale_factor?x.:format',
+        cors(),
+        userMiddleware,
+        this.prepareContext,
+        this.tile.bind(this),
+        vectorError()
+    );
 
-    app.get(app.base_url_mapconfig +
-        '/:token/:z/:x/:y.:format', cors(), userMiddleware,
-        this.tile.bind(this));
+    app.get(
+        app.base_url_mapconfig + '/:token/:z/:x/:y.:format',
+        cors(),
+        userMiddleware,
+        this.prepareContext,
+        this.tile.bind(this),
+        vectorError()
+    );
 
-    app.get(app.base_url_mapconfig +
-        '/:token/:layer/:z/:x/:y.(:format)', cors(), userMiddleware,
-        this.layer.bind(this));
+    app.get(
+        app.base_url_mapconfig + '/:token/:layer/:z/:x/:y.(:format)',
+        cors(),
+        userMiddleware,
+        validateLayerRouteMiddleware,
+        this.prepareContext,
+        this.layer.bind(this),
+        vectorError()
+    );
 
-    app.get(app.base_url_mapconfig +
-        '/:token/:layer/attributes/:fid', cors(), userMiddleware,
-        this.attributes.bind(this));
+    app.get(
+        app.base_url_mapconfig + '/:token/:layer/attributes/:fid',
+        cors(),
+        userMiddleware,
+        this.prepareContext,
+        this.attributes.bind(this)
+    );
 
-    app.get(app.base_url_mapconfig +
-        '/static/center/:token/:z/:lat/:lng/:width/:height.:format',
-        cors(), userMiddleware, allowQueryParams(['layer']),
-        this.center.bind(this));
+    app.get(
+        app.base_url_mapconfig + '/static/center/:token/:z/:lat/:lng/:width/:height.:format',
+        cors(),
+        userMiddleware,
+        allowQueryParams(['layer']),
+        this.prepareContext,
+        this.center.bind(this)
+    );
 
-    app.get(app.base_url_mapconfig +
-        '/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format',
-        cors(), userMiddleware, allowQueryParams(['layer']),
-        this.bbox.bind(this));
+    app.get(
+        app.base_url_mapconfig + '/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format',
+        cors(),
+        userMiddleware,
+        allowQueryParams(['layer']),
+        this.prepareContext,
+        this.bbox.bind(this)
+    );
 
     // Undocumented/non-supported API endpoint methods.
     // Use at your own peril.
@@ -83,6 +106,7 @@ LayergroupController.prototype.register = function(app) {
     var allowedDataviewQueryParams = [
         'filters', // json
         'own_filter', // 0, 1
+        'no_filters', // 0, 1
         'bbox', // w,s,e,n
         'start', // number
         'end', // number
@@ -90,7 +114,8 @@ LayergroupController.prototype.register = function(app) {
         'bins', // number
         'aggregation', //string
         'offset', // number
-        'q' // widgets search
+        'q', // widgets search
+        'categories', // number
     ];
 
     app.get(
@@ -98,6 +123,7 @@ LayergroupController.prototype.register = function(app) {
         cors(),
         userMiddleware,
         allowQueryParams(allowedDataviewQueryParams),
+        this.prepareContext,
         this.dataview.bind(this)
     );
 
@@ -106,6 +132,7 @@ LayergroupController.prototype.register = function(app) {
         cors(),
         userMiddleware,
         allowQueryParams(allowedDataviewQueryParams),
+        this.prepareContext,
         this.dataview.bind(this)
     );
 
@@ -114,6 +141,7 @@ LayergroupController.prototype.register = function(app) {
         cors(),
         userMiddleware,
         allowQueryParams(allowedDataviewQueryParams),
+        this.prepareContext,
         this.dataviewSearch.bind(this)
     );
 
@@ -122,30 +150,32 @@ LayergroupController.prototype.register = function(app) {
         cors(),
         userMiddleware,
         allowQueryParams(allowedDataviewQueryParams),
+        this.prepareContext,
         this.dataviewSearch.bind(this)
     );
 
-    app.get(app.base_url_mapconfig +
-        '/:token/analysis/node/:nodeId', cors(), userMiddleware,
-        this.analysisNodeStatus.bind(this));
+    app.get(
+        app.base_url_mapconfig + '/:token/analysis/node/:nodeId',
+        cors(),
+        userMiddleware,
+        this.prepareContext,
+        this.analysisNodeStatus.bind(this)
+    );
 };
 
-LayergroupController.prototype.analysisNodeStatus = function(req, res) {
+LayergroupController.prototype.analysisNodeStatus = function(req, res, next) {
     var self = this;
 
     step(
-        function setupParams() {
-            self.req2params(req, this);
-        },
-        function retrieveNodeStatus(err) {
-            assert.ifError(err);
-            self.analysisStatusBackend.getNodeStatus(req.params, this);
+        function retrieveNodeStatus() {
+            self.analysisStatusBackend.getNodeStatus(res.locals, this);
         },
         function finish(err, nodeStatus, stats) {
             req.profiler.add(stats || {});
 
             if (err) {
-                self.sendError(req, res, err, 'GET NODE STATUS');
+                err.label = 'GET NODE STATUS';
+                next(err);
             } else {
                 self.sendResponse(req, res, nodeStatus, 200, {
                     'Cache-Control': 'public,max-age=5',
@@ -156,54 +186,50 @@ LayergroupController.prototype.analysisNodeStatus = function(req, res) {
     );
 };
 
-LayergroupController.prototype.dataview = function(req, res) {
+LayergroupController.prototype.dataview = function(req, res, next) {
     var self = this;
 
     step(
-        function setupParams() {
-            self.req2params(req, this);
-        },
-        function retrieveDataview(err) {
-            assert.ifError(err);
-
+        function retrieveDataview() {
             var mapConfigProvider = new MapStoreMapConfigProvider(
-                self.mapStore, req.context.user, self.userLimitsApi, req.params
+                self.mapStore, res.locals.user, self.userLimitsApi, res.locals
+            );
+            self.dataviewBackend.getDataview(
+                mapConfigProvider,
+                res.locals.user,
+                res.locals,
+                this
             );
-            self.dataviewBackend.getDataview(mapConfigProvider, req.context.user, req.params, this);
         },
         function finish(err, dataview, stats) {
             req.profiler.add(stats || {});
 
             if (err) {
-                self.sendError(req, res, err, 'GET DATAVIEW');
+                err.label = 'GET DATAVIEW';
+                next(err);
             } else {
                 self.sendResponse(req, res, dataview, 200);
             }
         }
     );
-
 };
 
-LayergroupController.prototype.dataviewSearch = function(req, res) {
+LayergroupController.prototype.dataviewSearch = function(req, res, next) {
     var self = this;
 
     step(
-        function setupParams() {
-            self.req2params(req, this);
-        },
-        function searchDataview(err) {
-            assert.ifError(err);
-
+        function searchDataview() {
             var mapConfigProvider = new MapStoreMapConfigProvider(
-                self.mapStore, req.context.user, self.userLimitsApi, req.params
+                self.mapStore, res.locals.user, self.userLimitsApi, res.locals
             );
-            self.dataviewBackend.search(mapConfigProvider, req.context.user, req.params, this);
+            self.dataviewBackend.search(mapConfigProvider, res.locals.user, req.params.dataviewName, res.locals, this);
         },
         function finish(err, searchResult, stats) {
             req.profiler.add(stats || {});
 
             if (err) {
-                self.sendError(req, res, err, 'GET DATAVIEW SEARCH');
+                err.label = 'GET DATAVIEW SEARCH';
+                next(err);
             } else {
                 self.sendResponse(req, res, searchResult, 200);
             }
@@ -212,28 +238,24 @@ LayergroupController.prototype.dataviewSearch = function(req, res) {
 
 };
 
-LayergroupController.prototype.attributes = function(req, res) {
+LayergroupController.prototype.attributes = function(req, res, next) {
     var self = this;
 
     req.profiler.start('windshaft.maplayer_attribute');
 
     step(
-        function setupParams() {
-            self.req2params(req, this);
-        },
-        function retrieveFeatureAttributes(err) {
-            assert.ifError(err);
-
+        function retrieveFeatureAttributes() {
             var mapConfigProvider = new MapStoreMapConfigProvider(
-                self.mapStore, req.context.user, self.userLimitsApi, req.params
+                self.mapStore, res.locals.user, self.userLimitsApi, res.locals
             );
-            self.attributesBackend.getFeatureAttributes(mapConfigProvider, req.params, false, this);
+            self.attributesBackend.getFeatureAttributes(mapConfigProvider, res.locals, false, this);
         },
         function finish(err, tile, stats) {
             req.profiler.add(stats || {});
 
             if (err) {
-                self.sendError(req, res, err, 'GET ATTRIBUTES');
+                err.label = 'GET ATTRIBUTES';
+                next(err);
             } else {
                 self.sendResponse(req, res, tile, 200);
             }
@@ -243,44 +265,41 @@ LayergroupController.prototype.attributes = function(req, res) {
 };
 
 // Gets a tile for a given token and set of tile ZXY coords. (OSM style)
-LayergroupController.prototype.tile = function(req, res) {
+LayergroupController.prototype.tile = function(req, res, next) {
     req.profiler.start('windshaft.map_tile');
-    this.tileOrLayer(req, res);
+    this.tileOrLayer(req, res, next);
 };
 
 // Gets a tile for a given token, layer set of tile ZXY coords. (OSM style)
 LayergroupController.prototype.layer = function(req, res, next) {
-    if (req.params.token === 'static') {
-        return next();
-    }
     req.profiler.start('windshaft.maplayer_tile');
-    this.tileOrLayer(req, res);
+    this.tileOrLayer(req, res, next);
 };
 
-LayergroupController.prototype.tileOrLayer = function (req, res) {
+LayergroupController.prototype.tileOrLayer = function (req, res, next) {
     var self = this;
 
     step(
-        function mapController$prepareParams() {
-            self.req2params(req, this);
-        },
-        function mapController$getTileOrGrid(err) {
-            assert.ifError(err);
+        function mapController$getTileOrGrid() {
             self.tileBackend.getTile(
-                new MapStoreMapConfigProvider(self.mapStore, req.context.user, self.userLimitsApi, req.params),
+                new MapStoreMapConfigProvider(self.mapStore, res.locals.user, self.userLimitsApi, res.locals),
                 req.params, this
             );
         },
         function mapController$finalize(err, tile, headers, stats) {
             req.profiler.add(stats);
-            self.finalizeGetTileOrGrid(err, req, res, tile, headers);
+            self.finalizeGetTileOrGrid(err, req, res, tile, headers, next);
         }
     );
 };
 
+function getStatusCode(tile, format){
+    return tile.length===0 && format==='mvt'? 204:200;
+}
+
 // This function is meant for being called as the very last
 // step by all endpoints serving tiles or grids
-LayergroupController.prototype.finalizeGetTileOrGrid = function(err, req, res, tile, headers) {
+LayergroupController.prototype.finalizeGetTileOrGrid = function(err, req, res, tile, headers, next) {
     var supportedFormats = {
         grid_json: true,
         json_torque: true,
@@ -309,52 +328,52 @@ LayergroupController.prototype.finalizeGetTileOrGrid = function(err, req, res, t
         }
         err.message = errMsg;
 
-        this.sendError(req, res, err, 'TILE RENDER');
+        err.label = 'TILE RENDER';
+        next(err);
+
         global.statsClient.increment('windshaft.tiles.error');
         global.statsClient.increment('windshaft.tiles.' + formatStat + '.error');
     } else {
-        this.sendResponse(req, res, tile, 200, headers);
+        this.sendResponse(req, res, tile, getStatusCode(tile, formatStat), headers);
         global.statsClient.increment('windshaft.tiles.success');
         global.statsClient.increment('windshaft.tiles.' + formatStat + '.success');
     }
 };
 
-LayergroupController.prototype.bbox = function(req, res) {
+LayergroupController.prototype.bbox = function(req, res, next) {
     this.staticMap(req, res, +req.params.width, +req.params.height, {
         west: +req.params.west,
         north: +req.params.north,
         east: +req.params.east,
         south: +req.params.south
-    });
+    }, null, next);
 };
 
-LayergroupController.prototype.center = function(req, res) {
+LayergroupController.prototype.center = function(req, res, next) {
     this.staticMap(req, res, +req.params.width, +req.params.height, +req.params.z, {
         lng: +req.params.lng,
         lat: +req.params.lat
-    });
+    }, next);
 };
 
-LayergroupController.prototype.staticMap = function(req, res, width, height, zoom /* bounds */, center) {
+LayergroupController.prototype.staticMap = function(req, res, width, height, zoom /* bounds */, center, next) {
     var format = req.params.format === 'jpg' ? 'jpeg' : 'png';
-    req.params.layer = 'all';
-    req.params.format = 'png';
+    // We force always the tile to be generated using PNG because
+    // is the only format we support by now
+    res.locals.format = 'png';
+    res.locals.layer = res.locals.layer || 'all';
 
     var self = this;
 
     step(
-        function reqParams() {
-            self.req2params(req, this);
-        },
-        function getImage(err) {
-            assert.ifError(err);
+        function getImage() {
             if (center) {
                 self.previewBackend.getImage(
-                    new MapStoreMapConfigProvider(self.mapStore, req.context.user, self.userLimitsApi, req.params),
+                    new MapStoreMapConfigProvider(self.mapStore, res.locals.user, self.userLimitsApi, res.locals),
                     format, width, height, zoom, center, this);
             } else {
                 self.previewBackend.getImage(
-                    new MapStoreMapConfigProvider(self.mapStore, req.context.user, self.userLimitsApi, req.params),
+                    new MapStoreMapConfigProvider(self.mapStore, res.locals.user, self.userLimitsApi, res.locals),
                     format, width, height, zoom /* bounds */, this);
             }
         },
@@ -363,7 +382,8 @@ LayergroupController.prototype.staticMap = function(req, res, width, height, zoo
             req.profiler.add(stats || {});
 
             if (err) {
-                self.sendError(req, res, err, 'STATIC_MAP');
+                err.label = 'STATIC_MAP';
+                next(err);
             } else {
                 res.set('Content-Type', headers['Content-Type'] || 'image/' + format);
                 self.sendResponse(req, res, image, 200);
@@ -381,18 +401,18 @@ LayergroupController.prototype.sendResponse = function(req, res, body, status, h
 
     // Set Last-Modified header
     var lastUpdated;
-    if (req.params.cache_buster) {
+    if (res.locals.cache_buster) {
         // Assuming cache_buster is a timestamp
-        lastUpdated = new Date(parseInt(req.params.cache_buster));
+        lastUpdated = new Date(parseInt(res.locals.cache_buster));
     } else {
         lastUpdated = new Date();
     }
     res.set('Last-Modified', lastUpdated.toUTCString());
 
-    var dbName = req.params.dbname;
+    var dbName = res.locals.dbname;
     step(
         function getAffectedTables() {
-            self.getAffectedTables(req.context.user, dbName, req.params.token, this);
+            self.getAffectedTables(res.locals.user, dbName, res.locals.token, this);
         },
         function sendResponse(err, affectedTables) {
             req.profiler.done('affectedTables');
@@ -403,10 +423,24 @@ LayergroupController.prototype.sendResponse = function(req, res, body, status, h
                 res.set('X-Cache-Channel', affectedTables.getCacheChannel());
                 self.surrogateKeysCache.tag(res, affectedTables);
             }
-            self.send(req, res, body, status, headers);
+
+            if (headers) {
+                res.set(headers);
+            }
+
+            res.status(status);
+
+            if (!Buffer.isBuffer(body) && typeof body === 'object') {
+                if (req.query && req.query.callback) {
+                    res.jsonp(body);
+                } else {
+                    res.json(body);
+                }
+            } else {
+                res.send(body);
+            }
         }
     );
-
 };
 
 LayergroupController.prototype.getAffectedTables = function(user, dbName, layergroupId, callback) {
@@ -468,3 +502,12 @@ LayergroupController.prototype.getAffectedTables = function(user, dbName, layerg
         callback
     );
 };
+
+
+function validateLayerRouteMiddleware(req, res, next) {
+    if (req.params.token === 'static') {
+        return next('route');
+    }
+
+    next();
+}

lib/cartodb/controllers/map.js

@@ -1,16 +1,12 @@
 var _ = require('underscore');
-var assert = require('assert');
-var step = require('step');
 var windshaft = require('windshaft');
 var QueryTables = require('cartodb-query-tables');
 
 var ResourceLocator = require('../models/resource-locator');
 
-var util = require('util');
-var BaseController = require('./base');
-
 var cors = require('../middleware/cors');
 var userMiddleware = require('../middleware/user');
+const allowQueryParams = require('../middleware/allow-query-params');
 
 var MapConfig = windshaft.model.MapConfig;
 var Datasource = windshaft.model.Datasource;
@@ -20,7 +16,6 @@ var NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
 var NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider');
 var CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/create-layergroup-provider');
 
-
 /**
  * @param {AuthApi} authApi
  * @param {PgConnection} pgConnection
@@ -34,12 +29,9 @@ var CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/cr
  * @param {StatsBackend} statsBackend
  * @constructor
  */
-function MapController(authApi, pgConnection, templateMaps, mapBackend, metadataBackend,
+function MapController(prepareContext, pgConnection, templateMaps, mapBackend, metadataBackend,
                        surrogateKeysCache, userLimitsApi, layergroupAffectedTables, mapConfigAdapter,
                        statsBackend) {
-
-    BaseController.call(this, authApi, pgConnection);
-
     this.pgConnection = pgConnection;
     this.templateMaps = templateMaps;
     this.mapBackend = mapBackend;
@@ -52,334 +44,426 @@ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadata
     this.resourceLocator = new ResourceLocator(global.environment);
 
     this.statsBackend = statsBackend;
+    this.prepareContext = prepareContext;
 }
 
-util.inherits(MapController, BaseController);
-
 module.exports = MapController;
 
-
 MapController.prototype.register = function(app) {
-    app.get(app.base_url_mapconfig, cors(), userMiddleware, this.createGet.bind(this));
-    app.post(app.base_url_mapconfig, cors(), userMiddleware, this.createPost.bind(this));
-    app.get(app.base_url_templated + '/:template_id/jsonp', cors(), userMiddleware, this.jsonp.bind(this));
-    app.post(app.base_url_templated + '/:template_id', cors(), userMiddleware, this.instantiate.bind(this));
+    const { base_url_mapconfig, base_url_templated } = app;
+    const useTemplate = true;
+
+    app.get(base_url_mapconfig, this.composeCreateMapMiddleware());
+    app.post(base_url_mapconfig, this.composeCreateMapMiddleware());
+    app.get(`${base_url_templated}/:template_id/jsonp`, this.composeCreateMapMiddleware(useTemplate));
+    app.post(`${base_url_templated}/:template_id`, this.composeCreateMapMiddleware(useTemplate));
     app.options(app.base_url_mapconfig, cors('Content-Type'));
 };
 
-MapController.prototype.createGet = function(req, res){
-    req.profiler.start('windshaft.createmap_get');
+MapController.prototype.composeCreateMapMiddleware = function (useTemplate = false) {
+    const isTemplateInstantiation = useTemplate;
+    const useTemplateHash = useTemplate;
+    const includeQuery = !useTemplate;
+    const label =  useTemplate ? 'NAMED MAP LAYERGROUP' : 'ANONYMOUS LAYERGROUP';
+    const addContext = !useTemplate;
 
-    this.create(req, res, function createGet$prepareConfig(err, req) {
-        assert.ifError(err);
-        if ( ! req.params.config ) {
-            throw new Error('layergroup GET needs a "config" parameter');
-        }
-        return JSON.parse(req.params.config);
-    });
+    return [
+        cors(),
+        userMiddleware,
+        allowQueryParams(['aggregation']),
+        this.prepareContext,
+        this.initProfiler(isTemplateInstantiation),
+        this.checkJsonContentType(),
+        useTemplate ? this.checkInstantiteLayergroup() : this.checkCreateLayergroup(),
+        useTemplate ? this.getTemplate() : this.prepareAdapterMapConfig(),
+        useTemplate ? this.instantiateLayergroup() : this.createLayergroup(),
+        this.incrementMapViewCount(),
+        this.augmentLayergroupData(),
+        this.getAffectedTables(),
+        this.setCacheChannel(),
+        this.setLastModified(),
+        this.setLastUpdatedTimeToLayergroup(),
+        this.setCacheControl(),
+        this.setLayerStats(),
+        this.setLayergroupIdHeader(useTemplateHash),
+        this.setDataviewsAndWidgetsUrlsToLayergroupMetadata(),
+        this.setAnalysesMetadataToLayergroup(includeQuery),
+        this.setTurboCartoMetadataToLayergroup(),
+        this.setAggregationMetadataToLayergroup(),
+        this.setTilejsonMetadataToLayergroup(),
+        this.setSurrogateKeyHeader(),
+        this.sendResponse(),
+        this.augmentError({ label, addContext })
+    ];
 };
 
-MapController.prototype.createPost = function(req, res) {
-    req.profiler.start('windshaft.createmap_post');
+MapController.prototype.initProfiler = function (isTemplateInstantiation) {
+    const operation = isTemplateInstantiation ? 'instance_template' : 'createmap';
 
-    this.create(req, res, function createPost$prepareConfig(err, req) {
-        assert.ifError(err);
-        if (!req.is('application/json')) {
-            throw new Error('layergroup POST data must be of type application/json');
-        }
-        return req.body;
-    });
+    return function initProfilerMiddleware (req, res, next) {
+        req.profiler.start(`windshaft-cartodb.${operation}_${req.method.toLowerCase()}`);
+        req.profiler.done(`${operation}.initProfilerMiddleware`);
+        next();
+    };
 };
 
-MapController.prototype.instantiate = function(req, res) {
-    req.profiler.start('windshaft-cartodb.instance_template_post');
-
-    this.instantiateTemplate(req, res, function prepareTemplateParams(callback) {
-        if (!req.is('application/json')) {
-            return callback(new Error('Template POST data must be of type application/json'));
+MapController.prototype.checkJsonContentType = function () {
+    return function checkJsonContentTypeMiddleware(req, res, next) {
+        if (req.method === 'POST' && !req.is('application/json')) {
+            return next(new Error('POST data must be of type application/json'));
         }
-        return callback(null, req.body);
-    });
+
+        req.profiler.done('checkJsonContentTypeMiddleware');
+
+        next();
+    };
 };
 
-MapController.prototype.jsonp = function(req, res) {
-    req.profiler.start('windshaft-cartodb.instance_template_get');
+MapController.prototype.checkInstantiteLayergroup = function () {
+    return function checkInstantiteLayergroupMiddleware(req, res, next) {
+        if (req.method === 'GET') {
+            const { callback, config } = req.query;
 
-    this.instantiateTemplate(req, res, function prepareJsonTemplateParams(callback) {
-        var err = null;
-        if ( req.query.callback === undefined || req.query.callback.length === 0) {
-            err = new Error('callback parameter should be present and be a function name');
-        }
+            if (callback === undefined || callback.length === 0) {
+                return next(new Error('callback parameter should be present and be a function name'));
+            }
 
-        var templateParams = {};
-        if (req.query.config) {
-            try {
-                templateParams = JSON.parse(req.query.config);
-            } catch(e) {
-                err = new Error('Invalid config parameter, should be a valid JSON');
+            if (config) {
+                try {
+                    req.body = JSON.parse(config);
+                } catch(e) {
+                    return next(new Error('Invalid config parameter, should be a valid JSON'));
+                }
             }
         }
 
-        return callback(err, templateParams);
-    });
+        req.profiler.done('checkInstantiteLayergroup');
+
+        return next();
+    };
 };
 
-MapController.prototype.create = function(req, res, prepareConfigFn) {
-    var self = this;
+MapController.prototype.checkCreateLayergroup = function () {
+    return function checkCreateLayergroupMiddleware (req, res, next) {
+        if (req.method === 'GET') {
+            const { config } = res.locals;
 
-    var mapConfig;
+            if (!config) {
+                return next(new Error('layergroup GET needs a "config" parameter'));
+            }
 
-    var context = {};
+            try {
+                req.body = JSON.parse(config);
+            } catch (err) {
+                return next(err);
+            }
+        }
 
-    step(
-        function setupParams(){
-            self.req2params(req, this);
-        },
-        prepareConfigFn,
-        function prepareAdapterMapConfig(err, requestMapConfig) {
-            assert.ifError(err);
-            context.analysisConfiguration = {
-                user: req.context.user,
+        req.profiler.done('checkCreateLayergroup');
+        return next();
+    };
+};
+
+MapController.prototype.getTemplate = function () {
+    return function getTemplateMiddleware (req, res, next) {
+        const templateParams = req.body;
+        const { user } = res.locals;
+
+        const mapconfigProvider = new NamedMapMapConfigProvider(
+            this.templateMaps,
+            this.pgConnection,
+            this.metadataBackend,
+            this.userLimitsApi,
+            this.mapConfigAdapter,
+            user,
+            req.params.template_id,
+            templateParams,
+            res.locals.auth_token,
+            res.locals
+        );
+
+        mapconfigProvider.getMapConfig((err, mapconfig, rendererParams) => {
+            req.profiler.done('named.getMapConfig');
+            if (err) {
+                return next(err);
+            }
+
+            res.locals.mapconfig = mapconfig;
+            res.locals.rendererParams = rendererParams;
+            res.locals.mapconfigProvider = mapconfigProvider;
+
+            next();
+        });
+    }.bind(this);
+};
+
+MapController.prototype.prepareAdapterMapConfig = function () {
+    return function prepareAdapterMapConfigMiddleware(req, res, next) {
+        const requestMapConfig = req.body;
+        const { user, dbhost, dbport, dbname, dbuser, dbpassword, api_key } = res.locals;
+
+        const context = {
+            analysisConfiguration: {
+                user,
                 db: {
-                    host: req.params.dbhost,
-                    port: req.params.dbport,
-                    dbname: req.params.dbname,
-                    user: req.params.dbuser,
-                    pass: req.params.dbpassword
+                    host: dbhost,
+                    port: dbport,
+                    dbname: dbname,
+                    user: dbuser,
+                    pass: dbpassword
                 },
                 batch: {
-                    username: req.context.user,
-                    apiKey: req.params.api_key
+                    username: user,
+                    apiKey: api_key
                 }
-            };
-            self.mapConfigAdapter.getMapConfig(req.context.user, requestMapConfig, req.params, context, this);
-        },
-        function createLayergroup(err, requestMapConfig) {
-            assert.ifError(err);
-            var datasource = context.datasource || Datasource.EmptyDatasource();
-            mapConfig = new MapConfig(requestMapConfig, datasource);
-            self.mapBackend.createLayergroup(
-                mapConfig, req.params,
-                new CreateLayergroupMapConfigProvider(mapConfig, req.context.user, self.userLimitsApi, req.params),
-                this
-            );
-        },
-        function afterLayergroupCreate(err, layergroup) {
-            assert.ifError(err);
-            self.afterLayergroupCreate(req, res, mapConfig, layergroup, context.analysesResults, this);
-        },
-        function finish(err, layergroup) {
-            if (err) {
-                if (Number.isFinite(err.layerIndex)) {
-                    var error = new Error(err.message);
-                    error.http_status = err.http_status;
-
-                    if (!err.http_status && err.message.indexOf('column "the_geom_webmercator" does not exist') >= 0) {
-                        error.http_status = 400;
-                    }
-
-                    error.type = 'layer';
-                    error.subtype = err.message.indexOf('Postgis Plugin') >= 0 ? 'query' : undefined;
-                    error.layer = {
-                        id: mapConfig.getLayerId(err.layerIndex),
-                        index: err.layerIndex,
-                        type: mapConfig.layerType(err.layerIndex)
-                    };
-
-                    err = error;
-                }
-                self.sendError(req, res, err, 'ANONYMOUS LAYERGROUP');
-            } else {
-                var analysesResults = context.analysesResults || [];
-                self.addDataviewsAndWidgetsUrls(req.context.user, layergroup, mapConfig.obj());
-                self.addAnalysesMetadata(req.context.user, layergroup, analysesResults, true);
-                addContextMetadata(layergroup, mapConfig.obj(), context);
-                res.set('X-Layergroup-Id', layergroup.layergroupid);
-                self.send(req, res, layergroup, 200);
             }
-        }
-    );
+        };
+
+        this.mapConfigAdapter.getMapConfig(user, requestMapConfig, res.locals, context, (err, requestMapConfig) => {
+            req.profiler.done('anonymous.getMapConfig');
+            if (err) {
+                return next(err);
+            }
+
+            req.body = requestMapConfig;
+            res.locals.context = context;
+
+            next();
+        });
+    }.bind(this);
 };
 
-function addContextMetadata(layergroup, mapConfig, context) {
-    if (layergroup.metadata && Array.isArray(layergroup.metadata.layers) && Array.isArray(mapConfig.layers)) {
-        layergroup.metadata.layers = layergroup.metadata.layers.map(function(layer, layerIndex) {
-            if (context.turboCarto && Array.isArray(context.turboCarto.layers)) {
-                layer.meta.cartocss_meta = context.turboCarto.layers[layerIndex];
+MapController.prototype.createLayergroup = function () {
+    return function createLayergroupMiddleware (req, res, next) {
+        const requestMapConfig = req.body;
+        const { context, user } = res.locals;
+        const datasource = context.datasource || Datasource.EmptyDatasource();
+        const mapconfig = new MapConfig(requestMapConfig, datasource);
+        const mapconfigProvider =
+            new CreateLayergroupMapConfigProvider(mapconfig, user, this.userLimitsApi, res.locals);
+
+        res.locals.mapconfig = mapconfig;
+        res.locals.analysesResults = context.analysesResults;
+
+        this.mapBackend.createLayergroup(mapconfig, res.locals, mapconfigProvider, (err, layergroup) => {
+            req.profiler.done('createLayergroup');
+            if (err) {
+                return next(err);
             }
-            return layer;
+
+            res.locals.layergroup = layergroup;
+
+            next();
         });
-    }
+    }.bind(this);
+};
+
+MapController.prototype.instantiateLayergroup = function () {
+    return function instantiateLayergroupMiddleware (req, res, next) {
+        const { user, mapconfig, rendererParams } = res.locals;
+        const mapconfigProvider =
+            new CreateLayergroupMapConfigProvider(mapconfig, user, this.userLimitsApi, rendererParams);
+
+        this.mapBackend.createLayergroup(mapconfig, rendererParams, mapconfigProvider, (err, layergroup) => {
+            req.profiler.done('createLayergroup');
+            if (err) {
+                return next(err);
+            }
+
+            res.locals.layergroup = layergroup;
+
+            const { mapconfigProvider } = res.locals;
+
+            res.locals.analysesResults = mapconfigProvider.analysesResults;
+            res.locals.template = mapconfigProvider.template;
+            res.locals.templateName = mapconfigProvider.getTemplateName();
+            res.locals.context = mapconfigProvider.context;
+
+            next();
+        });
+    }.bind(this);
+};
+
+MapController.prototype.incrementMapViewCount = function () {
+    return function incrementMapViewCountMiddleware(req, res, next) {
+        const { mapconfig, user } = res.locals;
+
+        // Error won't blow up, just be logged.
+        this.metadataBackend.incMapviewCount(user, mapconfig.obj().stat_tag, (err) => {
+            req.profiler.done('incMapviewCount');
+
+            if (err) {
+                global.logger.log(`ERROR: failed to increment mapview count for user '${user}': ${err.message}`);
+            }
+
+            next();
+        });
+    }.bind(this);
+};
+
+MapController.prototype.augmentLayergroupData = function () {
+    return function augmentLayergroupDataMiddleware (req, res, next) {
+        const { layergroup } = res.locals;
+
+        // include in layergroup response the variables in serverMedata
+        // those variables are useful to send to the client information
+        // about how to reach this server or information about it
+        _.extend(layergroup, global.environment.serverMetadata);
+
+        next();
+    };
+};
+
+function getTemplateUrl(url) {
+    return url.https || url.http;
 }
 
-MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn) {
-    var self = this;
-
-    var cdbuser = req.context.user;
-
-    var mapConfigProvider;
-    var mapConfig;
-    step(
-        function setupParams(){
-            self.req2params(req, this);
-        },
-        function getTemplateParams() {
-            prepareParamsFn(this);
-        },
-        function getTemplate(err, templateParams) {
-            assert.ifError(err);
-            mapConfigProvider = new NamedMapMapConfigProvider(
-                self.templateMaps,
-                self.pgConnection,
-                self.metadataBackend,
-                self.userLimitsApi,
-                self.mapConfigAdapter,
-                cdbuser,
-                req.params.template_id,
-                templateParams,
-                req.query.auth_token,
-                req.params
-            );
-            mapConfigProvider.getMapConfig(this);
-        },
-        function createLayergroup(err, mapConfig_, rendererParams) {
-            assert.ifError(err);
-            mapConfig = mapConfig_;
-            self.mapBackend.createLayergroup(
-                mapConfig, rendererParams,
-                new CreateLayergroupMapConfigProvider(mapConfig, cdbuser, self.userLimitsApi, rendererParams),
-                this
-            );
-        },
-        function afterLayergroupCreate(err, layergroup) {
-            assert.ifError(err);
-            self.afterLayergroupCreate(req, res, mapConfig, layergroup,
-                                       mapConfigProvider.analysesResults,
-                                       this);
-        },
-        function finishTemplateInstantiation(err, layergroup) {
-            if (err) {
-                self.sendError(req, res, err, 'NAMED MAP LAYERGROUP');
-            } else {
-                var templateHash = self.templateMaps.fingerPrint(mapConfigProvider.template).substring(0, 8);
-                layergroup.layergroupid = cdbuser + '@' + templateHash + '@' + layergroup.layergroupid;
-
-                var _mapConfig = mapConfig.obj();
-                self.addDataviewsAndWidgetsUrls(cdbuser, layergroup, _mapConfig);
-                self.addAnalysesMetadata(cdbuser, layergroup, mapConfigProvider.analysesResults);
-                addContextMetadata(layergroup, _mapConfig, mapConfigProvider.context);
-
-                res.set('X-Layergroup-Id', layergroup.layergroupid);
-                self.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(cdbuser, mapConfigProvider.getTemplateName()));
-
-                self.send(req, res, layergroup, 200);
-            }
-        }
-    );
-};
-
-MapController.prototype.afterLayergroupCreate =
-function(req, res, mapconfig, layergroup, analysesResults, callback) {
-    var self = this;
-
-    var username = req.context.user;
-
-    var tasksleft = 2; // redis key and affectedTables
-    var errors = [];
-
-    var done = function(err) {
-        if ( err ) {
-            errors.push('' + err);
-        }
-        if ( ! --tasksleft ) {
-            err = errors.length ? new Error(errors.join('\n')) : null;
-            callback(err, layergroup);
-        }
+function getTilejson(tiles, grids) {
+    const tilejson = {
+        tilejson: '2.2.0',
+        tiles: tiles.https || tiles.http
     };
 
-    // include in layergroup response the variables in serverMedata
-    // those variables are useful to send to the client information
-    // about how to reach this server or information about it
-    _.extend(layergroup, global.environment.serverMetadata);
+    if (grids) {
+        tilejson.grids = grids.https || grids.http;
+    }
 
-    // Don't wait for the mapview count increment to
-    // take place before proceeding. Error will be logged
-    // asynchronously
-    this.metadataBackend.incMapviewCount(username, mapconfig.obj().stat_tag, function(err) {
-        req.profiler.done('incMapviewCount');
-        if ( err ) {
-            global.logger.log("ERROR: failed to increment mapview count for user '" + username + "': " + err);
-        }
-        done();
-    });
+    return tilejson;
+}
 
-    var sql = [];
-    mapconfig.getLayers().forEach(function(layer) {
-        sql.push(layer.options.sql);
-        if (layer.options.affected_tables) {
-            layer.options.affected_tables.map(function(table) {
-                sql.push('SELECT * FROM ' + table + ' LIMIT 0');
-            });
-        }
-    });
+MapController.prototype.setTilejsonMetadataToLayergroup = function () {
+    return function augmentLayergroupTilejsonMiddleware (req, res, next) {
+        const { layergroup, user, mapconfig } = res.locals;
 
-    var dbName = req.params.dbname;
-    var layergroupId = layergroup.layergroupid;
-    var dbConnection;
-
-    step(
-        function getPgConnection() {
-            self.pgConnection.getConnection(username, this);
-        },
-        function getAffectedTablesAndLastUpdatedTime(err, connection) {
-            assert.ifError(err);
-            dbConnection = connection;
-            QueryTables.getAffectedTablesFromQuery(dbConnection, sql.join(';'), this);
-        },
-        function handleAffectedTablesAndLastUpdatedTime(err, result) {
-            req.profiler.done('queryTablesAndLastUpdated');
-            assert.ifError(err);
-            // feed affected tables cache so it can be reused from, for instance, layergroup controller
-            self.layergroupAffectedTables.set(dbName, layergroupId, result);
-
-            var lastUpdateTime = result.getLastUpdatedAt();
-            lastUpdateTime = getLastUpdatedTime(analysesResults, lastUpdateTime) || lastUpdateTime;
-
-            // last update for layergroup cache buster
-            layergroup.layergroupid = layergroup.layergroupid + ':' + lastUpdateTime;
-            layergroup.last_updated = new Date(lastUpdateTime).toISOString();
-
-            if (req.method === 'GET') {
-                var ttl = global.environment.varnish.layergroupTtl || 86400;
-                res.set('Cache-Control', 'public,max-age='+ttl+',must-revalidate');
-                res.set('Last-Modified', (new Date()).toUTCString());
-                res.set('X-Cache-Channel', result.getCacheChannel());
-                if (result.tables && result.tables.length > 0) {
-                    self.surrogateKeysCache.tag(res, result);
+        const isVectorOnlyMapConfig = mapconfig.isVectorOnlyMapConfig();
+        let hasMapnikLayers = false;
+        layergroup.metadata.layers.forEach((layerMetadata, index) => {
+            const layerId = mapconfig.getLayerId(index);
+            const rasterResource = `${layergroup.layergroupid}/${layerId}/{z}/{x}/{y}.png`;
+            if (mapconfig.layerType(index) === 'mapnik') {
+                hasMapnikLayers = true;
+                const vectorResource = `${layergroup.layergroupid}/${layerId}/{z}/{x}/{y}.mvt`;
+                const layerTilejson = {
+                    vector: getTilejson(this.resourceLocator.getTileUrls(user, vectorResource))
+                };
+                if (!isVectorOnlyMapConfig) {
+                    let grids = null;
+                    const layer = mapconfig.getLayer(index);
+                    if (layer.options.interactivity) {
+                        const gridResource = `${layergroup.layergroupid}/${layerId}/{z}/{x}/{y}.grid.json`;
+                        grids = this.resourceLocator.getTileUrls(user, gridResource);
+                    }
+                    layerTilejson.raster = getTilejson(
+                        this.resourceLocator.getTileUrls(user, rasterResource),
+                        grids
+                    );
                 }
+                layerMetadata.tilejson = layerTilejson;
+            } else {
+                layerMetadata.tilejson = {
+                    raster: getTilejson(this.resourceLocator.getTileUrls(user, rasterResource))
+                };
+            }
+        });
+
+        const tilejson = {};
+        const url = {};
+
+        if (hasMapnikLayers) {
+            const vectorResource = `${layergroup.layergroupid}/{z}/{x}/{y}.mvt`;
+            tilejson.vector = getTilejson(
+                this.resourceLocator.getTileUrls(user, vectorResource)
+            );
+            url.vector = getTemplateUrl(this.resourceLocator.getTemplateUrls(user, vectorResource));
+
+            if (!isVectorOnlyMapConfig) {
+                const rasterResource = `${layergroup.layergroupid}/{z}/{x}/{y}.png`;
+                tilejson.raster = getTilejson(
+                    this.resourceLocator.getTileUrls(user, rasterResource)
+                );
+                url.raster = getTemplateUrl(this.resourceLocator.getTemplateUrls(user, rasterResource));
+            }
+        }
+
+        layergroup.metadata.tilejson = tilejson;
+        layergroup.metadata.url = url;
+
+        next();
+    }.bind(this);
+};
+
+MapController.prototype.getAffectedTables = function () {
+    return function getAffectedTablesMiddleware (req, res, next) {
+        const { dbname, layergroup, user, mapconfig } = res.locals;
+
+        this.pgConnection.getConnection(user, (err, connection) => {
+            if (err) {
+                return next(err);
             }
 
-            return null;
-        },
-        function fetchLayersStats(err) {
-            assert.ifError(err);
-            var next = this;
-            self.statsBackend.getStats(mapconfig, dbConnection, function(err, layersStats) {
+            const sql = [];
+            mapconfig.getLayers().forEach(function(layer) {
+                sql.push(layer.options.sql);
+                if (layer.options.affected_tables) {
+                    layer.options.affected_tables.map(function(table) {
+                        sql.push('SELECT * FROM ' + table + ' LIMIT 0');
+                    });
+                }
+            });
+
+            QueryTables.getAffectedTablesFromQuery(connection, sql.join(';'), (err, affectedTables) => {
+                req.profiler.done('getAffectedTablesFromQuery');
                 if (err) {
                     return next(err);
                 }
-                if (layersStats.length > 0) {
-                    layergroup.metadata.layers.forEach(function (layer, index) {
-                        layer.meta.stats = layersStats[index];
-                    });
-                }
-                return next();
+
+                // feed affected tables cache so it can be reused from, for instance, layergroup controller
+                this.layergroupAffectedTables.set(dbname, layergroup.layergroupId, affectedTables);
+
+                res.locals.affectedTables = affectedTables;
+
+                next();
             });
-        },
-        function finish(err) {
-            done(err);
+        });
+    }.bind(this);
+};
+
+MapController.prototype.setCacheChannel = function () {
+    return function setCacheChannelMiddleware (req, res, next) {
+        const { affectedTables } = res.locals;
+
+        if (req.method === 'GET') {
+            res.set('X-Cache-Channel', affectedTables.getCacheChannel());
         }
-    );
+
+        next();
+    };
+};
+
+MapController.prototype.setLastModified = function () {
+    return function setLastModifiedMiddleware (req, res, next) {
+        if (req.method === 'GET') {
+            res.set('Last-Modified', (new Date()).toUTCString());
+        }
+
+        next();
+    };
+};
+
+MapController.prototype.setLastUpdatedTimeToLayergroup = function () {
+    return function setLastUpdatedTimeToLayergroupMiddleware (req, res, next) {
+        const { affectedTables, layergroup, analysesResults } = res.locals;
+
+        var lastUpdateTime = affectedTables.getLastUpdatedAt();
+
+        lastUpdateTime = getLastUpdatedTime(analysesResults, lastUpdateTime) || lastUpdateTime;
+
+        // last update for layergroup cache buster
+        layergroup.layergroupid = layergroup.layergroupid + ':' + lastUpdateTime;
+        layergroup.last_updated = new Date(lastUpdateTime).toISOString();
+
+        next();
+    };
 };
 
 function getLastUpdatedTime(analysesResults, lastUpdateTime) {
@@ -395,34 +479,66 @@ function getLastUpdatedTime(analysesResults, lastUpdateTime) {
     }, lastUpdateTime);
 }
 
-MapController.prototype.addAnalysesMetadata = function(username, layergroup, analysesResults, includeQuery) {
-    includeQuery = includeQuery || false;
-    analysesResults = analysesResults || [];
-    layergroup.metadata.analyses = [];
+MapController.prototype.setCacheControl = function () {
+    return function setCacheControlMiddleware (req, res, next) {
+        if (req.method === 'GET') {
+            var ttl = global.environment.varnish.layergroupTtl || 86400;
+            res.set('Cache-Control', 'public,max-age='+ttl+',must-revalidate');
+        }
 
-    analysesResults.forEach(function(analysis) {
-        var nodes = analysis.getNodes();
-        layergroup.metadata.analyses.push({
-            nodes: nodes.reduce(function(nodesIdMap, node) {
-                if (node.params.id) {
-                    var nodeResource = layergroup.layergroupid + '/analysis/node/' + node.id();
-                    var nodeRepr = {
-                        status: node.getStatus(),
-                        url: this.resourceLocator.getUrls(username, nodeResource)
-                    };
-                    if (includeQuery) {
-                        nodeRepr.query = node.getQuery();
-                    }
-                    if (node.getStatus() === 'failed') {
-                        nodeRepr.error_message = node.getErrorMessage();
-                    }
-                    nodesIdMap[node.params.id] = nodeRepr;
+        next();
+    };
+};
+
+MapController.prototype.setLayerStats = function () {
+    return function setLayerStatsMiddleware(req, res, next) {
+        const { user, mapconfig, layergroup } = res.locals;
+
+        this.pgConnection.getConnection(user, (err, connection) => {
+            if (err) {
+                return next(err);
+            }
+
+            this.statsBackend.getStats(mapconfig, connection, function(err, layersStats) {
+                if (err) {
+                    return next(err);
                 }
 
-                return nodesIdMap;
-            }.bind(this), {})
+                if (layersStats.length > 0) {
+                    layergroup.metadata.layers.forEach(function (layer, index) {
+                        layer.meta.stats = layersStats[index];
+                    });
+                }
+
+                next();
+            });
         });
-    }.bind(this));
+    }.bind(this);
+};
+
+MapController.prototype.setLayergroupIdHeader = function (useTemplateHash) {
+    return function setLayergroupIdHeaderMiddleware (req, res, next) {
+        const { layergroup, user, template } = res.locals;
+
+        if (useTemplateHash) {
+            var templateHash = this.templateMaps.fingerPrint(template).substring(0, 8);
+            layergroup.layergroupid = `${user}@${templateHash}@${layergroup.layergroupid}`;
+        }
+
+        res.set('X-Layergroup-Id', layergroup.layergroupid);
+
+        next();
+    }.bind(this);
+};
+
+MapController.prototype.setDataviewsAndWidgetsUrlsToLayergroupMetadata = function () {
+    return function setDataviewsAndWidgetsUrlsToLayergroupMetadataMiddleware (req, res, next) {
+        const { layergroup, user, mapconfig } = res.locals;
+
+        this.addDataviewsAndWidgetsUrls(user, layergroup, mapconfig.obj());
+
+        next();
+    }.bind(this);
 };
 
 // TODO this should take into account several URL patterns
@@ -461,3 +577,153 @@ MapController.prototype.addWidgetsUrl = function(username, layergroup, mapConfig
         }.bind(this));
     }
 };
+
+MapController.prototype.setAnalysesMetadataToLayergroup = function (includeQuery) {
+    return function setAnalysesMetadataToLayergroupMiddleware (req, res, next) {
+        const { layergroup, user, analysesResults = [] } = res.locals;
+
+        this.addAnalysesMetadata(user, layergroup, analysesResults, includeQuery);
+
+        next();
+    }.bind(this);
+};
+
+MapController.prototype.addAnalysesMetadata = function(username, layergroup, analysesResults, includeQuery) {
+    includeQuery = includeQuery || false;
+    analysesResults = analysesResults || [];
+    layergroup.metadata.analyses = [];
+
+    analysesResults.forEach(function(analysis) {
+        var nodes = analysis.getNodes();
+        layergroup.metadata.analyses.push({
+            nodes: nodes.reduce(function(nodesIdMap, node) {
+                if (node.params.id) {
+                    var nodeResource = layergroup.layergroupid + '/analysis/node/' + node.id();
+                    var nodeRepr = {
+                        status: node.getStatus(),
+                        url: this.resourceLocator.getUrls(username, nodeResource)
+                    };
+                    if (includeQuery) {
+                        nodeRepr.query = node.getQuery();
+                    }
+                    if (node.getStatus() === 'failed') {
+                        nodeRepr.error_message = node.getErrorMessage();
+                    }
+                    nodesIdMap[node.params.id] = nodeRepr;
+                }
+
+                return nodesIdMap;
+            }.bind(this), {})
+        });
+    }.bind(this));
+};
+
+MapController.prototype.setTurboCartoMetadataToLayergroup = function () {
+    return function setTurboCartoMetadataToLayergroupMiddleware (req, res, next) {
+        const { layergroup, mapconfig, context } = res.locals;
+
+        addTurboCartoContextMetadata(layergroup, mapconfig.obj(), context);
+
+        next();
+    };
+};
+
+function addTurboCartoContextMetadata(layergroup, mapConfig, context) {
+    if (layergroup.metadata && Array.isArray(layergroup.metadata.layers) && Array.isArray(mapConfig.layers)) {
+        layergroup.metadata.layers = layergroup.metadata.layers.map(function(layer, layerIndex) {
+            if (context.turboCarto && Array.isArray(context.turboCarto.layers)) {
+                layer.meta.cartocss_meta = context.turboCarto.layers[layerIndex];
+            }
+            return layer;
+        });
+    }
+}
+
+// TODO: see how evolve this function, it's a good candidate to be refactored
+MapController.prototype.setAggregationMetadataToLayergroup = function () {
+    return function setAggregationMetadataToLayergroupMiddleware (req, res, next) {
+        const { layergroup, mapconfig, context } = res.locals;
+
+        addAggregationContextMetadata(layergroup, mapconfig.obj(), context);
+
+        next();
+    };
+};
+
+function addAggregationContextMetadata(layergroup, mapConfig, context) {
+    if (layergroup.metadata && Array.isArray(layergroup.metadata.layers) && Array.isArray(mapConfig.layers)) {
+        layergroup.metadata.layers = layergroup.metadata.layers.map(function(layer, layerIndex) {
+            if (context.aggregation && Array.isArray(context.aggregation.layers)) {
+                layer.meta.aggregation = context.aggregation.layers[layerIndex];
+            }
+            return layer;
+        });
+    }
+}
+
+MapController.prototype.setSurrogateKeyHeader = function () {
+    return function setSurrogateKeyHeaderMiddleware(req, res, next) {
+        const { affectedTables, user, templateName } = res.locals;
+
+        if (req.method === 'GET' && affectedTables.tables && affectedTables.tables.length > 0) {
+            this.surrogateKeysCache.tag(res, affectedTables);
+        }
+
+        if (templateName) {
+            this.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, templateName));
+        }
+
+        next();
+    }.bind(this);
+};
+
+MapController.prototype.sendResponse = function () {
+    return function sendResponseMiddleware (req, res) {
+        req.profiler.done('res');
+        const { layergroup } = res.locals;
+
+        res.status(200);
+
+        if (req.query && req.query.callback) {
+            res.jsonp(layergroup);
+        } else {
+            res.json(layergroup);
+        }
+    };
+};
+
+MapController.prototype.augmentError = function (options) {
+    const { addContext = false, label = 'MAPS CONTROLLER' } = options;
+
+    return function augmentErrorMiddleware (err, req, res, next) {
+        req.profiler.done('error');
+        const { mapconfig } = res.locals;
+
+        if (addContext) {
+            err = Number.isFinite(err.layerIndex) ? populateError(err, mapconfig) : err;
+        }
+
+        err.label = label;
+
+        next(err);
+    };
+};
+
+function populateError(err, mapConfig) {
+    var error = new Error(err.message);
+    error.http_status = err.http_status;
+
+    if (!err.http_status && err.message.indexOf('column "the_geom_webmercator" does not exist') >= 0) {
+        error.http_status = 400;
+    }
+
+    error.type = 'layer';
+    error.subtype = err.message.indexOf('Postgis Plugin') >= 0 ? 'query' : undefined;
+    error.layer = {
+        id: mapConfig.getLayerId(err.layerIndex),
+        index: err.layerIndex,
+        type: mapConfig.layerType(err.layerIndex)
+    };
+
+    return error;
+}

lib/cartodb/controllers/named_maps.js

@@ -1,232 +1,10 @@
-var step = require('step');
-var assert = require('assert');
-var _ = require('underscore');
-var NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
+const NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
+const cors = require('../middleware/cors');
+const userMiddleware = require('../middleware/user');
+const allowQueryParams = require('../middleware/allow-query-params');
+const vectorError = require('../middleware/vector-error');
 
-var util = require('util');
-var BaseController = require('./base');
-
-var cors = require('../middleware/cors');
-var userMiddleware = require('../middleware/user');
-var allowQueryParams = require('../middleware/allow-query-params');
-
-function NamedMapsController(authApi, pgConnection, namedMapProviderCache, tileBackend, previewBackend,
-                             surrogateKeysCache, tablesExtentApi, metadataBackend) {
-    BaseController.call(this, authApi, pgConnection);
-
-    this.namedMapProviderCache = namedMapProviderCache;
-    this.tileBackend = tileBackend;
-    this.previewBackend = previewBackend;
-    this.surrogateKeysCache = surrogateKeysCache;
-    this.tablesExtentApi = tablesExtentApi;
-    this.metadataBackend = metadataBackend;
-}
-
-util.inherits(NamedMapsController, BaseController);
-
-module.exports = NamedMapsController;
-
-NamedMapsController.prototype.register = function(app) {
-    app.get(app.base_url_templated +
-        '/:template_id/:layer/:z/:x/:y.(:format)', cors(), userMiddleware,
-        this.tile.bind(this));
-
-    app.get(app.base_url_mapconfig +
-        '/static/named/:template_id/:width/:height.:format', cors(), userMiddleware, allowQueryParams(['layer']),
-        this.staticMap.bind(this));
-};
-
-NamedMapsController.prototype.sendResponse = function(req, res, resource, headers, namedMapProvider) {
-    this.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(req.context.user, namedMapProvider.getTemplateName()));
-    res.set('Content-Type', headers['content-type'] || headers['Content-Type'] || 'image/png');
-    res.set('Cache-Control', 'public,max-age=7200,must-revalidate');
-
-    var self = this;
-
-    step(
-        function getAffectedTablesAndLastUpdatedTime() {
-            namedMapProvider.getAffectedTablesAndLastUpdatedTime(this);
-        },
-        function sendResponse(err, result) {
-            req.profiler.done('affectedTables');
-            if (err) {
-                global.logger.log('ERROR generating cache channel: ' + err);
-            }
-            if (!result || !!result.tables) {
-                // we increase cache control as we can invalidate it
-                res.set('Cache-Control', 'public,max-age=31536000');
-
-                var lastModifiedDate;
-                if (Number.isFinite(result.lastUpdatedTime)) {
-                    lastModifiedDate = new Date(result.getLastUpdatedAt());
-                } else {
-                    lastModifiedDate = new Date();
-                }
-                res.set('Last-Modified', lastModifiedDate.toUTCString());
-
-                res.set('X-Cache-Channel', result.getCacheChannel());
-                if (result.tables.length > 0) {
-                    self.surrogateKeysCache.tag(res, result);
-                }
-            }
-            self.send(req, res, resource, 200);
-        }
-    );
-};
-
-NamedMapsController.prototype.tile = function(req, res) {
-    var self = this;
-
-    var cdbUser = req.context.user;
-
-    var namedMapProvider;
-    step(
-        function reqParams() {
-            self.req2params(req, this);
-        },
-        function getNamedMapProvider(err) {
-            assert.ifError(err);
-            self.namedMapProviderCache.get(
-                cdbUser,
-                req.params.template_id,
-                req.query.config,
-                req.query.auth_token,
-                req.params,
-                this
-            );
-        },
-        function getTile(err, _namedMapProvider) {
-            assert.ifError(err);
-            namedMapProvider = _namedMapProvider;
-            self.tileBackend.getTile(namedMapProvider, req.params, this);
-        },
-        function handleImage(err, tile, headers, stats) {
-            req.profiler.add(stats);
-            if (err) {
-                self.sendError(req, res, err, 'NAMED_MAP_TILE');
-            } else {
-                self.sendResponse(req, res, tile, headers, namedMapProvider);
-            }
-        }
-    );
-};
-
-NamedMapsController.prototype.staticMap = function(req, res) {
-    var self = this;
-
-    var cdbUser = req.context.user;
-
-    var format = req.params.format === 'jpg' ? 'jpeg' : 'png';
-    req.params.format = 'png';
-    req.params.layer = 'all';
-
-    var namedMapProvider;
-    step(
-        function reqParams() {
-            self.req2params(req, this);
-        },
-        function getNamedMapProvider(err) {
-            assert.ifError(err);
-            self.namedMapProviderCache.get(
-                cdbUser,
-                req.params.template_id,
-                req.query.config,
-                req.query.auth_token,
-                req.params,
-                this
-            );
-        },
-        function prepareLayerVisibility(err, _namedMapProvider) {
-            assert.ifError(err);
-
-            namedMapProvider = _namedMapProvider;
-
-            self.prepareLayerFilterFromPreviewLayers(cdbUser, req, namedMapProvider, this);
-        },
-        function prepareImageOptions(err) {
-            assert.ifError(err);
-            self.getStaticImageOptions(cdbUser, req.params, namedMapProvider, this);
-        },
-        function getImage(err, imageOpts) {
-            assert.ifError(err);
-
-            var width = +req.params.width;
-            var height = +req.params.height;
-
-            if (!_.isUndefined(imageOpts.zoom) && imageOpts.center) {
-                self.previewBackend.getImage(
-                    namedMapProvider, format, width, height, imageOpts.zoom, imageOpts.center, this);
-            } else {
-                self.previewBackend.getImage(
-                    namedMapProvider, format, width, height, imageOpts.bounds, this);
-            }
-        },
-        function incrementMapViews(err, image, headers, stats) {
-            assert.ifError(err);
-
-            var next = this;
-            namedMapProvider.getMapConfig(function(mapConfigErr, mapConfig) {
-                self.metadataBackend.incMapviewCount(cdbUser, mapConfig.obj().stat_tag, function(sErr) {
-                    if (err) {
-                        global.logger.log("ERROR: failed to increment mapview count for user '%s': %s", cdbUser, sErr);
-                    }
-                    next(err, image, headers, stats);
-                });
-            });
-        },
-        function handleImage(err, image, headers, stats) {
-            req.profiler.done('render-' + format);
-            req.profiler.add(stats || {});
-
-            if (err) {
-                self.sendError(req, res, err, 'STATIC_VIZ_MAP');
-            } else {
-                self.sendResponse(req, res, image, headers, namedMapProvider);
-            }
-        }
-    );
-};
-
-NamedMapsController.prototype.prepareLayerFilterFromPreviewLayers = function (user, req, namedMapProvider, callback) {
-    var self = this;
-    namedMapProvider.getTemplate(function (err, template) {
-        if (err) {
-            return callback(err);
-        }
-
-        if (!template || !template.view || !template.view.preview_layers) {
-           return callback();
-        }
-
-        var previewLayers = template.view.preview_layers;
-        var layerVisibilityFilter = [];
-
-        template.layergroup.layers.forEach(function (layer, index) {
-            if (previewLayers[''+index] !== false && previewLayers[layer.id] !== false) {
-                layerVisibilityFilter.push(''+index);
-            }
-        });
-
-        if (!layerVisibilityFilter.length) {
-            return callback();
-        }
-
-        // overwrites 'all' default filter
-        req.params.layer = layerVisibilityFilter.join(',');
-
-        // recreates the provider
-        self.namedMapProviderCache.get(
-            user,
-            req.params.template_id,
-            req.query.config,
-            req.query.auth_token,
-            req.params,
-            callback
-        );
-    });
-};
-
-var DEFAULT_ZOOM_CENTER = {
+const DEFAULT_ZOOM_CENTER = {
     zoom: 1,
     center: {
         lng: 0,
@@ -238,88 +16,368 @@ function numMapper(n) {
     return +n;
 }
 
-NamedMapsController.prototype.getStaticImageOptions = function(cdbUser, params, namedMapProvider, callback) {
-    var self = this;
+function getRequestParams(locals) {
+    const params = Object.assign({}, locals);
 
-    if ([params.zoom, params.lon, params.lat].map(numMapper).every(Number.isFinite)) {
-        return callback(null, {
-            zoom: params.zoom,
-            center: {
-                lng: params.lon,
-                lat: params.lat
-            }
-        });
-    }
+    delete params.template;
+    delete params.affectedTablesAndLastUpdate;
+    delete params.namedMapProvider;
+    delete params.allowedQueryParams;
 
-    if (params.bbox) {
-        var bbox = params.bbox.split(',').map(numMapper);
-        if (bbox.length === 4 && bbox.every(Number.isFinite)) {
-            return callback(null, {
-                bounds: {
-                    west: bbox[0],
-                    south: bbox[1],
-                    east: bbox[2],
-                    north: bbox[3]
-                }
-            });
-        }
-    }
+    return params;
+}
 
-    step(
-        function getTemplate() {
-            namedMapProvider.getTemplate(this);
-        },
-        function handleTemplateView(err, template) {
-            assert.ifError(err);
+function NamedMapsController(prepareContext, namedMapProviderCache, tileBackend, previewBackend,
+                             surrogateKeysCache, tablesExtentApi, metadataBackend) {
+    this.namedMapProviderCache = namedMapProviderCache;
+    this.tileBackend = tileBackend;
+    this.previewBackend = previewBackend;
+    this.surrogateKeysCache = surrogateKeysCache;
+    this.tablesExtentApi = tablesExtentApi;
+    this.metadataBackend = metadataBackend;
+    this.prepareContext = prepareContext;
+}
 
-            if (template.view) {
-                var zoomCenter = templateZoomCenter(template.view);
-                if (zoomCenter) {
-                    if (Number.isFinite(+params.zoom)) {
-                        zoomCenter.zoom = +params.zoom;
-                    }
-                    return zoomCenter;
-                }
+module.exports = NamedMapsController;
 
-                var bounds = templateBounds(template.view);
-                if (bounds) {
-                    return bounds;
-                }
-            }
+NamedMapsController.prototype.register = function(app) {
+    app.get(
+        app.base_url_templated + '/:template_id/:layer/:z/:x/:y.(:format)',
+        cors(),
+        userMiddleware,
+        this.prepareContext,
+        this.getNamedMapProvider('NAMED_MAP_TILE'),
+        this.getAffectedTables(),
+        this.getTile('NAMED_MAP_TILE'),
+        this.setSurrogateKey(),
+        this.setCacheChannelHeader(),
+        this.setLastModifiedHeader(),
+        this.setCacheControlHeader(),
+        this.setContentTypeHeader(),
+        this.respond(),
+        vectorError()
+    );
 
-            return false;
-        },
-        function estimateBoundsIfNoImageOpts(err, imageOpts) {
-            if (imageOpts) {
-                return imageOpts;
-            }
-
-            var next = this;
-            namedMapProvider.getAffectedTablesAndLastUpdatedTime(function(err, affectedTablesAndLastUpdate) {
-                if (err) {
-                    return next(null);
-                }
-
-                var affectedTables = affectedTablesAndLastUpdate.tables || [];
-
-                if (affectedTables.length === 0) {
-                    return next(null);
-                }
-
-                self.tablesExtentApi.getBounds(cdbUser, affectedTables, function(err, result) {
-                    return next(null, result);
-                });
-            });
-
-        },
-        function returnCallback(err, imageOpts) {
-            return callback(err, imageOpts || DEFAULT_ZOOM_CENTER);
-        }
+    app.get(
+        app.base_url_mapconfig + '/static/named/:template_id/:width/:height.:format',
+        cors(),
+        userMiddleware,
+        allowQueryParams(['layer', 'zoom', 'lon', 'lat', 'bbox']),
+        this.prepareContext,
+        this.getNamedMapProvider('STATIC_VIZ_MAP'),
+        this.getAffectedTables(),
+        this.getTemplate('STATIC_VIZ_MAP'),
+        this.prepareLayerFilterFromPreviewLayers('STATIC_VIZ_MAP'),
+        this.getStaticImageOptions(),
+        this.getImage('STATIC_VIZ_MAP'),
+        this.incrementMapViews(),
+        this.setSurrogateKey(),
+        this.setCacheChannelHeader(),
+        this.setLastModifiedHeader(),
+        this.setCacheControlHeader(),
+        this.setContentTypeHeader(),
+        this.respond()
     );
 };
 
+NamedMapsController.prototype.getNamedMapProvider = function (label) {
+    return function getNamedMapProviderMiddleware (req, res, next) {
+        const { user } = res.locals;
+        const { config, auth_token } = req.query;
+        const { template_id } = req.params;
+
+        // We force always the tile to be generated using PNG because
+        // is the only format we support by now
+        res.locals.format = 'png';
+        res.locals.layer = res.locals.layer || 'all';
+
+        const params = getRequestParams(res.locals);
+
+        this.namedMapProviderCache.get(user, template_id, config, auth_token, params, (err, namedMapProvider) => {
+            if (err) {
+                err.label = label;
+                return next(err);
+            }
+
+            res.locals.namedMapProvider = namedMapProvider;
+
+            next();
+        });
+    }.bind(this);
+};
+
+NamedMapsController.prototype.getAffectedTables = function () {
+    return function getAffectedTables (req, res, next) {
+        const { namedMapProvider } = res.locals;
+
+        namedMapProvider.getAffectedTablesAndLastUpdatedTime((err, affectedTablesAndLastUpdate) => {
+            req.profiler.done('affectedTables');
+
+            if (err) {
+                return next(err);
+            }
+
+            res.locals.affectedTablesAndLastUpdate = affectedTablesAndLastUpdate;
+
+            next();
+        });
+    }.bind(this);
+};
+
+NamedMapsController.prototype.getTemplate = function (label) {
+    return function getTemplateMiddleware (req, res, next) {
+        const { namedMapProvider } = res.locals;
+
+        namedMapProvider.getTemplate((err, template) => {
+            if (err) {
+                err.label = label;
+                return next(err);
+            }
+
+            res.locals.template = template;
+
+            next();
+        });
+    };
+};
+
+NamedMapsController.prototype.prepareLayerFilterFromPreviewLayers = function (label) {
+    return function prepareLayerFilterFromPreviewLayersMiddleware (req, res, next) {
+        const { user, template } = res.locals;
+        const { template_id } = req.params;
+        const { config, auth_token } = req.query;
+
+        if (!template || !template.view || !template.view.preview_layers) {
+            return next();
+        }
+
+        var previewLayers = template.view.preview_layers;
+        var layerVisibilityFilter = [];
+
+        template.layergroup.layers.forEach((layer, index) => {
+            if (previewLayers[''+index] !== false && previewLayers[layer.id] !== false) {
+                layerVisibilityFilter.push(''+index);
+            }
+        });
+
+        if (!layerVisibilityFilter.length) {
+            return next();
+        }
+
+        const params = getRequestParams(res.locals);
+
+        // overwrites 'all' default filter
+        params.layer = layerVisibilityFilter.join(',');
+
+        // recreates the provider
+        this.namedMapProviderCache.get(user, template_id, config, auth_token, params, (err, provider) => {
+            if (err) {
+                err.label = label;
+                return next(err);
+            }
+
+            res.locals.namedMapProvider = provider;
+
+            next();
+        });
+    }.bind(this);
+};
+
+NamedMapsController.prototype.getTile = function (label) {
+    return function getTileMiddleware (req, res, next) {
+        const { namedMapProvider } = res.locals;
+
+        this.tileBackend.getTile(namedMapProvider, req.params, (err, tile, headers, stats) => {
+            req.profiler.add(stats);
+
+            if (err) {
+                err.label = label;
+                return next(err);
+            }
+
+            res.locals.body = tile;
+            res.locals.headers = headers;
+            res.locals.stats = stats;
+
+            next();
+        });
+    }.bind(this);
+};
+
+NamedMapsController.prototype.getStaticImageOptions = function () {
+    return function getStaticImageOptionsMiddleware(req, res, next) {
+        const { user, namedMapProvider, template } = res.locals;
+
+        const imageOpts = getImageOptions(res.locals, template);
+
+        if (imageOpts) {
+            res.locals.imageOpts = imageOpts;
+            return next();
+        }
+
+        res.locals.imageOpts = DEFAULT_ZOOM_CENTER;
+
+        namedMapProvider.getAffectedTablesAndLastUpdatedTime((err, affectedTablesAndLastUpdate) => {
+            if (err) {
+                return next();
+            }
+
+            var affectedTables = affectedTablesAndLastUpdate.tables || [];
+
+            if (affectedTables.length === 0) {
+                return next();
+            }
+
+            this.tablesExtentApi.getBounds(user, affectedTables, (err, bounds) => {
+                if (err) {
+                    return next();
+                }
+
+                res.locals.imageOpts = bounds;
+
+                return next();
+            });
+        });
+    }.bind(this);
+};
+
+function getImageOptions (params, template) {
+    const { zoom, lon, lat, bbox } = params;
+
+    let imageOpts = getImageOptionsFromCoordinates(zoom, lon, lat);
+    if (imageOpts) {
+        return imageOpts;
+    }
+
+    imageOpts = getImageOptionsFromBoundingBox(bbox);
+    if (imageOpts) {
+        return imageOpts;
+    }
+
+    imageOpts = getImageOptionsFromTemplate(template, zoom);
+    if (imageOpts) {
+        return imageOpts;
+    }
+}
+
+function getImageOptionsFromCoordinates (zoom, lon, lat) {
+    if ([zoom, lon, lat].map(numMapper).every(Number.isFinite)) {
+        return {
+            zoom: zoom,
+            center: {
+                lng: lon,
+                lat: lat
+            }
+        };
+    }
+}
+
+
+function getImageOptionsFromTemplate (template, zoom) {
+    if (template.view) {
+        var zoomCenter = templateZoomCenter(template.view);
+        if (zoomCenter) {
+            if (Number.isFinite(+zoom)) {
+                zoomCenter.zoom = +zoom;
+            }
+
+            return zoomCenter;
+        }
+
+        var bounds = templateBounds(template.view);
+        if (bounds) {
+            return bounds;
+        }
+    }
+}
+
+function getImageOptionsFromBoundingBox (bbox = '') {
+    var _bbox = bbox.split(',').map(numMapper);
+
+    if (_bbox.length === 4 && _bbox.every(Number.isFinite)) {
+        return {
+            bounds: {
+                west: _bbox[0],
+                south: _bbox[1],
+                east: _bbox[2],
+                north: _bbox[3]
+            }
+        };
+    }
+}
+
+NamedMapsController.prototype.getImage = function (label) {
+    return function getImageMiddleware (req, res, next) {
+        const { imageOpts, namedMapProvider } = res.locals;
+        const { zoom, center, bounds } = imageOpts;
+
+        let { width, height } = req.params;
+
+        width = +width;
+        height = +height;
+
+        const format = req.params.format === 'jpg' ? 'jpeg' : 'png';
+
+        if (zoom !== undefined && center) {
+            return this.previewBackend.getImage(namedMapProvider, format, width, height, zoom, center,
+                (err, image, headers, stats) => {
+                if (err) {
+                    err.label = label;
+                    return next(err);
+                }
+
+                res.locals.body = image;
+                res.locals.headers = headers;
+                res.locals.stats = stats;
+
+                next();
+            });
+        }
+
+        this.previewBackend.getImage(namedMapProvider, format, width, height, bounds, (err, image, headers, stats) => {
+            if (err) {
+                err.label = label;
+                return next(err);
+            }
+
+            res.locals.body = image;
+            res.locals.headers = headers;
+            res.locals.stats = stats;
+
+            next();
+        });
+    }.bind(this);
+};
+
+function incrementMapViewsError (ctx) {
+    return `ERROR: failed to increment mapview count for user '${ctx.user}': ${ctx.err}`;
+}
+
+NamedMapsController.prototype.incrementMapViews = function () {
+    return function incrementMapViewsMiddleware(req, res, next) {
+        const { user, namedMapProvider } = res.locals;
+
+        namedMapProvider.getMapConfig((err, mapConfig) => {
+            if (err) {
+                global.logger.log(incrementMapViewsError({ user, err }));
+                return next();
+            }
+
+            const statTag = mapConfig.obj().stat_tag;
+
+            this.metadataBackend.incMapviewCount(user, statTag, (err) => {
+                if (err) {
+                    global.logger.log(incrementMapViewsError({ user, err }));
+                }
+
+                next();
+            });
+        });
+    }.bind(this);
+};
+
 function templateZoomCenter(view) {
-    if (!_.isUndefined(view.zoom) && view.center) {
+    if (view.zoom !== undefined && view.center) {
         return {
             zoom: view.zoom,
             center: view.center
@@ -330,9 +388,8 @@ function templateZoomCenter(view) {
 
 function templateBounds(view) {
     if (view.bounds) {
-        var hasAllBounds = _.every(['west', 'south', 'east', 'north'], function(prop) {
-            return Number.isFinite(view.bounds[prop]);
-        });
+        var hasAllBounds = ['west', 'south', 'east', 'north'].every(prop => Number.isFinite(view.bounds[prop]));
+
         if (hasAllBounds) {
             return {
                 bounds: {
@@ -348,3 +405,86 @@ function templateBounds(view) {
     }
     return false;
 }
+
+NamedMapsController.prototype.setCacheChannelHeader = function () {
+    return function setCacheChannelHeaderMiddleware (req, res, next) {
+        const { affectedTablesAndLastUpdate } = res.locals;
+
+        if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) {
+            res.set('X-Cache-Channel', affectedTablesAndLastUpdate.getCacheChannel());
+        }
+
+        next();
+    };
+};
+
+NamedMapsController.prototype.setSurrogateKey = function () {
+    return function setSurrogateKeyMiddleware(req, res, next) {
+        const { user, namedMapProvider, affectedTablesAndLastUpdate } = res.locals;
+
+        this.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, namedMapProvider.getTemplateName()));
+        if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) {
+            if (affectedTablesAndLastUpdate.tables.length > 0) {
+                this.surrogateKeysCache.tag(res, affectedTablesAndLastUpdate);
+            }
+        }
+
+        next();
+    }.bind(this);
+};
+
+NamedMapsController.prototype.setLastModifiedHeader = function () {
+    return function setLastModifiedHeaderMiddleware(req, res, next) {
+        const { affectedTablesAndLastUpdate } = res.locals;
+
+        if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) {
+            var lastModifiedDate;
+            if (Number.isFinite(affectedTablesAndLastUpdate.lastUpdatedTime)) {
+                lastModifiedDate = new Date(affectedTablesAndLastUpdate.getLastUpdatedAt());
+            } else {
+                lastModifiedDate = new Date();
+            }
+
+            res.set('Last-Modified', lastModifiedDate.toUTCString());
+        }
+
+        next();
+    };
+ };
+
+NamedMapsController.prototype.setCacheControlHeader = function () {
+    return function setCacheControlHeaderMiddleware(req, res, next) {
+        const { affectedTablesAndLastUpdate } = res.locals;
+
+        res.set('Cache-Control', 'public,max-age=7200,must-revalidate');
+
+        if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) {
+            // we increase cache control as we can invalidate it
+            res.set('Cache-Control', 'public,max-age=31536000');
+        }
+
+        next();
+    };
+ };
+
+NamedMapsController.prototype.setContentTypeHeader = function () {
+    return function setContentTypeHeaderMiddleware(req, res, next) {
+        const { headers = {} } = res.locals;
+
+        res.set('Content-Type', headers['content-type'] || headers['Content-Type'] || 'image/png');
+
+        next();
+    };
+};
+
+NamedMapsController.prototype.respond = function () {
+    return function respondMiddleware (req, res) {
+        const { body, stats = {}, format } = res.locals;
+
+        req.profiler.done('render-' + format);
+        req.profiler.add(stats);
+
+        res.status(200);
+        res.send(body);
+    };
+};

lib/cartodb/controllers/named_maps_admin.js

@@ -1,13 +1,6 @@
-var step = require('step');
-var assert = require('assert');
-var templateName = require('../backends/template_maps').templateName;
-
-var util = require('util');
-var BaseController = require('./base');
-
-var cors = require('../middleware/cors');
-var userMiddleware = require('../middleware/user');
-
+const { templateName } = require('../backends/template_maps');
+const cors = require('../middleware/cors');
+const userMiddleware = require('../middleware/user');
 
 /**
  * @param {AuthApi} authApi
@@ -15,183 +8,197 @@ var userMiddleware = require('../middleware/user');
  * @param {TemplateMaps} templateMaps
  * @constructor
  */
-function NamedMapsAdminController(authApi, pgConnection, templateMaps) {
-    BaseController.call(this, authApi, pgConnection);
-
+function NamedMapsAdminController(authApi, templateMaps) {
     this.authApi = authApi;
     this.templateMaps = templateMaps;
 }
 
-util.inherits(NamedMapsAdminController, BaseController);
-
 module.exports = NamedMapsAdminController;
 
-NamedMapsAdminController.prototype.register = function(app) {
-    app.post(app.base_url_templated, cors(), userMiddleware, this.create.bind(this));
-    app.put(app.base_url_templated + '/:template_id', cors(), userMiddleware, this.update.bind(this));
-    app.get(app.base_url_templated + '/:template_id', cors(), userMiddleware, this.retrieve.bind(this));
-    app.delete(app.base_url_templated + '/:template_id', cors(), userMiddleware, this.destroy.bind(this));
-    app.get(app.base_url_templated, cors(), userMiddleware, this.list.bind(this));
-    app.options(app.base_url_templated + '/:template_id', cors('Content-Type'));
-};
+NamedMapsAdminController.prototype.register = function (app) {
+    const { base_url_templated } = app;
 
-NamedMapsAdminController.prototype.create = function(req, res) {
-    var self = this;
+    app.post(
+        `${base_url_templated}/`,
+        cors(),
+        userMiddleware,
+        this.checkContentType('POST', 'POST TEMPLATE'),
+        this.authorizedByAPIKey('create', 'POST TEMPLATE'),
+        this.create()
+    );
 
-    var cdbuser = req.context.user;
+    app.put(
+        `${base_url_templated}/:template_id`,
+        cors(),
+        userMiddleware,
+        this.checkContentType('PUT', 'PUT TEMPLATE'),
+        this.authorizedByAPIKey('update', 'PUT TEMPLATE'),
+        this.update()
+    );
 
-    step(
-        function checkPerms(){
-            self.authApi.authorizedByAPIKey(cdbuser, req, this);
-        },
-        function addTemplate(err, authenticated) {
-            assert.ifError(err);
-            ifUnauthenticated(authenticated, 'Only authenticated users can get template maps');
-            ifInvalidContentType(req, 'template POST data must be of type application/json');
-            var cfg = req.body;
-            self.templateMaps.addTemplate(cdbuser, cfg, this);
-        },
-        function prepareResponse(err, tpl_id){
-            assert.ifError(err);
-            return { template_id: tpl_id };
-        },
-        finishFn(self, req, res, 'POST TEMPLATE')
+    app.get(
+        `${base_url_templated}/:template_id`,
+        cors(),
+        userMiddleware,
+        this.authorizedByAPIKey('get', 'GET TEMPLATE'),
+        this.retrieve()
+    );
+
+    app.delete(
+        `${base_url_templated}/:template_id`,
+        cors(),
+        userMiddleware,
+        this.authorizedByAPIKey('delete', 'DELETE TEMPLATE'),
+        this.destroy()
+    );
+
+    app.get(
+        `${base_url_templated}/`,
+        cors(),
+        userMiddleware,
+        this.authorizedByAPIKey('list', 'GET TEMPLATE LIST'),
+        this.list()
+    );
+
+    app.options(
+        `${base_url_templated}/:template_id`,
+        cors('Content-Type')
     );
 };
 
-NamedMapsAdminController.prototype.update = function(req, res) {
-    var self = this;
+NamedMapsAdminController.prototype.authorizedByAPIKey = function (action, label) {
+    return function authorizedByAPIKeyMiddleware (req, res, next) {
+        const { user } = res.locals;
 
-    var cdbuser = req.context.user;
-    var template;
-    var tpl_id;
+        this.authApi.authorizedByAPIKey(user, req, (err, authenticated) => {
+            if (err) {
+                return next(err);
+            }
 
-    step(
-        function checkPerms(){
-            self.authApi.authorizedByAPIKey(cdbuser, req, this);
-        },
-        function updateTemplate(err, authenticated) {
-            assert.ifError(err);
-            ifUnauthenticated(authenticated, 'Only authenticated user can update templated maps');
-            ifInvalidContentType(req, 'template PUT data must be of type application/json');
+            if (!authenticated) {
+                const error = new Error(`Only authenticated user can ${action} templated maps`);
+                error.http_status = 403;
+                error.label = label;
+                return next(error);
+            }
 
-            template = req.body;
-            tpl_id = templateName(req.params.template_id);
-            self.templateMaps.updTemplate(cdbuser, tpl_id, template, this);
-        },
-        function prepareResponse(err){
-            assert.ifError(err);
-
-            return { template_id: tpl_id };
-        },
-        finishFn(self, req, res, 'PUT TEMPLATE')
-    );
+            next();
+        });
+    }.bind(this);
 };
 
-NamedMapsAdminController.prototype.retrieve = function(req, res) {
-    var self = this;
+NamedMapsAdminController.prototype.checkContentType = function (action, label) {
+    return function checkContentTypeMiddleware (req, res, next) {
+        if (!req.is('application/json')) {
+            const error = new Error(`template ${action} data must be of type application/json`);
+            error.label = label;
+            return next(error);
+        }
+        next();
+    };
+};
 
-    req.profiler.start('windshaft-cartodb.get_template');
+NamedMapsAdminController.prototype.create = function () {
+    return function createTemplateMiddleware (req, res, next) {
+        const { user } = res.locals;
+        const template = req.body;
 
-    var cdbuser = req.context.user;
-    var tpl_id;
-    step(
-        function checkPerms(){
-            self.authApi.authorizedByAPIKey(cdbuser, req, this);
-        },
-        function getTemplate(err, authenticated) {
-            assert.ifError(err);
-            ifUnauthenticated(authenticated, 'Only authenticated users can get template maps');
+        this.templateMaps.addTemplate(user, template, (err, templateId) => {
+            if (err) {
+                return next(err);
+            }
 
-            tpl_id = templateName(req.params.template_id);
-            self.templateMaps.getTemplate(cdbuser, tpl_id, this);
-        },
-        function prepareResponse(err, tpl_val) {
-            assert.ifError(err);
-            if ( ! tpl_val ) {
-                err = new Error("Cannot find template '" + tpl_id + "' of user '" + cdbuser + "'");
-                err.http_status = 404;
-                throw err;
+            res.status(200);
+
+            const method = req.query.callback ? 'jsonp' : 'json';
+            res[method]({ template_id: templateId });
+        });
+    }.bind(this);
+};
+
+NamedMapsAdminController.prototype.update = function () {
+    return function updateTemplateMiddleware (req, res, next) {
+        const { user } = res.locals;
+        const template = req.body;
+        const templateId = templateName(req.params.template_id);
+
+        this.templateMaps.updTemplate(user, templateId, template, (err) => {
+            if (err) {
+                return next(err);
+            }
+
+            res.status(200);
+
+            const method = req.query.callback ? 'jsonp' : 'json';
+            res[method]({ template_id: templateId });
+        });
+    }.bind(this);
+};
+
+NamedMapsAdminController.prototype.retrieve =  function () {
+    return function retrieveTemplateMiddleware (req, res, next) {
+        req.profiler.start('windshaft-cartodb.get_template');
+
+        const { user } = res.locals;
+        const templateId = templateName(req.params.template_id);
+
+        this.templateMaps.getTemplate(user, templateId, (err, template) => {
+            if (err) {
+                return next(err);
+            }
+
+            if (!template) {
+                const error = new Error(`Cannot find template '${templateId}' of user '${user}'`);
+                error.http_status = 404;
+                return next(error);
             }
             // auth_id was added by ourselves,
             // so we remove it before returning to the user
-            delete tpl_val.auth_id;
-            return { template: tpl_val };
-        },
-        finishFn(self, req, res, 'GET TEMPLATE')
-    );
+            delete template.auth_id;
+
+            res.status(200);
+
+            const method = req.query.callback ? 'jsonp' : 'json';
+            res[method]({ template });
+        });
+    }.bind(this);
 };
 
-NamedMapsAdminController.prototype.destroy = function(req, res) {
-    var self = this;
+NamedMapsAdminController.prototype.destroy = function () {
+    return function destroyTemplateMiddleware (req, res, next) {
+        req.profiler.start('windshaft-cartodb.delete_template');
 
-    req.profiler.start('windshaft-cartodb.delete_template');
+        const { user } = res.locals;
+        const templateId = templateName(req.params.template_id);
 
-    var cdbuser = req.context.user;
-    var tpl_id;
-    step(
-        function checkPerms(){
-            self.authApi.authorizedByAPIKey(cdbuser, req, this);
-        },
-        function deleteTemplate(err, authenticated) {
-            assert.ifError(err);
-            ifUnauthenticated(authenticated, 'Only authenticated users can delete template maps');
+        this.templateMaps.delTemplate(user, templateId, (err/* , tpl_val */) => {
+            if (err) {
+                return next(err);
+            }
 
-            tpl_id = templateName(req.params.template_id);
-            self.templateMaps.delTemplate(cdbuser, tpl_id, this);
-        },
-        function prepareResponse(err/*, tpl_val*/){
-            assert.ifError(err);
-            return '';
-        },
-        finishFn(self, req, res, 'DELETE TEMPLATE', 204)
-    );
+            res.status(204);
+
+            const method = req.query.callback ? 'jsonp' : 'json';
+            res[method]('');
+        });
+    }.bind(this);
 };
 
-NamedMapsAdminController.prototype.list = function(req, res) {
-    var self = this;
-    req.profiler.start('windshaft-cartodb.get_template_list');
+NamedMapsAdminController.prototype.list = function () {
+    return function listTemplatesMiddleware (req, res, next) {
+        req.profiler.start('windshaft-cartodb.get_template_list');
 
-    var cdbuser = req.context.user;
+        const { user } = res.locals;
 
-    step(
-        function checkPerms(){
-            self.authApi.authorizedByAPIKey(cdbuser, req, this);
-        },
-        function listTemplates(err, authenticated) {
-            assert.ifError(err);
-            ifUnauthenticated(authenticated, 'Only authenticated user can list templated maps');
+        this.templateMaps.listTemplates(user, (err, templateIds) => {
+            if (err) {
+                return next(err);
+            }
 
-            self.templateMaps.listTemplates(cdbuser, this);
-        },
-        function prepareResponse(err, tpl_ids){
-            assert.ifError(err);
-            return { template_ids: tpl_ids };
-        },
-        finishFn(self, req, res, 'GET TEMPLATE LIST')
-    );
+            res.status(200);
+
+            const method = req.query.callback ? 'jsonp' : 'json';
+            res[method]({ template_ids: templateIds });
+        });
+    }.bind(this);
 };
-
-function finishFn(controller, req, res, description, status) {
-    return function finish(err, response){
-        if (err) {
-            controller.sendError(req, res, err, description);
-        } else {
-            controller.send(req, res, response, status || 200);
-        }
-    };
-}
-
-function ifUnauthenticated(authenticated, description) {
-    if (!authenticated) {
-        var err = new Error(description);
-        err.http_status = 403;
-        throw err;
-    }
-}
-
-function ifInvalidContentType(req, description) {
-    if (!req.is('application/json')) {
-        throw new Error(description);
-    }
-}

lib/cartodb/middleware/allow-query-params.js

@@ -3,7 +3,7 @@ module.exports = function allowQueryParams(params) {
         throw new Error('allowQueryParams must receive an Array of params');
     }
     return function allowQueryParamsMiddleware(req, res, next) {
-        req.context.allowedQueryParams = params;
+        res.locals.allowedQueryParams = params;
         next();
     };
 };

lib/cartodb/middleware/context/authorize.js

@@ -0,0 +1,20 @@
+module.exports = function authorizeMiddleware (authApi) {
+    return function (req, res, next) {
+        req.profiler.done('req2params.setup');
+
+        authApi.authorize(req, res, (err, authorized) => {
+            req.profiler.done('authorize');
+            if (err) {
+                return next(err);
+            }
+
+            if(!authorized) {
+                err = new Error("Sorry, you are unauthorized (permission denied)");
+                err.http_status = 403;
+                return next(err);
+            }
+
+            return next();
+        });
+    };
+};

lib/cartodb/middleware/context/clean-up-query-params.js

@@ -0,0 +1,32 @@
+const _ = require('underscore');
+
+// Whitelist query parameters and attach format
+const REQUEST_QUERY_PARAMS_WHITELIST = [
+    'config',
+    'map_key',
+    'api_key',
+    'auth_token',
+    'callback',
+    'zoom',
+    'lon',
+    'lat',
+    // analysis
+    'filters' // json
+];
+
+module.exports = function cleanUpQueryParamsMiddleware () {
+    return function cleanUpQueryParams (req, res, next) {
+        var allowedQueryParams = REQUEST_QUERY_PARAMS_WHITELIST;
+
+        if (Array.isArray(res.locals.allowedQueryParams)) {
+            allowedQueryParams = allowedQueryParams.concat(res.locals.allowedQueryParams);
+        }
+
+        req.query = _.pick(req.query, allowedQueryParams);
+
+        // bring all query values onto res.locals object
+        _.extend(res.locals, req.query);
+        
+        next();
+    };
+};

lib/cartodb/middleware/context/db-conn-setup.js

@@ -0,0 +1,31 @@
+const _ = require('underscore');
+
+module.exports = function dbConnSetupMiddleware(pgConnection) {
+    return function dbConnSetup(req, res, next) {
+        const user = res.locals.user;
+        pgConnection.setDBConn(user, res.locals, (err) => {
+            if (err) {
+                if (err.message && -1 !== err.message.indexOf('name not found')) {
+                    err.http_status = 404;
+                }
+                req.profiler.done('req2params');
+                return next(err);
+            }
+
+            // Add default database connection parameters
+            // if none given
+            _.defaults(res.locals, {
+                dbuser: global.environment.postgres.user,
+                dbpassword: global.environment.postgres.password,
+                dbhost: global.environment.postgres.host,
+                dbport: global.environment.postgres.port
+            });
+            
+            res.set('X-Served-By-DB-Host', res.locals.dbhost);
+
+            req.profiler.done('req2params');
+            
+            next(null);
+        });
+    };
+};

lib/cartodb/middleware/context/index.js

@@ -0,0 +1,15 @@
+const locals = require('./locals');
+const cleanUpQueryParams = require('./clean-up-query-params');
+const layergroupToken = require('./layergroup-token');
+const authorize = require('./authorize');
+const dbConnSetup = require('./db-conn-setup');
+
+module.exports = function prepareContextMiddleware(authApi, pgConnection) {
+    return [
+        locals,
+        cleanUpQueryParams(),
+        layergroupToken,
+        authorize(authApi),
+        dbConnSetup(pgConnection)
+    ];
+};

lib/cartodb/middleware/context/layergroup-token.js

@@ -0,0 +1,32 @@
+var LayergroupToken = require('../../models/layergroup-token');
+
+module.exports = function layergroupTokenMiddleware(req, res, next) {
+    if (!res.locals.token) {
+        return next();
+    }
+
+    var user = res.locals.user;
+
+    var layergroupToken = LayergroupToken.parse(res.locals.token);
+    res.locals.token = layergroupToken.token;
+    res.locals.cache_buster = layergroupToken.cacheBuster;
+
+    if (layergroupToken.signer) {
+        res.locals.signer = layergroupToken.signer;
+        if (!res.locals.signer) {
+            res.locals.signer = user;
+        } else if (res.locals.signer !== user) {
+            var err = new Error(`Cannot use map signature of user "${res.locals.signer}" on db of user "${user}"`);
+            err.type = 'auth';
+            err.http_status = 403;
+            if (req.query && req.query.callback) {
+                err.http_status = 200;
+            }
+            
+            req.profiler.done('req2params');
+            return next(err);
+        }
+    }
+
+    return next();
+};

lib/cartodb/middleware/context/locals.js

@@ -0,0 +1,6 @@
+module.exports = function localsMiddleware(req, res, next) {
+    // save req.params in res.locals
+    res.locals = Object.assign(req.params || {}, res.locals);
+
+    next();
+};

lib/cartodb/middleware/error-middleware.js

@@ -0,0 +1,213 @@
+const _ = require('underscore');
+const debug = require('debug')('windshaft:cartodb:error-middleware');
+
+module.exports = function errorMiddleware (/* options */) {
+    return function error (err, req, res, next) {
+        // jshint unused:false
+        // jshint maxcomplexity:9
+        var allErrors = Array.isArray(err) ? err : [err];
+
+        allErrors = populateTimeoutErrors(allErrors);
+
+        const label = err.label || 'UNKNOWN';
+        err = allErrors[0] || new Error(label);
+        allErrors[0] = err;
+
+        var statusCode = findStatusCode(err);
+
+        if (err.message === 'Tile does not exist' && res.locals.format === 'mvt') {
+            statusCode = 204;
+        }
+        
+        setErrorHeader(allErrors, statusCode, res);
+        debug('[%s ERROR] -- %d: %s, %s', label, statusCode, err, err.stack);
+
+        // If a callback was requested, force status to 200
+        if (req.query && req.query.callback) {
+            statusCode = 200;
+        }
+
+        var errorResponseBody = {
+            errors: allErrors.map(errorMessage),
+            errors_with_context: allErrors.map(errorMessageWithContext)
+        };
+
+        res.status(statusCode);
+
+        if (req.query && req.query.callback) {
+            res.jsonp(errorResponseBody);
+        } else {
+            res.json(errorResponseBody);
+        }
+    };
+};
+
+function isRenderTimeoutError (err) {
+    return err.message === 'Render timed out';
+}
+
+function isDatasourceTimeoutError (err) {
+    return err.message && err.message.match(/canceling statement due to statement timeout/i);
+}
+
+function isTimeoutError (err) {
+    return isRenderTimeoutError(err) || isDatasourceTimeoutError(err);
+}
+
+function populateTimeoutErrors (errors) {
+    return errors.map(function (error) {
+        if (isRenderTimeoutError(error)) {
+            error.subtype = 'render';
+        }
+
+        if (isDatasourceTimeoutError(error)) {
+            error.subtype = 'datasource';
+        }
+
+        if (isTimeoutError(error)) {
+            error.message = 'You are over platform\'s limits. Please contact us to know more details';
+            error.type = 'limit';
+            error.http_status = 429;
+        }
+
+        return error;
+    });
+}
+
+function findStatusCode(err) {
+    var statusCode;
+    if ( err.http_status ) {
+        statusCode = err.http_status;
+    } else {
+        statusCode = statusFromErrorMessage('' + err);
+    }
+    return statusCode;
+}
+
+module.exports.findStatusCode = findStatusCode;
+
+function statusFromErrorMessage(errMsg) {
+    // Find an appropriate statusCode based on message
+    // jshint maxcomplexity:7
+    var statusCode = 400;
+    if ( -1 !== errMsg.indexOf('permission denied') ) {
+        statusCode = 403;
+    }
+    else if ( -1 !== errMsg.indexOf('authentication failed') ) {
+        statusCode = 403;
+    }
+    else if (errMsg.match(/Postgis Plugin.*[\s|\n].*column.*does not exist/)) {
+        statusCode = 400;
+    }
+    else if ( -1 !== errMsg.indexOf('does not exist') ) {
+        if ( -1 !== errMsg.indexOf(' role ') ) {
+            statusCode = 403; // role 'xxx' does not exist
+        } else if ( errMsg.match(/function .* does not exist/) ) {
+            statusCode = 400; // invalid SQL (SQL function does not exist)
+        } else {
+            statusCode = 404;
+        }
+    }
+
+
+    return statusCode;
+}
+
+function errorMessage(err) {
+    // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
+    var message = (_.isString(err) ? err : err.message) || 'Unknown error';
+
+    return stripConnectionInfo(message);
+}
+
+module.exports.errorMessage = errorMessage;
+
+function stripConnectionInfo(message) {
+    // Strip connection info, if any
+    return message
+        // See https://github.com/CartoDB/Windshaft/issues/173
+        .replace(/Connection string: '[^']*'\n\s/im, '')
+        // See https://travis-ci.org/CartoDB/Windshaft/jobs/20703062#L1644
+        .replace(/is the server.*encountered/im, 'encountered');
+}
+
+var ERROR_INFO_TO_EXPOSE = {
+    message: true,
+    layer: true,
+    type: true,
+    analysis: true,
+    subtype: true
+};
+
+function shouldBeExposed (prop) {
+    return !!ERROR_INFO_TO_EXPOSE[prop];
+}
+
+function errorMessageWithContext(err) {
+    // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
+    var message = (_.isString(err) ? err : err.message) || 'Unknown error';
+
+    var error = {
+        type: err.type || 'unknown',
+        message: stripConnectionInfo(message),
+    };
+
+    for (var prop in err) {
+        // type & message are properties from Error's prototype and will be skipped
+        if (err.hasOwnProperty(prop) && shouldBeExposed(prop)) {
+            error[prop] = err[prop];
+        }
+    }
+
+    return error;
+}
+
+function setErrorHeader(errors, statusCode, res) {
+    let errorsCopy = errors.slice(0);
+    const mainError = errorsCopy.shift();
+
+    let errorsLog = {
+        mainError: {
+            statusCode: statusCode || 200,
+            message:    mainError.message,
+            name:       mainError.name,
+            label:      mainError.label,
+            type:       mainError.type,
+            subtype:    mainError.subtype
+        }
+    };
+
+    errorsLog.moreErrors = errorsCopy.map(error => {
+        return {
+            message: error.message,
+            name:    error.name,
+            label:   error.label,
+            type:    error.type,
+            subtype: error.subtype
+        };
+    });
+    
+    res.set('X-Tiler-Errors', stringifyForLogs(errorsLog));
+}
+
+/**
+ * Remove problematic nested characters 
+ * from object for logs RegEx
+ * 
+ * @param {Object} object 
+ */
+function stringifyForLogs(object) {
+    Object.keys(object).map(key => {
+        if(typeof object[key] === 'string') {
+            object[key] = object[key].replace(/[^a-zA-Z0-9]/g, ' ');
+        } else if (typeof object[key] === 'object') {
+            stringifyForLogs(object[key]);
+        } else if (object[key] instanceof Array) {
+            for (let element of object[key]) {
+                stringifyForLogs(element);
+            }
+        }
+    });
+
+    return JSON.stringify(object);
+}

lib/cartodb/middleware/lzma.js

@@ -0,0 +1,30 @@
+'use strict';
+
+const LZMA = require('lzma').LZMA;
+
+const lzmaWorker = new LZMA();
+
+module.exports = function lzmaMiddleware(req, res, next) {
+    if (!req.query.hasOwnProperty('lzma')) {
+        return next();
+    }
+
+    // Decode (from base64)
+    var lzma = new Buffer(req.query.lzma, 'base64')
+        .toString('binary')
+        .split('')
+        .map(function(c) {
+            return c.charCodeAt(0) - 128;
+        });
+
+    // Decompress
+    lzmaWorker.decompress(lzma, function(result) {
+        try {
+            delete req.query.lzma;
+            Object.assign(req.query, JSON.parse(result));
+            next();
+        } catch (err) {
+            next(new Error('Error parsing lzma as JSON: ' + err));
+        }
+    });
+};

lib/cartodb/middleware/stats.js

@@ -0,0 +1,27 @@
+const Profiler = require('../stats/profiler_proxy');
+const debug = require('debug')('windshaft:cartodb:stats');
+const onHeaders = require('on-headers');
+
+module.exports = function statsMiddleware(options) {
+    const { enabled = true, statsClient } = options;
+
+    return function stats(req, res, next) {
+        req.profiler = new Profiler({
+            statsd_client: statsClient,
+            profile: enabled
+        });
+
+        onHeaders(res, () => res.set('X-Tiler-Profiler', req.profiler.toJSONString()));
+
+        res.on('finish', () => {
+            try {
+                // May throw due to dns, see: http://github.com/CartoDB/Windshaft/issues/166
+                req.profiler.sendStats();
+            } catch (err) {
+                debug("error sending profiling stats: " + err);
+            }
+        });
+
+        next();
+    };
+};

lib/cartodb/middleware/user.js

@@ -2,6 +2,7 @@ var CdbRequest = require('../models/cdb_request');
 var cdbRequest = new CdbRequest();
 
 module.exports = function userMiddleware(req, res, next) {
-    req.context.user = cdbRequest.userByReq(req);
+    res.locals.user = cdbRequest.userByReq(req);
+
     next();
 };

lib/cartodb/middleware/vector-error.js

@@ -0,0 +1,30 @@
+const fs = require('fs');
+
+const timeoutErrorVectorTile = fs.readFileSync(__dirname + '/../../../assets/render-timeout-fallback.mvt');
+
+module.exports = function vectorError() {
+    return function vectorErrorMiddleware(err, req, res, next) {
+        if(req.params.format === 'mvt') {
+
+            if (isTimeoutError(err)) {
+                res.set('Content-Type', 'application/x-protobuf');
+                return res.status(429).send(timeoutErrorVectorTile);
+            }
+        }
+
+        next(err);
+    };
+};
+
+
+function isRenderTimeoutError (err) {
+    return err.message === 'Render timed out';
+}
+
+function isDatasourceTimeoutError (err) {
+    return err.message && err.message.match(/canceling statement due to statement timeout/i);
+}
+
+function isTimeoutError (err) {
+    return isRenderTimeoutError(err) || isDatasourceTimeoutError(err);
+}

lib/cartodb/models/aggregation/aggregation-mapconfig.js

@@ -0,0 +1,233 @@
+const MapConfig = require('windshaft').model.MapConfig;
+const aggregationQuery = require('./aggregation-query');
+const aggregationValidator = require('./aggregation-validator');
+const {
+    createPositiveNumberValidator,
+    createIncludesValueValidator,
+    createAggregationColumnsValidator
+} = aggregationValidator;
+
+const SubstitutionTokens = require('../../utils/substitution-tokens');
+
+const removeDuplicates = arr => [...new Set(arr)];
+
+function prepareSql(sql) {
+    return sql && SubstitutionTokens.replace(sql, {
+        bbox: 'ST_MakeEnvelope(0,0,0,0)',
+        scale_denominator: '0',
+        pixel_width: '1',
+        pixel_height: '1'
+    });
+}
+
+module.exports = class AggregationMapConfig extends MapConfig {
+    static get AGGREGATIONS () {
+        return aggregationQuery.SUPPORTED_AGGREGATE_FUNCTIONS;
+    }
+
+    static get PLACEMENTS () {
+        return aggregationQuery.SUPPORTED_PLACEMENTS;
+    }
+
+    static get THRESHOLD () {
+        return 1e5; // 100K
+    }
+
+    static get RESOLUTION () {
+        return 1;
+    }
+
+    static get SUPPORTED_GEOMETRY_TYPES () {
+        return [
+            'ST_Point'
+        ];
+    }
+
+    static supportsGeometryType(geometryType) {
+        return AggregationMapConfig.SUPPORTED_GEOMETRY_TYPES.includes(geometryType);
+    }
+
+    static  getAggregationGeometryColumn() {
+        return aggregationQuery.GEOMETRY_COLUMN;
+    }
+
+    constructor (user, config, connection, datasource) {
+        super(config, datasource);
+
+        const validate = aggregationValidator(this);
+        const positiveNumberValidator = createPositiveNumberValidator(this);
+        const includesValidPlacementsValidator = createIncludesValueValidator(this, AggregationMapConfig.PLACEMENTS);
+        const aggregationColumnsValidator = createAggregationColumnsValidator(this, AggregationMapConfig.AGGREGATIONS);
+
+        validate('resolution', positiveNumberValidator);
+        validate('placement', includesValidPlacementsValidator);
+        validate('threshold', positiveNumberValidator);
+        validate('columns', aggregationColumnsValidator);
+
+        this.user = user;
+        this.pgConnection = connection;
+    }
+
+    getAggregatedQuery (index) {
+        const { sql_raw, sql } = this.getLayer(index).options;
+        const {
+            // The default aggregation has no placement, columns or dimensions;
+            // this enables the special "full-sample" aggregation.
+            resolution = AggregationMapConfig.RESOLUTION,
+            threshold = AggregationMapConfig.THRESHOLD,
+            placement,
+            columns = {},
+            dimensions = {}
+        } = this.getAggregation(index);
+
+        return aggregationQuery({
+            query: sql_raw || sql,
+            resolution,
+            threshold,
+            placement,
+            columns,
+            dimensions,
+            isDefaultAggregation: this._isDefaultLayerAggregation(index)
+        });
+    }
+
+    isAggregationMapConfig () {
+        return this.isVectorOnlyMapConfig() || this.hasAnyLayerAggregation();
+    }
+
+    isAggregationLayer (index) {
+        return this.isVectorOnlyMapConfig() || this.hasLayerAggregation(index);
+    }
+
+    hasAnyLayerAggregation () {
+        const layers = this.getLayers();
+
+        for (let index = 0; index < layers.length; index++) {
+            if (this.hasLayerAggregation(index)) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    hasLayerAggregation (index) {
+        const layer = this.getLayer(index);
+        const { aggregation } = layer.options;
+
+        return aggregation !== undefined && (typeof aggregation === 'object' || typeof aggregation === 'boolean');
+    }
+
+    getAggregation (index) {
+        if (this.isVectorOnlyMapConfig() && !this.hasLayerAggregation(index)) {
+            return {};
+        }
+
+        const { aggregation } = this.getLayer(index).options;
+
+        if (typeof aggregation === 'boolean') {
+            return {};
+        }
+
+        return aggregation;
+    }
+
+    getLayerAggregationColumns (index, callback) {
+        if (this._isDefaultLayerAggregation(index)) {
+            const skipGeoms = true;
+            return this.getLayerColumns(index, skipGeoms, (err, columns) => {
+                if (err) {
+                    return callback(err);
+                }
+
+                return callback(null, columns);
+            });
+        }
+
+        const columns = this._getLayerAggregationRequiredColumns(index);
+
+        return callback(null, columns);
+    }
+
+    _getLayerAggregationRequiredColumns (index) {
+        const { columns, dimensions } = this.getAggregation(index);
+
+        let aggregatedColumns = [];
+        if (columns) {
+            aggregatedColumns = Object.keys(columns)
+                .map(key => columns[key].aggregated_column)
+                .filter(aggregatedColumn => typeof aggregatedColumn === 'string');
+        }
+
+        let dimensionsColumns = [];
+        if (dimensions) {
+            dimensionsColumns = Object.keys(dimensions)
+                .map(key => dimensions[key])
+                .filter(dimension => typeof dimension === 'string');
+        }
+
+        return removeDuplicates(aggregatedColumns.concat(dimensionsColumns));
+    }
+
+    doesLayerReachThreshold(index, featureCount) {
+        const threshold = this.getAggregation(index) && this.getAggregation(index).threshold ?
+            this.getAggregation(index).threshold :
+            AggregationMapConfig.THRESHOLD;
+
+        return featureCount >= threshold;
+    }
+
+    getLayerColumns (index, skipGeoms, callback) {
+        const geomColumns = ['the_geom', 'the_geom_webmercator'];
+        const limitedQuery = ctx => `SELECT * FROM (${ctx.query}) __cdb_schema LIMIT 0`;
+        const layer = this.getLayer(index);
+
+        this.pgConnection.getConnection(this.user, (err, connection) => {
+            if (err) {
+                return callback(err);
+            }
+
+            const sql = limitedQuery({
+                query: prepareSql(layer.options.sql)
+            });
+
+            connection.query(sql, (err, result) => {
+                if (err) {
+                    return callback(err);
+                }
+
+                let columns = result.fields || [];
+
+                columns = columns.map(({ name }) => name);
+
+                if (skipGeoms) {
+                    columns = columns.filter((column) => !geomColumns.includes(column));
+                }
+
+                return callback(err, columns);
+            });
+        });
+    }
+
+    _isDefaultLayerAggregation (index) {
+        const aggregation = this.getAggregation(index);
+
+        return (this.isVectorOnlyMapConfig() && !this.hasLayerAggregation(index)) ||
+            aggregation === true ||
+            this._isDefaultAggregation(aggregation);
+    }
+
+    _isDefaultAggregation (aggregation) {
+        return aggregation.placement === undefined &&
+            aggregation.columns === undefined &&
+            this._isEmptyParameter(aggregation.dimensions);
+    }
+
+    _isEmptyParameter(parameter) {
+        return parameter === undefined || parameter === null || this._isEmptyObject(parameter);
+    }
+
+    _isEmptyObject (parameter) {
+        return typeof parameter === 'object' && Object.keys(parameter).length === 0;
+    }
+};

lib/cartodb/models/aggregation/aggregation-query.js

@@ -0,0 +1,253 @@
+const DEFAULT_PLACEMENT = 'point-sample';
+
+/**
+ * Returns a template function (function that accepts template parameters and returns a string)
+ * to generate an aggregation query.
+ * Valid options to define the query template are:
+ * - placement
+ * - columns
+ * - dimensions*
+ * The query template parameters taken by the result template function are:
+ * - sourceQuery
+ * - res
+ * - columns
+ * - dimensions
+ */
+const templateForOptions = (options) => {
+    let templateFn = defaultAggregationQueryTemplate;
+    if (!options.isDefaultAggregation) {
+        templateFn = aggregationQueryTemplates[options.placement || DEFAULT_PLACEMENT];
+        if (!templateFn) {
+            throw new Error("Invalid Aggregation placement: '" + options.placement + "'");
+        }
+    }
+    return templateFn;
+};
+
+/**
+ * Generates an aggregation query given the aggregation options:
+ * - query
+ * - resolution - defined as in torque:
+ *   aggregation cell is resolution*resolution pixels, where tiles are always 256x256 pixels
+ * - columns
+ * - placement
+ * - dimensions
+ *
+ * The default aggregation (when no explicit placement, columns or dimensions are present) returns
+ * a sample record (with all the original columns and _cdb_feature_count) for each aggregation group.
+ * When placement, columns or dimensions are specified, columns are aggregated as requested
+ * (by default only _cdb_feature_count) and with the_geom_webmercator as defined by placement.
+ */
+const queryForOptions = (options) => templateForOptions(options)({
+    sourceQuery: options.query,
+    res: 256/options.resolution,
+    columns: options.columns,
+    dimensions: options.dimensions
+});
+
+module.exports = queryForOptions;
+
+const SUPPORTED_AGGREGATE_FUNCTIONS = {
+    'count': {
+        sql: (column_name, params) => `count(${params.aggregated_column || '*'})`
+    },
+    'avg': {
+        sql: (column_name, params) => `avg(${params.aggregated_column || column_name})`
+    },
+    'sum': {
+        sql: (column_name, params) => `sum(${params.aggregated_column || column_name})`
+    },
+    'min': {
+        sql: (column_name, params) => `min(${params.aggregated_column || column_name})`
+    },
+    'max': {
+        sql: (column_name, params) => `max(${params.aggregated_column || column_name})`
+    },
+    'mode': {
+        sql: (column_name, params) => `_cdb_mode(${params.aggregated_column || column_name})`
+    }
+};
+
+module.exports.SUPPORTED_AGGREGATE_FUNCTIONS = Object.keys(SUPPORTED_AGGREGATE_FUNCTIONS);
+
+const sep = (list) => {
+    let expr = list.join(', ');
+    return expr ? ', ' + expr : expr;
+};
+
+const aggregateColumns = ctx => {
+    return Object.assign({
+        _cdb_feature_count: {
+            aggregate_function: 'count'
+        }
+    }, ctx.columns || {});
+};
+
+const aggregateColumnNames = (ctx, table) => {
+    let columns = aggregateColumns(ctx);
+    if (table) {
+        return sep(Object.keys(columns).map(
+            column_name => `${table}.${column_name}`
+        ));
+    }
+    return sep(Object.keys(columns));
+};
+
+const aggregateColumnDefs = ctx => {
+    let columns = aggregateColumns(ctx);
+    return sep(Object.keys(columns).map(column_name => {
+        const aggregate_function = columns[column_name].aggregate_function || 'count';
+        const aggregate_definition = SUPPORTED_AGGREGATE_FUNCTIONS[aggregate_function];
+        if (!aggregate_definition) {
+            throw new Error("Invalid Aggregate function: '" + aggregate_function + "'");
+        }
+        const aggregate_expression = aggregate_definition.sql(column_name, columns[column_name]);
+        return `${aggregate_expression} AS ${column_name}`;
+    }));
+};
+
+
+const aggregateDimensions = ctx => ctx.dimensions || {};
+
+const dimensionNames = (ctx, table) => {
+    let dimensions = aggregateDimensions(ctx);
+    if (table) {
+        return sep(Object.keys(dimensions).map(
+            dimension_name => `${table}.${dimension_name}`
+        ));
+    }
+    return sep(Object.keys(dimensions));
+};
+
+const dimensionDefs = ctx => {
+    let dimensions = aggregateDimensions(ctx);
+    return sep(Object.keys(dimensions).map(dimension_name => {
+        const expression = dimensions[dimension_name];
+        return `${expression} AS ${dimension_name}`;
+    }));
+};
+
+// SQL expression to compute the aggregation resolution (grid cell size).
+// This is equivalent to `${256/ctx.res}*CDB_XYZ_Resolution(CDB_ZoomFromScale(!scale_denominator!))`
+// This is defined by the ctx.res parameter, which is the number of grid cells per tile linear dimension
+// (i.e. each tile is divided into ctx.res*ctx.res cells).
+const gridResolution = ctx => `(${256*0.00028/ctx.res}*!scale_denominator!)::double precision`;
+
+// Notes:
+// * We need to filter spatially using !bbox! to make the queries efficient because
+//   the filter added by Mapnik (wrapping the query)
+//   is only applied after the aggregation.
+// * This queries are used for rendering and the_geom is omitted in the results for better performance
+
+// The special default aggregation includes all the columns of a sample row per grid cell and
+// the count (_cdb_feature_count) of the aggregated rows.
+const defaultAggregationQueryTemplate = ctx => `
+    WITH
+    _cdb_params AS (
+        SELECT
+        ${gridResolution(ctx)} AS res,
+        !bbox! AS bbox
+    ),
+    _cdb_clusters AS (
+        SELECT
+            MIN(cartodb_id) AS cartodb_id
+            ${dimensionDefs(ctx)}
+            ${aggregateColumnDefs(ctx)}
+        FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params
+        WHERE _cdb_query.the_geom_webmercator && _cdb_params.bbox
+        GROUP BY
+            Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res),
+            Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)
+            ${dimensionNames(ctx)}
+    ) SELECT
+        _cdb_query.*
+        ${aggregateColumnNames(ctx)}
+    FROM
+        _cdb_clusters INNER JOIN (${ctx.sourceQuery}) _cdb_query
+        ON (_cdb_clusters.cartodb_id = _cdb_query.cartodb_id)
+`;
+
+const aggregationQueryTemplates = {
+    'centroid': ctx => `
+        WITH
+        _cdb_params AS (
+            SELECT
+            ${gridResolution(ctx)} AS res,
+            !bbox! AS bbox
+        )
+        SELECT
+            row_number() over() AS cartodb_id,
+            ST_SetSRID(
+                ST_MakePoint(
+                    AVG(ST_X(_cdb_query.the_geom_webmercator)),
+                    AVG(ST_Y(_cdb_query.the_geom_webmercator))
+                ), 3857
+            ) AS the_geom_webmercator
+            ${dimensionDefs(ctx)}
+            ${aggregateColumnDefs(ctx)}
+        FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params
+        WHERE _cdb_query.the_geom_webmercator && _cdb_params.bbox
+        GROUP BY
+            Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res),
+            Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)
+            ${dimensionNames(ctx)}
+    `,
+
+    'point-grid': ctx => `
+        WITH
+        _cdb_params AS (
+            SELECT
+            ${gridResolution(ctx)} AS res,
+            !bbox! AS bbox
+        ),
+        _cdb_clusters AS (
+            SELECT
+            Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res)::int AS _cdb_gx,
+            Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)::int AS _cdb_gy
+            ${dimensionDefs(ctx)}
+            ${aggregateColumnDefs(ctx)}
+            FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params
+            WHERE the_geom_webmercator && _cdb_params.bbox
+            GROUP BY _cdb_gx, _cdb_gy ${dimensionNames(ctx)}
+        )
+        SELECT
+            row_number() over() AS cartodb_id,
+            ST_SetSRID(ST_MakePoint((_cdb_gx+0.5)*res, (_cdb_gy+0.5)*res), 3857) AS the_geom_webmercator
+            ${dimensionNames(ctx)}
+            ${aggregateColumnNames(ctx)}
+            FROM _cdb_clusters, _cdb_params
+    `,
+
+    'point-sample': ctx => `
+        WITH
+        _cdb_params AS (
+            SELECT
+            ${gridResolution(ctx)} AS res,
+            !bbox! AS bbox
+        ),
+        _cdb_clusters AS (
+            SELECT
+                MIN(cartodb_id) AS cartodb_id
+                ${dimensionDefs(ctx)}
+                ${aggregateColumnDefs(ctx)}
+            FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params
+            WHERE _cdb_query.the_geom_webmercator && _cdb_params.bbox
+            GROUP BY
+                Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res),
+                Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)
+                ${dimensionNames(ctx)}
+        )
+        SELECT
+            _cdb_clusters.cartodb_id,
+            the_geom, the_geom_webmercator
+            ${dimensionNames(ctx, '_cdb_query')}
+            ${aggregateColumnNames(ctx, '_cdb_clusters')}
+        FROM
+            _cdb_clusters INNER JOIN (${ctx.sourceQuery}) _cdb_query
+            ON (_cdb_clusters.cartodb_id = _cdb_query.cartodb_id)
+    `
+
+};
+
+module.exports.SUPPORTED_PLACEMENTS = Object.keys(aggregationQueryTemplates);
+module.exports.GEOMETRY_COLUMN = 'the_geom_webmercator';

lib/cartodb/models/aggregation/aggregation-validator.js

@@ -0,0 +1,93 @@
+module.exports = function aggregationValidator (mapconfig) {
+    return function validateProperty (key, validator) {
+        for (let index = 0; index < mapconfig.getLayers().length; index++) {
+            const aggregation = mapconfig.getAggregation(index);
+
+            if (aggregation === undefined || aggregation[key] === undefined) {
+                continue;
+            }
+
+            validator(aggregation[key], key, index);
+        }
+    };
+};
+
+module.exports.createIncludesValueValidator = function (mapconfig, validValues) {
+    return function validateIncludesValue (value, key, index) {
+        if (!validValues.includes(value)) {
+            const message = `Invalid ${key}. Valid values: ${validValues.join(', ')}`;
+            throw createLayerError(message, mapconfig, index);
+        }
+    };
+};
+
+module.exports.createPositiveNumberValidator = function (mapconfig) {
+    return function validatePositiveNumber (value, key, index) {
+        if (!Number.isFinite(value) || value <= 0) {
+            const message = `Invalid ${key}, should be a number greather than 0`;
+            throw createLayerError(message, mapconfig, index);
+        }
+    };
+};
+
+module.exports.createAggregationColumnsValidator = function (mapconfig, validAggregatedFunctions) {
+    const validateAggregationColumnNames = createAggregationColumnNamesValidator(mapconfig);
+    const validateAggregateFunction = createAggregateFunctionValidator(mapconfig, validAggregatedFunctions);
+    const validateAggregatedColumn = createAggregatedColumnValidator(mapconfig);
+
+    return function validateAggregationColumns (value, key, index) {
+        validateAggregationColumnNames(value, key, index);
+        validateAggregateFunction(value, key, index);
+        validateAggregatedColumn(value, key, index);
+    };
+};
+
+function createAggregationColumnNamesValidator(mapconfig) {
+    return function validateAggregationColumnNames (value, key, index) {
+        Object.keys(value).forEach((columnName) => {
+            if (columnName.length <= 0) {
+                const message = `Invalid column name, should be a non empty string`;
+                throw createLayerError(message, mapconfig, index);
+            }
+        });
+    };
+}
+
+function createAggregateFunctionValidator (mapconfig, validAggregatedFunctions) {
+    return function validateAggregateFunction (value, key, index) {
+        Object.keys(value).forEach((columnName) => {
+            const { aggregate_function } = value[columnName];
+
+            if (!validAggregatedFunctions.includes(aggregate_function)) {
+                const message = `Unsupported aggregation function ${aggregate_function},` +
+                                ` valid ones: ${validAggregatedFunctions.join(', ')}`;
+                throw createLayerError(message, mapconfig, index);
+            }
+        });
+    };
+}
+
+function createAggregatedColumnValidator (mapconfig) {
+    return function validateAggregatedColumn (value, key, index) {
+        Object.keys(value).forEach((columnName) => {
+            const { aggregated_column } = value[columnName];
+
+            if (typeof aggregated_column !== 'string' || aggregated_column <= 0) {
+                const message = `Invalid aggregated column, should be a non empty string`;
+                throw createLayerError(message, mapconfig, index);
+            }
+        });
+    };
+}
+
+function createLayerError(message, mapconfig, index) {
+    const error = new Error(message);
+    error.type = 'layer';
+    error.layer = {
+        id: mapconfig.getLayerId(index),
+        index: index,
+        type: mapconfig.layerType(index)
+    };
+
+    return error;
+}

lib/cartodb/models/dataview/aggregation.js

@@ -1,95 +1,178 @@
-var _ = require('underscore');
-var BaseWidget = require('./base');
-var debug = require('debug')('windshaft:widget:aggregation');
+const BaseDataview = require('./base');
+const debug = require('debug')('windshaft:dataview:aggregation');
 
-var dot = require('dot');
-dot.templateSettings.strip = false;
+const filteredQueryTpl = ctx => `
+    filtered_source AS (
+        SELECT *
+        FROM (${ctx.query}) _cdb_filtered_source
+        ${ctx.aggregationColumn && ctx.isFloatColumn ? `
+        WHERE
+            ${ctx.aggregationColumn} != 'infinity'::float
+        AND
+            ${ctx.aggregationColumn} != '-infinity'::float
+        AND
+            ${ctx.aggregationColumn} != 'NaN'::float` :
+        ''
+        }
+    )
+`;
 
-var filteredQueryTpl = dot.template([
-    'filtered_source AS (',
-    '  SELECT *',
-    '  FROM ({{=it._query}}) _cdb_filtered_source',
-    '  {{?it._aggregationColumn && it._isFloatColumn}}WHERE',
-    '    {{=it._aggregationColumn}} != \'infinity\'::float',
-    '  AND',
-    '    {{=it._aggregationColumn}} != \'-infinity\'::float',
-    '  AND',
-    '    {{=it._aggregationColumn}} != \'NaN\'::float{{?}}',
-    ')'
-].join(' \n'));
+const summaryQueryTpl = ctx => `
+    summary AS (
+        SELECT
+            count(1) AS count,
+            sum(CASE WHEN ${ctx.column} IS NULL THEN 1 ELSE 0 END) AS nulls_count
+            ${ctx.isFloatColumn ? `,
+            sum(
+                CASE
+                    WHEN ${ctx.aggregationColumn} = 'infinity'::float OR ${ctx.aggregationColumn} = '-infinity'::float
+                    THEN 1
+                    ELSE 0
+                    END
+            ) AS infinities_count,
+            sum(CASE WHEN ${ctx.aggregationColumn} = 'NaN'::float THEN 1 ELSE 0 END) AS nans_count` :
+            ''
+            }
+        FROM (${ctx.query}) _cdb_aggregation_nulls
+    )
+`;
 
-var summaryQueryTpl = dot.template([
-    'summary AS (',
-    '  SELECT',
-    '  count(1) AS count,',
-    '  sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count',
-    '  {{?it._isFloatColumn}},sum(',
-    '    CASE',
-    '      WHEN {{=it._aggregationColumn}} = \'infinity\'::float OR {{=it._aggregationColumn}} = \'-infinity\'::float',
-    '      THEN 1',
-    '      ELSE 0',
-    '    END',
-    '  ) AS infinities_count,',
-    '  sum(CASE WHEN {{=it._aggregationColumn}} = \'NaN\'::float THEN 1 ELSE 0 END) AS nans_count{{?}}',
-    '  FROM ({{=it._query}}) _cdb_aggregation_nulls',
-    ')'
-].join('\n'));
+const rankedCategoriesQueryTpl = ctx => `
+    categories AS(
+        SELECT
+            ${ctx.column} AS category,
+            ${ctx.aggregationFn} AS value,
+            row_number() OVER (ORDER BY ${ctx.aggregationFn} desc) as rank
+        FROM filtered_source
+        ${ctx.aggregationColumn !== null ? `WHERE ${ctx.aggregationColumn} IS NOT NULL` : ''}
+        GROUP BY ${ctx.column}
+        ORDER BY 2 DESC
+    )
+`;
 
-var rankedCategoriesQueryTpl = dot.template([
-    'categories AS(',
-    '  SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,',
-    '    row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank',
-    '  FROM filtered_source',
-    '  {{?it._aggregationColumn!==null}}WHERE {{=it._aggregationColumn}} IS NOT NULL{{?}}',
-    '  GROUP BY {{=it._column}}',
-    '  ORDER BY 2 DESC',
-    ')'
-].join('\n'));
+const categoriesSummaryMinMaxQueryTpl = () => `
+    categories_summary_min_max AS(
+        SELECT
+            max(value) max_val,
+            min(value) min_val
+        FROM categories
+    )
+`;
 
-var categoriesSummaryMinMaxQueryTpl = dot.template([
-    'categories_summary_min_max AS(',
-    '  SELECT max(value) max_val, min(value) min_val',
-    '  FROM categories',
-    ')'
-].join('\n'));
+const categoriesSummaryCountQueryTpl = ctx => `
+    categories_summary_count AS(
+        SELECT count(1) AS categories_count
+        FROM (
+            SELECT ${ctx.column} AS category
+            FROM (${ctx.query}) _cdb_categories
+            GROUP BY ${ctx.column}
+        ) _cdb_categories_count
+    )
+`;
 
-var categoriesSummaryCountQueryTpl = dot.template([
-    'categories_summary_count AS(',
-    '  SELECT count(1) AS categories_count',
-    '  FROM (',
-    '    SELECT {{=it._column}} AS category',
-    '    FROM ({{=it._query}}) _cdb_categories',
-    '    GROUP BY {{=it._column}}',
-    '  ) _cdb_categories_count',
-    ')'
-].join('\n'));
+const specialNumericValuesColumns = () => `, nans_count, infinities_count`;
 
-var rankedAggregationQueryTpl = dot.template([
-    'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val,',
-    '    count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
-    '  FROM categories, summary, categories_summary_min_max, categories_summary_count',
-    '  WHERE rank < {{=it._limit}}',
-    'UNION ALL',
-    'SELECT \'Other\' category, {{=it._aggregationFn}}(value) as value, true as agg, nulls_count,',
-    '       min_val, max_val, count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
-    '  FROM categories, summary, categories_summary_min_max, categories_summary_count',
-    '  WHERE rank >= {{=it._limit}}',
-    'GROUP BY nulls_count, min_val, max_val, count,',
-    '  categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}'
-].join('\n'));
+const rankedAggregationQueryTpl = ctx => `
+    SELECT
+        category,
+        value,
+        false as agg,
+        nulls_count,
+        min_val,
+        max_val,
+        count,
+        categories_count
+        ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' }
+        FROM categories, summary, categories_summary_min_max, categories_summary_count
+        WHERE rank < ${ctx.limit}
+    UNION ALL
+    SELECT
+        null category,
+        ${ctx.aggregation !== 'count' ? ctx.aggregation : 'sum'}(value) as value,
+        true as agg,
+        nulls_count,
+        min_val,
+        max_val,
+        count,
+        categories_count
+        ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' }
+        FROM categories, summary, categories_summary_min_max, categories_summary_count
+        WHERE rank >= ${ctx.limit}
+    GROUP BY
+        nulls_count,
+        min_val,
+        max_val,
+        count,
+        categories_count
+        ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' }
+`;
 
-var aggregationQueryTpl = dot.template([
-    'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,',
-    '  nulls_count, min_val, max_val, count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
-    'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count',
-    'GROUP BY category, nulls_count, min_val, max_val, count,',
-    '  categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
-    'ORDER BY value DESC'
-].join('\n'));
+const aggregationQueryTpl = ctx => `
+    SELECT
+        ${ctx.column} AS category,
+        ${ctx.aggregationFn} AS value,
+        false as agg,
+        nulls_count,
+        min_val,
+        max_val,
+        count,
+        categories_count
+        ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' }
+    FROM (${ctx.query}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count
+    GROUP BY
+        ${ctx.column},
+        nulls_count,
+        min_val,
+        max_val,
+        count,
+        categories_count
+        ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' }
+    ORDER BY value DESC
+`;
 
-var CATEGORIES_LIMIT = 6;
+const aggregationFnQueryTpl = ctx => `${ctx.aggregation}(${ctx.aggregationColumn})`;
 
-var VALID_OPERATIONS = {
+const aggregationDataviewQueryTpl = ctx => `
+    WITH
+    ${filteredQueryTpl(ctx)},
+    ${summaryQueryTpl(ctx)},
+    ${rankedCategoriesQueryTpl(ctx)},
+    ${categoriesSummaryMinMaxQueryTpl(ctx)},
+    ${categoriesSummaryCountQueryTpl(ctx)}
+    ${!!ctx.override.ownFilter ? `${aggregationQueryTpl(ctx)}` : `${rankedAggregationQueryTpl(ctx)}`}
+`;
+
+const filterCategoriesQueryTpl = ctx => `
+    SELECT
+        ${ctx.column} AS category,
+        ${ctx.value} AS value
+    FROM (${ctx.query}) _cdb_aggregation_search
+    WHERE CAST(${ctx.column} as text) ILIKE ${ctx.userQuery}
+    GROUP BY ${ctx.column}
+`;
+
+const searchQueryTpl = ctx => `
+    WITH
+    search_unfiltered AS (
+        ${ctx.searchUnfiltered}
+    ),
+    search_filtered AS (
+        ${ctx.searchFiltered}
+    ),
+    search_union AS (
+        SELECT * FROM search_unfiltered
+        UNION ALL
+        SELECT * FROM search_filtered
+    )
+    SELECT category, sum(value) AS value
+    FROM search_union
+    GROUP BY category
+    ORDER BY value desc
+`;
+
+const CATEGORIES_LIMIT = 6;
+
+const VALID_OPERATIONS = {
     count: [],
     sum: ['aggregationColumn'],
     avg: ['aggregationColumn'],
@@ -97,7 +180,7 @@ var VALID_OPERATIONS = {
     max: ['aggregationColumn']
 };
 
-var TYPE = 'aggregation';
+const TYPE = 'aggregation';
 
 /**
  {
@@ -108,256 +191,160 @@ var TYPE = 'aggregation';
      }
  }
  */
-function Aggregation(query, options, queries) {
-    if (!_.isString(options.column)) {
-        throw new Error('Aggregation expects `column` in widget options');
+module.exports = class Aggregation extends BaseDataview {
+    constructor (query, options = {}, queries = {}) {
+        super();
+
+        this._checkOptions(options);
+
+        this.query = query;
+        this.queries = queries;
+        this.column = options.column;
+        this.aggregation = options.aggregation;
+        this.aggregationColumn = options.aggregationColumn;
+        this._isFloatColumn = null;
     }
 
-    if (!_.isString(options.aggregation)) {
-        throw new Error('Aggregation expects `aggregation` operation in widget options');
-    }
-
-    if (!VALID_OPERATIONS[options.aggregation]) {
-        throw new Error("Aggregation does not support '" + options.aggregation + "' operation");
-    }
-
-    var requiredOptions = VALID_OPERATIONS[options.aggregation];
-    var missingOptions = _.difference(requiredOptions, Object.keys(options));
-    if (missingOptions.length > 0) {
-        throw new Error(
-            "Aggregation '" + options.aggregation + "' is missing some options: " + missingOptions.join(',')
-        );
-    }
-
-    BaseWidget.apply(this);
-
-    this.query = query;
-    this.queries = queries;
-    this.column = options.column;
-    this.aggregation = options.aggregation;
-    this.aggregationColumn = options.aggregationColumn;
-    this._isFloatColumn = null;
-}
-
-Aggregation.prototype = new BaseWidget();
-Aggregation.prototype.constructor = Aggregation;
-
-module.exports = Aggregation;
-
-Aggregation.prototype.sql = function(psql, override, callback) {
-    var self = this;
-
-    if (!callback) {
-        callback = override;
-        override = {};
-    }
-
-    if (this.aggregationColumn && this._isFloatColumn === null) {
-        this._isFloatColumn = false;
-        this.getColumnType(psql, this.aggregationColumn, this.queries.no_filters, function (err, type) {
-            if (!err && !!type) {
-                self._isFloatColumn = type.float;
-            }
-            self.sql(psql, override, callback);
-        });
-        return null;
-    }
-
-    var _query = this.query;
-
-    var aggregationSql;
-
-    if (!!override.ownFilter) {
-        aggregationSql = [
-            this.getCategoriesCTESql(
-                _query,
-                this.column,
-                this.aggregation,
-                this.aggregationColumn,
-                this._isFloatColumn
-            ),
-            aggregationQueryTpl({
-                _isFloatColumn: this._isFloatColumn,
-                _query: _query,
-                _column: this.column,
-                _aggregation: this.getAggregationSql(),
-                _limit: CATEGORIES_LIMIT
-            })
-        ].join('\n');
-    } else {
-        aggregationSql = [
-            this.getCategoriesCTESql(
-                _query,
-                this.column,
-                this.aggregation,
-                this.aggregationColumn,
-                this._isFloatColumn
-            ),
-            rankedAggregationQueryTpl({
-                _isFloatColumn: this._isFloatColumn,
-                _query: _query,
-                _column: this.column,
-                _aggregationFn: this.aggregation !== 'count' ? this.aggregation : 'sum',
-                _limit: CATEGORIES_LIMIT
-            })
-        ].join('\n');
-    }
-
-    debug(aggregationSql);
-
-    return callback(null, aggregationSql);
-};
-
-Aggregation.prototype.getCategoriesCTESql = function(query, column, aggregation, aggregationColumn, isFloatColumn) {
-    return [
-        "WITH",
-        [
-            filteredQueryTpl({
-                _isFloatColumn: isFloatColumn,
-                _query: this.query,
-                _column: this.column,
-                _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null
-            }),
-            summaryQueryTpl({
-                _isFloatColumn: isFloatColumn,
-                _query: query,
-                _column: column,
-                _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null
-            }),
-            rankedCategoriesQueryTpl({
-                _query: query,
-                _column: column,
-                _aggregation: this.getAggregationSql(),
-                _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null
-            }),
-            categoriesSummaryMinMaxQueryTpl({
-                _query: query,
-                _column: column
-            }),
-            categoriesSummaryCountQueryTpl({
-                _query: query,
-                _column: column
-            })
-        ].join(',\n')
-    ].join('\n');
-};
-
-var aggregationFnQueryTpl = dot.template('{{=it._aggregationFn}}({{=it._aggregationColumn}})');
-Aggregation.prototype.getAggregationSql = function() {
-    return aggregationFnQueryTpl({
-        _aggregationFn: this.aggregation,
-        _aggregationColumn: this.aggregationColumn || 1
-    });
-};
-
-Aggregation.prototype.format = function(result) {
-    var categories = [];
-    var count = 0;
-    var nulls = 0;
-    var nans = 0;
-    var infinities = 0;
-    var minValue = 0;
-    var maxValue = 0;
-    var categoriesCount = 0;
-
-
-    if (result.rows.length) {
-        var firstRow = result.rows[0];
-        count = firstRow.count;
-        nulls = firstRow.nulls_count;
-        nans = firstRow.nans_count;
-        infinities = firstRow.infinities_count;
-        minValue = firstRow.min_val;
-        maxValue = firstRow.max_val;
-        categoriesCount = firstRow.categories_count;
-
-        result.rows.forEach(function(row) {
-            categories.push(_.omit(row, 'count', 'nulls_count', 'min_val',
-                'max_val', 'categories_count', 'nans_count', 'infinities_count'));
-        });
-    }
-
-    return {
-        aggregation: this.aggregation,
-        count: count,
-        nulls: nulls,
-        nans: nans,
-        infinities: infinities,
-        min: minValue,
-        max: maxValue,
-        categoriesCount: categoriesCount,
-        categories: categories
-    };
-};
-
-var filterCategoriesQueryTpl = dot.template([
-    'SELECT {{=it._column}} AS category, {{=it._value}} AS value',
-    'FROM ({{=it._query}}) _cdb_aggregation_search',
-    'WHERE CAST({{=it._column}} as text) ILIKE {{=it._userQuery}}',
-    'GROUP BY {{=it._column}}'
-].join('\n'));
-
-var searchQueryTpl = dot.template([
-    'WITH',
-    'search_unfiltered AS (',
-    '  {{=it._searchUnfiltered}}',
-    '),',
-    'search_filtered AS (',
-    '  {{=it._searchFiltered}}',
-    '),',
-    'search_union AS (',
-    '  SELECT * FROM search_unfiltered',
-    '  UNION ALL',
-    '  SELECT * FROM search_filtered',
-    ')',
-    'SELECT category, sum(value) AS value',
-    'FROM search_union',
-    'GROUP BY category',
-    'ORDER BY value desc'
-].join('\n'));
-
-
-Aggregation.prototype.search = function(psql, userQuery, callback) {
-    var self = this;
-
-    var _userQuery = psql.escapeLiteral('%' + userQuery + '%');
-    var _value = this.aggregation !== 'count' && this.aggregationColumn ?
-        this.aggregation + '(' + this.aggregationColumn + ')' : 'count(1)';
-
-    // TODO unfiltered will be wrong as filters are already applied at this point
-    var query = searchQueryTpl({
-        _searchUnfiltered: filterCategoriesQueryTpl({
-            _query: this.query,
-            _column: this.column,
-            _value: '0',
-            _userQuery: _userQuery
-        }),
-        _searchFiltered: filterCategoriesQueryTpl({
-            _query: this.query,
-            _column: this.column,
-            _value: _value,
-            _userQuery: _userQuery
-        })
-    });
-
-    psql.query(query, function(err, result) {
-        if (err) {
-            return callback(err, result);
+    _checkOptions (options) {
+        if (typeof options.column !== 'string') {
+            throw new Error(`Aggregation expects 'column' in dataview options`);
         }
 
-        return callback(null, {type: self.getType(), categories: result.rows });
-    }, true); // use read-only transaction
-};
+        if (typeof options.aggregation !== 'string') {
+            throw new Error(`Aggregation expects 'aggregation' operation in dataview options`);
+        }
 
-Aggregation.prototype.getType = function() {
-    return TYPE;
-};
+        if (!VALID_OPERATIONS[options.aggregation]) {
+            throw new Error(`Aggregation does not support '${options.aggregation}' operation`);
+        }
 
-Aggregation.prototype.toString = function() {
-    return JSON.stringify({
-        _type: TYPE,
-        _query: this.query,
-        _column: this.column,
-        _aggregation: this.aggregation
-    });
+        const requiredOptions = VALID_OPERATIONS[options.aggregation];
+        const missingOptions = requiredOptions.filter(requiredOption => !options.hasOwnProperty(requiredOption));
+
+        if (missingOptions.length > 0) {
+            throw new Error(
+                `Aggregation '${options.aggregation}' is missing some options: ${missingOptions.join(',')}`
+            );
+        }
+    }
+
+    sql (psql, override, callback) {
+        if (!callback) {
+            callback = override;
+            override = {};
+        }
+
+        if (this._shouldCheckColumnType()) {
+            this._isFloatColumn = false;
+            this.getColumnType(psql, this.aggregationColumn, this.queries.no_filters, (err, type) => {
+                if (!err && !!type) {
+                    this._isFloatColumn = type.float;
+                }
+                this.sql(psql, override, callback);
+            });
+            return null;
+        }
+
+        const limit = Number.isFinite(override.categories) && override.categories > 0 ?
+            override.categories :
+            CATEGORIES_LIMIT;
+
+        const aggregationSql = aggregationDataviewQueryTpl({
+            override: override,
+            query: this.query,
+            column: this.column,
+            aggregation: this.aggregation,
+            aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null,
+            aggregationFn: aggregationFnQueryTpl({
+                aggregation: this.aggregation,
+                aggregationColumn: this.aggregationColumn || 1
+            }),
+            isFloatColumn: this._isFloatColumn,
+            limit
+        });
+
+        debug(aggregationSql);
+
+        return callback(null, aggregationSql);
+    }
+
+    _shouldCheckColumnType () {
+        return this.aggregationColumn && this._isFloatColumn === null;
+    }
+
+    format (result) {
+        const {
+            count = 0,
+            nulls_count = 0,
+            nans_count = 0,
+            infinities_count = 0,
+            min_val = 0,
+            max_val = 0,
+            categories_count = 0
+         } = result.rows[0] || {};
+        
+        return {
+            aggregation: this.aggregation,
+            count: count,
+            nulls: nulls_count,
+            nans: nans_count,
+            infinities: infinities_count,
+            min: min_val,
+            max: max_val,
+            categoriesCount: categories_count,
+            categories: result.rows.map(({ category, value, agg }) => {
+                return { 
+                    category: agg ? 'Other' : category,
+                    value, 
+                    agg 
+                };
+            })
+        };
+    }
+
+    search (psql, userQuery, callback) {
+        const escapedUserQuery = psql.escapeLiteral(`%${userQuery}%`);
+        const value = this.aggregation !== 'count' && this.aggregationColumn ?
+            `${this.aggregation}(${this.aggregationColumn})` :
+            'count(1)';
+
+        // TODO unfiltered will be wrong as filters are already applied at this point
+        const query = searchQueryTpl({
+            searchUnfiltered: filterCategoriesQueryTpl({
+                query: this.query,
+                column: this.column,
+                value: '0',
+                userQuery: escapedUserQuery
+            }),
+            searchFiltered: filterCategoriesQueryTpl({
+                query: this.query,
+                column: this.column,
+                value: value,
+                userQuery: escapedUserQuery
+            })
+        });
+
+        debug(query);
+
+        psql.query(query, (err, result) => {
+            if (err) {
+                return callback(err, result);
+            }
+
+            return callback(null, {type: this.getType(), categories: result.rows });
+        }, true); // use read-only transaction
+    }
+
+    getType () {
+        return TYPE;
+    }
+
+    toString () {
+        return JSON.stringify({
+            _type: TYPE,
+            _query: this.query,
+            _column: this.column,
+            _aggregation: this.aggregation
+        });
+    }
 };

lib/cartodb/models/dataview/base.js

@@ -1,67 +1,16 @@
-var dot = require('dot');
-dot.templateSettings.strip = false;
-
-function BaseDataview() {}
-
-module.exports = BaseDataview;
-
-BaseDataview.prototype.getResult = function(psql, override, callback) {
-    var self = this;
-    this.sql(psql, override, function(err, query) {
-        if (err) {
-            return callback(err);
-        }
-
-        psql.query(query, function(err, result) {
-            if (err) {
-                return callback(err, result);
-            }
-
-            result = self.format(result, override);
-            result.type = self.getType();
-
-            return callback(null, result);
-
-        }, true); // use read-only transaction
-    });
-
-};
-
-BaseDataview.prototype.search = function(psql, userQuery, callback) {
-    return callback(null, this.format({ rows: [] }));
-};
-
-var FLOAT_OIDS = {
+const FLOAT_OIDS = {
     700: true,
     701: true,
     1700: true
 };
 
-var DATE_OIDS = {
+const DATE_OIDS = {
     1082: true,
     1114: true,
     1184: true
 };
 
-var columnTypeQueryTpl = dot.template(
-    'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_column_type limit 1'
-);
-
-BaseDataview.prototype.getColumnType = function (psql, column, query, callback) {
-    var readOnlyTransaction = true;
-
-    var columnTypeQuery = columnTypeQueryTpl({
-        column: column, query: query
-    });
-
-    psql.query(columnTypeQuery, function(err, result) {
-        if (err) {
-            return callback(err);
-        }
-        var pgType = result.rows[0].pg_typeof;
-        callback(null, getPGTypeName(pgType));
-    }, readOnlyTransaction);
-};
+const columnTypeQueryTpl = ctx => `SELECT pg_typeof(${ctx.column})::oid FROM (${ctx.query}) _cdb_column_type limit 1`;
 
 function getPGTypeName (pgType) {
     return {
@@ -69,3 +18,42 @@ function getPGTypeName (pgType) {
         date: DATE_OIDS.hasOwnProperty(pgType)
     };
 }
+
+module.exports = class BaseDataview {
+    getResult (psql, override, callback) {
+        this.sql(psql, override, (err, query) => {
+            if (err) {
+                return callback(err);
+            }
+
+            psql.query(query, (err, result) => {
+                if (err) {
+                    return callback(err, result);
+                }
+
+                result = this.format(result, override);
+                result.type = this.getType();
+
+                return callback(null, result);
+
+            }, true); // use read-only transaction
+        });
+    }
+
+    search (psql, userQuery, callback) {
+        return callback(null, this.format({ rows: [] }));
+    }
+
+    getColumnType (psql, column, query, callback) {
+        const readOnlyTransaction = true;
+        const columnTypeQuery = columnTypeQueryTpl({ column, query });
+
+        psql.query(columnTypeQuery, (err, result) => {
+            if (err) {
+                return callback(err);
+            }
+            const pgType = result.rows[0].pg_typeof;
+            callback(null, getPGTypeName(pgType));
+        }, readOnlyTransaction);
+    }
+};

lib/cartodb/models/dataview/factory.js

@@ -1,18 +1,20 @@
-var dataviews = require('./');
+const dataviews = require('./');
 
-var DataviewFactory = {
-    dataviews: Object.keys(dataviews).reduce(function(allDataviews, dataviewClassName) {
-        allDataviews[dataviewClassName.toLowerCase()] = dataviews[dataviewClassName];
-        return allDataviews;
-    }, {}),
+module.exports = class DataviewFactory {
+    static get dataviews() {
+        return Object.keys(dataviews).reduce((allDataviews, dataviewClassName) => {
+            allDataviews[dataviewClassName.toLowerCase()] = dataviews[dataviewClassName];
+            return allDataviews;
+        }, {});
+    }
+
+    static getDataview (query, dataviewDefinition) {
+        const { type, options, sql } = dataviewDefinition;
 
-    getDataview: function(query, dataviewDefinition) {
-        var type = dataviewDefinition.type;
         if (!this.dataviews[type]) {
             throw new Error('Invalid dataview type: "' + type + '"');
         }
-        return new this.dataviews[type](query, dataviewDefinition.options, dataviewDefinition.sql);
+
+        return new this.dataviews[type](query, options, sql);
     }
 };
-
-module.exports = DataviewFactory;

lib/cartodb/models/dataview/formula.js

@@ -1,28 +1,16 @@
-var _ = require('underscore');
-var BaseWidget = require('./base');
-var debug = require('debug')('windshaft:widget:formula');
+const BaseDataview = require('./base');
+const debug = require('debug')('windshaft:dataview:formula');
+const utils = require('../../utils/query-utils');
 
-var dot = require('dot');
-dot.templateSettings.strip = false;
+const formulaQueryTpl = ctx =>
+`SELECT
+    ${ctx.operation}(${utils.handleFloatColumn(ctx)}) AS result,
+    ${utils.countNULLs(ctx)} AS nulls_count
+    ${ctx.isFloatColumn ? `,${utils.countInfinites(ctx)} AS infinities_count,` : ``}
+    ${ctx.isFloatColumn ? `${utils.countNaNs(ctx)} AS nans_count` : ``}
+FROM (${ctx.query}) __cdb_formula`;
 
-var formulaQueryTpl = dot.template([
-    'SELECT',
-    '  {{=it._operation}}({{=it._column}}) AS result,',
-    '  (SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
-    '  {{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls',
-    '      WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count',
-    '  ,(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls',
-    '      WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}',
-    'FROM ({{=it._query}}) _cdb_formula',
-    '{{?it._isFloatColumn && it._operation !== \'count\'}}WHERE',
-    '  {{=it._column}} != \'infinity\'::float',
-    'AND',
-    '  {{=it._column}} != \'-infinity\'::float',
-    'AND',
-    '  {{=it._column}} != \'NaN\'::float{{?}}'
-].join('\n'));
-
-var VALID_OPERATIONS = {
+const VALID_OPERATIONS = {
     count: true,
     avg: true,
     sum: true,
@@ -30,7 +18,7 @@ var VALID_OPERATIONS = {
     max: true
 };
 
-var TYPE = 'formula';
+const TYPE = 'formula';
 
 /**
  {
@@ -41,93 +29,90 @@ var TYPE = 'formula';
      }
  }
  */
-function Formula(query, options, queries) {
-    if (!_.isString(options.operation)) {
-        throw new Error('Formula expects `operation` in widget options');
+module.exports = class Formula extends BaseDataview {
+    constructor (query, options = {}, queries = {}) {
+        super();
+
+        this._checkOptions(options);
+
+        this.query = query;
+        this.queries = queries;
+        this.column = options.column || '1';
+        this.operation = options.operation;
+        this._isFloatColumn = null;
     }
 
-    if (!VALID_OPERATIONS[options.operation]) {
-        throw new Error("Formula does not support '" + options.operation + "' operation");
+    _checkOptions (options) {
+        if (typeof options.operation !== 'string') {
+            throw new Error(`Formula expects 'operation' in dataview options`);
+        }
+
+        if (!VALID_OPERATIONS[options.operation]) {
+            throw new Error(`Formula does not support '${options.operation}' operation`);
+        }
+
+        if (options.operation !== 'count' && typeof options.column !== 'string') {
+            throw new Error(`Formula expects 'column' in dataview options`);
+        }
     }
 
-    if (options.operation !== 'count' && !_.isString(options.column)) {
-        throw new Error('Formula expects `column` in widget options');
-    }
 
-    BaseWidget.apply(this);
+    sql (psql, override, callback) {
+        if (!callback) {
+            callback = override;
+            override = {};
+        }
 
-    this.query = query;
-    this.queries = queries;
-    this.column = options.column || '1';
-    this.operation = options.operation;
-    this._isFloatColumn = null;
-}
+        if (this._isFloatColumn === null) {
+            this._isFloatColumn = false;
+            this.getColumnType(psql, this.column, this.queries.no_filters, (err, type) => {
+                if (!err && !!type) {
+                    this._isFloatColumn = type.float;
+                }
+                this.sql(psql, override, callback);
+            });
+            return null;
+        }
 
-Formula.prototype = new BaseWidget();
-Formula.prototype.constructor = Formula;
-
-module.exports = Formula;
-
-Formula.prototype.sql = function(psql, override, callback) {
-    var self = this;
-
-    if (!callback) {
-        callback = override;
-        override = {};
-    }
-
-    if (this._isFloatColumn === null) {
-        this._isFloatColumn = false;
-        this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) {
-            if (!err && !!type) {
-                self._isFloatColumn = type.float;
-            }
-            self.sql(psql, override, callback);
+        const formulaSql = formulaQueryTpl({
+            isFloatColumn: this._isFloatColumn,
+            query: this.query,
+            operation: this.operation,
+            column: this.column
         });
-        return null;
+
+        debug(formulaSql);
+
+        return callback(null, formulaSql);
     }
 
-    var formulaSql = formulaQueryTpl({
-        _isFloatColumn: this._isFloatColumn,
-        _query: this.query,
-        _operation: this.operation,
-        _column: this.column
-    });
+    format (res) {
+        const {
+            result = 0,
+            nulls_count = 0,
+            nans_count,
+            infinities_count
+        } = res.rows[0] || {};
 
-    debug(formulaSql);
-
-    return callback(null, formulaSql);
-};
-
-Formula.prototype.format = function(result) {
-    var formattedResult = {
-        operation: this.operation,
-        result: 0,
-        nulls: 0,
-        nans: 0,
-        infinities: 0
-    };
-
-    if (result.rows.length) {
-        formattedResult.operation = this.operation;
-        formattedResult.result = result.rows[0].result;
-        formattedResult.nulls = result.rows[0].nulls_count;
-        formattedResult.nans = result.rows[0].nans_count;
-        formattedResult.infinities = result.rows[0].infinities_count;
+        return {
+            operation: this.operation,
+            result,
+            nulls: nulls_count,
+            nans: nans_count,
+            infinities: infinities_count
+        };
     }
 
-    return formattedResult;
-};
+    getType () {
+        return TYPE;
+    }
 
-Formula.prototype.getType = function() {
-    return TYPE;
-};
-
-Formula.prototype.toString = function() {
-    return JSON.stringify({
-        _type: TYPE,
-        _query: this.query,
-        _column: this.column,
-        _operation: this.operation
-    });
+    toString () {
+        return JSON.stringify({
+            _type: TYPE,
+            _query: this.query,
+            _column: this.column,
+            _operation: this.operation
+        });
+    }
 };

lib/cartodb/models/dataview/histogram.js

@@ -1,717 +1,72 @@
-var _ = require('underscore');
-var BaseWidget = require('./base');
-var debug = require('debug')('windshaft:dataview:histogram');
+const debug = require('debug')('windshaft:dataview:histogram');
+const NumericHistogram = require('./histograms/numeric-histogram');
+const DateHistogram = require('./histograms/date-histogram');
 
-var dot = require('dot');
-dot.templateSettings.strip = false;
+const DATE_HISTOGRAM = 'DateHistogram';
+const NUMERIC_HISTOGRAM = 'NumericHistogram';
 
-var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})");
+module.exports = class Histogram {
+    constructor (query, options, queries) {
+        this.query = query;
+        this.options = options || {};
+        this.queries = queries;
 
-var dateIntervalQueryTpl = dot.template([
-    'WITH',
-    '__cdb_dates AS (',
-    '    SELECT',
-    '        MAX({{=it.column}}::timestamp) AS __cdb_end,',
-    '        MIN({{=it.column}}::timestamp) AS __cdb_start',
-    '    FROM ({{=it.query}}) __cdb_source',
-    '),',
-    '__cdb_interval_in_days AS (',
-    '    SELECT' ,
-    '        DATE_PART(\'day\', __cdb_end - __cdb_start) AS __cdb_days',
-    '    FROM __cdb_dates',
-    '),',
-    '__cdb_interval_in_hours AS (',
-    '    SELECT',
-    '        __cdb_days * 24 + DATE_PART(\'hour\', __cdb_end - __cdb_start) AS __cdb_hours',
-    '    FROM __cdb_interval_in_days, __cdb_dates',
-    '),',
-    '__cdb_interval_in_minutes AS (',
-    '    SELECT',
-    '        __cdb_hours * 60 + DATE_PART(\'minute\', __cdb_end - __cdb_start) AS __cdb_minutes',
-    '    FROM __cdb_interval_in_hours, __cdb_dates',
-    '),',
-    '__cdb_interval_in_seconds AS (',
-    '    SELECT',
-    '        __cdb_minutes * 60 + DATE_PART(\'second\', __cdb_end - __cdb_start) AS __cdb_seconds',
-    '    FROM __cdb_interval_in_minutes, __cdb_dates',
-    ')',
-    'SELECT',
-    '    ROUND(__cdb_days / 365) AS year,',
-    '    ROUND(__cdb_days / 90) AS quarter,',
-    '    ROUND(__cdb_days / 30) AS month,',
-    '    ROUND(__cdb_days / 7) AS week,',
-    '    __cdb_days AS day,',
-    '    __cdb_hours AS hour,',
-    '    __cdb_minutes AS minute,',
-    '    __cdb_seconds AS second',
-    'FROM __cdb_interval_in_days, __cdb_interval_in_hours, __cdb_interval_in_minutes, __cdb_interval_in_seconds'
-].join('\n'));
-
-var MAX_INTERVAL_VALUE = 366;
-var BIN_MIN_NUMBER = 6;
-var BIN_MAX_NUMBER = 48;
-
-var filteredQueryTpl = dot.template([
-    '__cdb_filtered_source AS (',
-    '  SELECT *',
-    '  FROM ({{=it._query}}) __cdb_filtered_source_query',
-    '  WHERE',
-    '    {{=it._column}} IS NOT NULL',
-    '  {{?it._isFloatColumn}}AND',
-    '    {{=it._column}} != \'infinity\'::float',
-    '  AND',
-    '    {{=it._column}} != \'-infinity\'::float',
-    '  AND',
-    '    {{=it._column}} != \'NaN\'::float{{?}}',
-    ')'
-].join(' \n'));
-
-var basicsQueryTpl = dot.template([
-    '__cdb_basics AS (',
-    '  SELECT',
-    '    max({{=it._column}}) AS __cdb_max_val, min({{=it._column}}) AS __cdb_min_val,',
-    '    avg({{=it._column}}) AS __cdb_avg_val, count(1) AS __cdb_total_rows',
-    '  FROM __cdb_filtered_source',
-    ')'
-].join(' \n'));
-
-var overrideBasicsQueryTpl = dot.template([
-    '__cdb_basics AS (',
-    '  SELECT',
-    '    max({{=it._end}}) AS __cdb_max_val, min({{=it._start}}) AS __cdb_min_val,',
-    '    avg({{=it._column}}) AS __cdb_avg_val, count(1) AS __cdb_total_rows',
-    '  FROM __cdb_filtered_source',
-    ')'
-].join('\n'));
-
-var iqrQueryTpl = dot.template([
-    '__cdb_iqrange AS (',
-    '  SELECT max(quartile_max) - min(quartile_max) AS __cdb_iqr',
-    '  FROM (',
-    '    SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (',
-    '      SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}',
-    '    ) AS quartile',
-    '    FROM __cdb_filtered_source) _cdb_quartiles',
-    '    WHERE quartile = 1 or quartile = 3',
-    '    GROUP BY quartile',
-    '  ) __cdb_iqr',
-    ')'
-].join('\n'));
-
-var binsQueryTpl = dot.template([
-    '__cdb_bins AS (',
-    '  SELECT CASE WHEN __cdb_total_rows = 0 OR __cdb_iqr = 0',
-    '      THEN 1',
-    '      ELSE GREATEST(',
-    '        LEAST({{=it._minBins}}, CAST(__cdb_total_rows AS INT)),',
-    '        LEAST(',
-    '          CAST(((__cdb_max_val - __cdb_min_val) / (2 * __cdb_iqr * power(__cdb_total_rows, 1/3))) AS INT),',
-    '          {{=it._maxBins}}',
-    '        )',
-    '      )',
-    '    END AS __cdb_bins_number',
-    '  FROM __cdb_basics, __cdb_iqrange, __cdb_filtered_source',
-    '  LIMIT 1',
-    ')'
-].join('\n'));
-
-var overrideBinsQueryTpl = dot.template([
-    '__cdb_bins AS (',
-    '  SELECT {{=it._bins}} AS __cdb_bins_number',
-    ')'
-].join('\n'));
-
-var nullsQueryTpl = dot.template([
-    '__cdb_nulls AS (',
-    '  SELECT',
-    '    count(*) AS __cdb_nulls_count',
-    '  FROM ({{=it._query}}) __cdb_histogram_nulls',
-    '  WHERE {{=it._column}} IS NULL',
-    ')'
-].join('\n'));
-
-var infinitiesQueryTpl = dot.template([
-    '__cdb_infinities AS (',
-    '  SELECT',
-    '    count(*) AS __cdb_infinities_count',
-    '  FROM ({{=it._query}}) __cdb_infinities_query',
-    '  WHERE',
-    '    {{=it._column}} = \'infinity\'::float',
-    '  OR',
-    '    {{=it._column}} = \'-infinity\'::float',
-    ')'
-].join('\n'));
-
-var nansQueryTpl = dot.template([
-    '__cdb_nans AS (',
-    '  SELECT',
-    '    count(*) AS __cdb_nans_count',
-    '  FROM ({{=it._query}}) __cdb_nans_query',
-    '  WHERE {{=it._column}} = \'NaN\'::float',
-    ')'
-].join('\n'));
-
-var histogramQueryTpl = dot.template([
-    'SELECT',
-    '    (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width,',
-    '    __cdb_bins_number AS bins_number,',
-    '    __cdb_nulls_count AS nulls_count,',
-    '    {{?it._isFloatColumn}}__cdb_infinities_count AS infinities_count,',
-    '    __cdb_nans_count AS nans_count,{{?}}',
-    '    __cdb_avg_val AS avg_val,',
-    '    CASE WHEN __cdb_min_val = __cdb_max_val',
-    '      THEN 0',
-    '      ELSE GREATEST(',
-    '           1,',
-    '           LEAST(',
-    '               WIDTH_BUCKET({{=it._column}}, __cdb_min_val, __cdb_max_val, __cdb_bins_number),',
-    '               __cdb_bins_number',
-    '           )',
-    '      ) - 1',
-    '    END AS bin,',
-    '    min({{=it._column}})::numeric AS min,',
-    '    max({{=it._column}})::numeric AS max,',
-    '    avg({{=it._column}})::numeric AS avg,',
-    '    count(*) AS freq',
-    'FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls,',
-    '       __cdb_bins{{?it._isFloatColumn}}, __cdb_infinities, __cdb_nans{{?}}',
-    'GROUP BY bin, bins_number, bin_width, nulls_count,',
-    '  avg_val{{?it._isFloatColumn}}, infinities_count, nans_count{{?}}',
-    'ORDER BY bin'
-].join('\n'));
-
-var dateBasicsQueryTpl = dot.template([
-    '__cdb_basics AS (',
-    '    SELECT',
-    '        max(date_part(\'epoch\', {{=it._column}})) AS __cdb_max_val,',
-    '        min(date_part(\'epoch\', {{=it._column}})) AS __cdb_min_val,',
-    '        avg(date_part(\'epoch\', {{=it._column}})) AS __cdb_avg_val,',
-    '        min(date_trunc(',
-    '           \'{{=it._aggregation}}\', {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\'',
-    '        )) AS __cdb_start_date,',
-    '        max({{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\') AS __cdb_end_date,',
-    '        count(1) AS __cdb_total_rows',
-    '    FROM ({{=it._query}}) __cdb_basics_query',
-    ')'
-].join(' \n'));
-
-var dateOverrideBasicsQueryTpl = dot.template([
-    '__cdb_basics AS (',
-    '    SELECT',
-    '        max({{=it._end}}) AS __cdb_max_val,',
-    '        min({{=it._start}}) AS __cdb_min_val,',
-    '        avg(date_part(\'epoch\', {{=it._column}})) AS __cdb_avg_val,',
-    '        min(',
-    '           date_trunc(',
-    '               \'{{=it._aggregation}}\',',
-    '               TO_TIMESTAMP({{=it._start}})::timestamp AT TIME ZONE \'{{=it._offset}}\'',
-    '           )',
-    '        ) AS __cdb_start_date,',
-    '        max(',
-    '           TO_TIMESTAMP({{=it._end}})::timestamp AT TIME ZONE \'{{=it._offset}}\'',
-    '        ) AS __cdb_end_date,',
-    '        count(1) AS __cdb_total_rows',
-    '    FROM ({{=it._query}}) __cdb_basics_query',
-    ')'
-].join(' \n'));
-
-var dateBinsQueryTpl = dot.template([
-    '__cdb_bins AS (',
-    '    SELECT',
-    '        __cdb_bins_array,',
-    '        ARRAY_LENGTH(__cdb_bins_array, 1) AS __cdb_bins_number',
-    '    FROM (',
-    '       SELECT',
-    '           ARRAY(',
-    '               SELECT GENERATE_SERIES(',
-    '                   __cdb_start_date::timestamptz,',
-    '                   __cdb_end_date::timestamptz,',
-    '                   {{?it._aggregation==="quarter"}}\'3 month\'{{??}}\'1 {{=it._aggregation}}\'{{?}}::interval',
-    '                )',
-    '           ) AS __cdb_bins_array',
-    '       FROM __cdb_basics',
-    '    ) __cdb_bins_array_query',
-    ')'
-].join('\n'));
-
-var dateHistogramQueryTpl = dot.template([
-    'SELECT',
-    '    (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width,',
-    '    __cdb_bins_number AS bins_number,',
-    '    __cdb_nulls_count AS nulls_count,',
-    '    CASE WHEN __cdb_min_val = __cdb_max_val',
-    '      THEN 0',
-    '      ELSE GREATEST(1, LEAST(',
-    '        WIDTH_BUCKET(',
-    '          {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\',',
-    '          __cdb_bins_array',
-    '        ),',
-    '        __cdb_bins_number',
-    '      )) - 1',
-    '    END AS bin,',
-    '    min(',
-    '       date_part(',
-    '           \'epoch\', ',
-    '           date_trunc(',
-    '               \'{{=it._aggregation}}\', {{=it._column}}::timestamp AT TIME ZONE \'{{=it._offset}}\'',
-    '           ) AT TIME ZONE \'{{=it._offset}}\'',
-    '       )',
-    '    )::numeric AS timestamp,',
-    '    date_part(\'epoch\', __cdb_start_date)::numeric AS timestamp_start,',
-    '    min(date_part(\'epoch\', {{=it._column}}))::numeric AS min,',
-    '    max(date_part(\'epoch\', {{=it._column}}))::numeric AS max,',
-    '    avg(date_part(\'epoch\', {{=it._column}}))::numeric AS avg,',
-    '    count(*) AS freq',
-    'FROM ({{=it._query}}) __cdb_histogram, __cdb_basics, __cdb_bins, __cdb_nulls',
-    'WHERE date_part(\'epoch\', {{=it._column}}) IS NOT NULL',
-    'GROUP BY bin, bins_number, bin_width, nulls_count, timestamp_start',
-    'ORDER BY bin'
-].join('\n'));
-
-var TYPE = 'histogram';
-
-/**
-Numeric histogram:
-{
-    type: 'histogram',
-    options: {
-        column: 'name', // column data type: numeric
-        bins: 10 // OPTIONAL
-    }
-}
-
-Time series:
-{
-    type: 'histogram',
-    options: {
-        column: 'date', // column data type: date
-        aggregation: 'day' // OPTIONAL (if undefined then it'll be built as numeric)
-        offset: -7200 // OPTIONAL (UTC offset in seconds)
-    }
- }
- */
-function Histogram(query, options, queries) {
-    if (!_.isString(options.column)) {
-        throw new Error('Histogram expects `column` in widget options');
+        this.histogramImplementation = this._getHistogramImplementation();
     }
 
-    this.query = query;
-    this.queries = queries;
-    this.column = options.column;
-    this.bins = options.bins;
-    this.aggregation = options.aggregation;
-    this.offset = options.offset;
+    _getHistogramImplementation (override) {
+        let implementation = null;
 
-    this._columnType = null;
-}
-
-Histogram.prototype = new BaseWidget();
-Histogram.prototype.constructor = Histogram;
-
-module.exports = Histogram;
-
-Histogram.prototype.sql = function(psql, override, callback) {
-    var self = this;
-
-    if (!callback) {
-        callback = override;
-        override = {};
-    }
-
-    if (this._columnType === null) {
-        this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) {
-            // assume numeric, will fail later
-            self._columnType = 'numeric';
-            if (!err && !!type) {
-                self._columnType = Object.keys(type).find(function (key) {
-                    return type[key];
-                });
-            }
-            self.sql(psql, override, callback);
-        }, true); // use read-only transaction
-        return null;
-    }
-
-    this._buildQuery(psql, override, callback);
-};
-
-Histogram.prototype.isDateHistogram = function (override) {
-    return this._columnType === 'date' && (this.aggregation !== undefined || override.aggregation !== undefined);
-};
-
-Histogram.prototype._buildQuery = function (psql, override, callback) {
-    var filteredQuery, basicsQuery, binsQuery;
-    var _column = this.column;
-    var _query = this.query;
-
-    if (this.isDateHistogram(override)) {
-        return this._buildDateHistogramQuery(psql, override, callback);
-    }
-
-    if (this._columnType === 'date') {
-        _column = columnCastTpl({column: _column});
-    }
-
-    filteredQuery = filteredQueryTpl({
-        _isFloatColumn: this._columnType === 'float',
-        _query: _query,
-        _column: _column
-    });
-
-    if (this._shouldOverride(override)) {
-        debug('overriding with %j', override);
-        basicsQuery = overrideBasicsQueryTpl({
-            _query: _query,
-            _column: _column,
-            _start: getBinStart(override),
-            _end: getBinEnd(override)
-        });
-
-        binsQuery = [
-            overrideBinsQueryTpl({
-                _bins: override.bins
-            })
-        ].join(',\n');
-    } else {
-        basicsQuery = basicsQueryTpl({
-            _query: _query,
-            _column: _column
-        });
-
-        if (this._shouldOverrideBins(override)) {
-            binsQuery = [
-                overrideBinsQueryTpl({
-                    _bins: override.bins
-                })
-            ].join(',\n');
-        } else {
-            binsQuery = [
-                iqrQueryTpl({
-                    _query: _query,
-                    _column: _column
-                }),
-                binsQueryTpl({
-                    _query: _query,
-                    _minBins: BIN_MIN_NUMBER,
-                    _maxBins: BIN_MAX_NUMBER
-                })
-            ].join(',\n');
-        }
-    }
-
-    var cteSql = [
-        filteredQuery,
-        basicsQuery,
-        binsQuery,
-        nullsQueryTpl({
-            _query: _query,
-            _column: _column
-        })
-    ];
-
-    if (this._columnType === 'float') {
-        cteSql.push(
-            infinitiesQueryTpl({
-                _query: _query,
-                _column: _column
-            }),
-            nansQueryTpl({
-                _query: _query,
-                _column: _column
-            })
-        );
-    }
-
-    var histogramSql = [
-        "WITH",
-        cteSql.join(',\n'),
-        histogramQueryTpl({
-            _isFloatColumn: this._columnType === 'float',
-            _query: _query,
-            _column: _column
-        })
-    ].join('\n');
-
-    debug(histogramSql);
-
-    return callback(null, histogramSql);
-};
-
-Histogram.prototype._shouldOverride = function (override) {
-    return override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins');
-};
-
-Histogram.prototype._shouldOverrideBins = function (override) {
-    return override && _.has(override, 'bins');
-};
-
-var DATE_AGGREGATIONS = {
-    'auto': true,
-    'minute': true,
-    'hour': true,
-    'day': true,
-    'week': true,
-    'month': true,
-    'quarter': true,
-    'year': true
-};
-
-Histogram.prototype._buildDateHistogramQuery = function (psql, override, callback) {
-    var _column = this.column;
-    var _query = this.query;
-    var _aggregation = override && override.aggregation ? override.aggregation : this.aggregation;
-    var _offset = override && Number.isFinite(override.offset) ? override.offset : this.offset;
-
-    if (!DATE_AGGREGATIONS.hasOwnProperty(_aggregation)) {
-        return callback(new Error('Invalid aggregation value. Valid ones: ' +
-            Object.keys(DATE_AGGREGATIONS).join(', ')
-        ));
-    }
-
-    if (_aggregation === 'auto') {
-        this.getAutomaticAggregation(psql, function (err, aggregation) {
-            if (err || aggregation === 'none') {
-                this.aggregation = 'day';
-            } else {
-                this.aggregation = aggregation;
-            }
-            override.aggregation = this.aggregation;
-            this._buildDateHistogramQuery(psql, override, callback);
-        }.bind(this));
-        return null;
-    }
-
-    var dateBasicsQuery;
-
-    if (override && _.has(override, 'start') && _.has(override, 'end')) {
-        dateBasicsQuery = dateOverrideBasicsQueryTpl({
-            _query: _query,
-            _column: _column,
-            _aggregation: _aggregation,
-            _start: getBinStart(override),
-            _end: getBinEnd(override),
-            _offset: parseOffset(_offset, _aggregation)
-        });
-    } else {
-        dateBasicsQuery = dateBasicsQueryTpl({
-            _query: _query,
-            _column: _column,
-            _aggregation: _aggregation,
-            _offset: parseOffset(_offset, _aggregation)
-        });
-    }
-
-    var dateBinsQuery = [
-        dateBinsQueryTpl({
-            _aggregation: _aggregation
-        })
-    ].join(',\n');
-
-    var nullsQuery = nullsQueryTpl({
-        _query: _query,
-        _column: _column
-    });
-
-    var dateHistogramQuery = dateHistogramQueryTpl({
-        _query: _query,
-        _column: _column,
-        _aggregation: _aggregation,
-        _offset: parseOffset(_offset, _aggregation)
-    });
-
-    var histogramSql = [
-        "WITH",
-        [
-            dateBasicsQuery,
-            dateBinsQuery,
-            nullsQuery
-        ].join(',\n'),
-        dateHistogramQuery
-    ].join('\n');
-
-    debug(histogramSql);
-
-    return callback(null, histogramSql);
-};
-
-Histogram.prototype.getAutomaticAggregation = function (psql, callback) {
-    var dateIntervalQuery = dateIntervalQueryTpl({
-        query: this.query,
-        column: this.column
-    });
-
-    debug(dateIntervalQuery);
-
-    psql.query(dateIntervalQuery, function (err, result) {
-        if (err) {
-            return callback(err);
+        switch (this._getHistogramSubtype(override)) {
+            case DATE_HISTOGRAM:
+                debug('Delegating to DateHistogram with options: %j and overriding: %j', this.options, override);
+                implementation = new DateHistogram(this.query, this.options, this.queries);
+                break;
+            case NUMERIC_HISTOGRAM:
+                debug('Delegating to NumericHistogram with options: %j and overriding: %j', this.options, override);
+                implementation = new NumericHistogram(this.query, this.options, this.queries);
+                break;
+            default:
+                throw new Error('Unsupported Histogram type');
         }
 
-        var aggegations = result.rows[0];
-        var aggregation = Object.keys(aggegations)
-            .map(function (key) {
-                return {
-                    name: key,
-                    value: aggegations[key]
-                };
-            })
-            .reduce(function (closer, current) {
-                if (current.value > MAX_INTERVAL_VALUE) {
-                    return closer;
-                }
+        return implementation;
+    }
 
-                var closerDiff = MAX_INTERVAL_VALUE - closer.value;
-                var currentDiff = MAX_INTERVAL_VALUE - current.value;
-
-                if (Number.isFinite(current.value) && closerDiff > currentDiff) {
-                    return current;
-                }
-
-                return closer;
-            }, { name: 'none', value: -1 });
-
-        callback(null, aggregation.name);
-    });
-};
-
-Histogram.prototype.format = function(result, override) {
-    override = override || {};
-    var buckets = [];
-
-    var binsCount = getBinsCount(override);
-    var width = getWidth(override);
-    var binsStart = getBinStart(override);
-    var nulls = 0;
-    var infinities = 0;
-    var nans = 0;
-    var avg;
-    var timestampStart;
-    var aggregation;
-    var offset;
-
-    if (result.rows.length) {
-        var firstRow = result.rows[0];
-        binsCount = firstRow.bins_number;
-        width = firstRow.bin_width || width;
-        avg = firstRow.avg_val;
-        nulls = firstRow.nulls_count;
-        timestampStart = firstRow.timestamp_start;
-        infinities = firstRow.infinities_count;
-        nans = firstRow.nans_count;
-        binsStart = populateBinStart(override, firstRow);
-
-        if (Number.isFinite(timestampStart)) {
-            aggregation = getAggregation(override, this.aggregation);
-            offset = getOffset(override, this.offset);
+    _getHistogramSubtype (override) {
+        if(this._isDateHistogram(override)) {
+            return DATE_HISTOGRAM;
         }
 
-        buckets = result.rows.map(function(row) {
-            return _.omit(
-                row,
-                'bins_number',
-                'bin_width',
-                'nulls_count',
-                'infinities_count',
-                'nans_count',
-                'avg_val',
-                'timestamp_start'
-            );
-        });
+        return NUMERIC_HISTOGRAM;
     }
 
-    return {
-        aggregation: aggregation,
-        offset: offset,
-        timestamp_start: timestampStart,
-        bin_width: width,
-        bins_count: binsCount,
-        bins_start: binsStart,
-        nulls: nulls,
-        infinities: infinities,
-        nans: nans,
-        avg: avg,
-        bins: buckets
-    };
-};
-
-function getAggregation(override, aggregation) {
-    return override && override.aggregation ? override.aggregation : aggregation;
-}
-
-function getOffset(override, offset) {
-    if (override && override.offset) {
-        return override.offset;
-    }
-    if (offset) {
-        return offset;
-    }
-
-    return 0;
-}
-
-function getBinStart(override) {
-    if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) {
-        return Math.min(override.start, override.end);
-    }
-    return override.start || 0;
-}
-
-function getBinEnd(override) {
-    if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) {
-        return Math.max(override.start, override.end);
-    }
-    return override.end || 0;
-}
-
-function getBinsCount(override) {
-    return override.bins || 0;
-}
-
-function getWidth(override) {
-    var width = 0;
-    var binsCount = override.bins;
-
-    if (binsCount && Number.isFinite(override.start) && Number.isFinite(override.end)) {
-        width = (override.end - override.start) / binsCount;
-    }
-
-    return width;
-}
-
-function parseOffset(offset, aggregation) {
-    if (!offset) {
-        return '0';
-    }
-    if (aggregation === 'hour' || aggregation === 'minute') {
-        return '0';
-    }
-
-    var offsetInHours = Math.ceil(offset / 3600);
-    return '' + offsetInHours;
-}
-
-function populateBinStart(override, firstRow) {
-    var binStart;
-
-    if (firstRow.hasOwnProperty('timestamp')) {
-        binStart = firstRow.timestamp;
-    } else if (override.hasOwnProperty('start')) {
-        binStart = getBinStart(override);
-    } else {
-        binStart = firstRow.min;
-    }
-
-    return binStart;
-}
-
-Histogram.prototype.getType = function() {
-    return TYPE;
-};
-
-Histogram.prototype.toString = function() {
-    return JSON.stringify({
-        _type: TYPE,
-        _column: this.column,
-        _query: this.query
-    });
+    _isDateHistogram (override = {}) {
+        return (this.options.hasOwnProperty('aggregation') || override.hasOwnProperty('aggregation'));
+    }
+
+    getResult (psql, override, callback) {
+        this.histogramImplementation = this._getHistogramImplementation(override);
+        this.histogramImplementation.getResult(psql, override, callback);
+    }
+
+    // In order to keep previous behaviour with overviews,
+    // we have to expose the following methods to bypass
+    // the concrete overview implementation
+
+    sql (psql, override, callback) {
+        this.histogramImplementation.sql(psql, override, callback);
+    }
+
+    format (result, override) {
+        return this.histogramImplementation.format(result, override);
+    }
+
+    getType () {
+        return this.histogramImplementation.getType();
+    }
+
+    toString () {
+        return this.histogramImplementation.toString();
+    }
 };

lib/cartodb/models/dataview/histograms/base-histogram.js

@@ -0,0 +1,85 @@
+const BaseDataview = require('../base');
+
+const TYPE = 'histogram';
+
+module.exports = class BaseHistogram extends BaseDataview {
+    constructor (query, options, queries) {
+        super();
+
+        if (typeof options.column !== 'string') {
+            throw new Error('Histogram expects `column` in widget options');
+        }
+
+        this.query = query;
+        this.queries = queries;
+        this.column = options.column;
+        this.bins = options.bins;
+
+        this._columnType = null;
+    }
+
+    sql (psql, override, callback) {
+        if (!callback) {
+            callback = override;
+            override = {};
+        }
+
+        if (this._columnType === null) {
+            this.getColumnType(psql, this.column, this.queries.no_filters, (err, type) => {
+                // assume numeric, will fail later
+                this._columnType = 'numeric';
+                if (!err && !!type) {
+                    this._columnType = Object.keys(type).find(function (key) {
+                        return type[key];
+                    });
+                }
+                this.sql(psql, override, callback);
+            }, true); // use read-only transaction
+            return null;
+        }
+
+        return this._buildQuery(psql, override, callback);
+    }
+
+    format (result, override) {
+        const histogram = this._getSummary(result, override);
+        histogram.bins = this._getBuckets(result);
+        return histogram;
+    }
+
+    getType () {
+        return TYPE;
+    }
+
+    toString () {
+        return JSON.stringify({
+            _type: TYPE,
+            _column: this.column,
+            _query: this.query
+        });
+    }
+
+    _hasOverridenRange (override) {
+        return override && override.hasOwnProperty('start') && override.hasOwnProperty('end');
+    }
+
+    _getBinStart (override = {}) {
+        if (this._hasOverridenRange(override)) {
+            return Math.min(override.start, override.end);
+        }
+
+        return override.start || 0;
+    }
+
+    _getBinEnd (override = {}) {
+        if (this._hasOverridenRange(override)) {
+            return Math.max(override.start, override.end);
+        }
+
+        return override.end || 0;
+    }
+
+    _getBinsCount (override = {}) {
+        return override.bins || 0;
+    }
+};

lib/cartodb/models/dataview/histograms/date-histogram.js

@@ -0,0 +1,322 @@
+const BaseHistogram = require('./base-histogram');
+const debug = require('debug')('windshaft:dataview:date-histogram');
+const utils = require('../../../utils/query-utils');
+
+/**
+ * Gets the name of a timezone with the same offset as the required
+ * using the pg_timezone_names table. We do this because it's simpler to pass
+ * the name than to pass the offset itself as PostgreSQL uses different
+ * sign convention. For example: TIME ZONE 'CET' is equal to TIME ZONE 'UTC-1',
+ * not 'UTC+1' which would be expected.
+ * Gives priority to Etc/GMT±N timezones but still support odd offsets like 8.5
+ * hours for Asia/Pyongyang.
+ * It also makes it easier to, in the future, support the input of expected timezone
+ * instead of the offset; that is using 'Europe/Madrid' instead of
+ * '+3600' or '+7200'. The daylight saving status can be handled by postgres.
+ */
+const offsetNameQueryTpl = ctx => `
+WITH __wd_tz AS
+(
+    SELECT name
+    FROM pg_timezone_names
+    WHERE utc_offset = interval '${ctx.offset} hours'
+    ORDER BY CASE WHEN name LIKE 'Etc/GMT%' THEN 0 ELSE 1 END
+    LIMIT 1
+),`;
+
+/**
+ * Function to get the subquery that places each row in its bin depending on
+ * the aggregation. Since the data stored is in epoch we need to adapt it to
+ * our timezone so when calling date_trunc it falls into the correct bin
+ */
+function dataBucketsQuery(ctx) {
+    var condition_str = '';
+
+    if (ctx.start !== 0) {
+        condition_str = `WHERE ${ctx.column} >= to_timestamp(${ctx.start})`;
+    }
+    if (ctx.end !== 0) {
+        if (condition_str === '') {
+            condition_str = `WHERE ${ctx.column} <= to_timestamp(${ctx.end})`;
+        }
+        else {
+            condition_str += ` and ${ctx.column} <= to_timestamp(${ctx.end})`;
+        }
+    }
+
+    return `
+__wd_buckets AS
+(
+    SELECT
+        date_trunc('${ctx.aggregation}', timezone(__wd_tz.name, ${ctx.column}::timestamptz)) as timestamp,
+        count(*) as freq,
+        ${utils.countNULLs(ctx)} as nulls_count
+    FROM
+    (
+        ${ctx.query}
+    ) __source, __wd_tz
+    ${condition_str}
+    GROUP BY 1, __wd_tz.name
+),`;
+}
+
+/**
+ * Function that generates an array with all the possible bins between the
+ * start and end date. If not provided we use the min and max generated from
+ * the dataBucketsQuery
+ */
+function allBucketsArrayQuery(ctx) {
+    var extra_from = ``;
+    var series_start = ``;
+    var series_end = ``;
+
+    if (ctx.start === 0) {
+        extra_from = `, __wd_buckets GROUP BY __wd_tz.name`;
+        series_start = `min(__wd_buckets.timestamp)`;
+    } else {
+        series_start = `date_trunc('${ctx.aggregation}', timezone(__wd_tz.name, to_timestamp(${ctx.start})))`;
+    }
+
+    if (ctx.end === 0) {
+        extra_from = `, __wd_buckets GROUP BY __wd_tz.name`;
+        series_end = `max(__wd_buckets.timestamp)`;
+    } else {
+        series_end = `date_trunc('${ctx.aggregation}', timezone(__wd_tz.name, to_timestamp(${ctx.end})))`;
+    }
+
+    return `
+__wd_all_buckets AS
+(
+    SELECT ARRAY(
+        SELECT
+            generate_series(
+                    ${series_start},
+                    ${series_end},
+                    interval '${ctx.interval}') as bin_start
+            FROM __wd_tz${extra_from}
+    ) as bins
+)`;
+}
+
+const dateIntervalQueryTpl = ctx => `
+    WITH
+    __cdb_dates AS (
+        SELECT
+            MAX(${ctx.column}::timestamp) AS __cdb_end,
+            MIN(${ctx.column}::timestamp) AS __cdb_start
+        FROM (${ctx.query}) __cdb_source
+    ),
+    __cdb_interval_in_days AS (
+        SELECT
+            DATE_PART('day', __cdb_end - __cdb_start) AS __cdb_days
+        FROM __cdb_dates
+    ),
+    __cdb_interval_in_hours AS (
+        SELECT
+            __cdb_days * 24 + DATE_PART('hour', __cdb_end - __cdb_start) AS __cdb_hours
+        FROM __cdb_interval_in_days, __cdb_dates
+    ),
+    __cdb_interval_in_minutes AS (
+        SELECT
+            __cdb_hours * 60 + DATE_PART('minute', __cdb_end - __cdb_start) AS __cdb_minutes
+        FROM __cdb_interval_in_hours, __cdb_dates
+    ),
+    __cdb_interval_in_seconds AS (
+        SELECT
+            __cdb_minutes * 60 + DATE_PART('second', __cdb_end - __cdb_start) AS __cdb_seconds
+        FROM __cdb_interval_in_minutes, __cdb_dates
+    )
+    SELECT
+        ROUND(__cdb_days / 365243) AS millennium,
+        ROUND(__cdb_days / 36525) AS century,
+        ROUND(__cdb_days / 3652) AS decade,
+        ROUND(__cdb_days / 365) AS year,
+        ROUND(__cdb_days / 91) AS quarter,
+        ROUND(__cdb_days / 30) AS month,
+        ROUND(__cdb_days / 7) AS week,
+        __cdb_days AS day,
+        __cdb_hours AS hour,
+        __cdb_minutes AS minute,
+        __cdb_seconds AS second
+    FROM __cdb_interval_in_days, __cdb_interval_in_hours, __cdb_interval_in_minutes, __cdb_interval_in_seconds
+`;
+
+/** Constant to switch between aggregations in auto mode */
+const MAX_INTERVAL_VALUE = 100;
+
+const DATE_AGGREGATIONS = {
+    'auto': true,
+    'second' : true,
+    'minute': true,
+    'hour': true,
+    'day': true,
+    'week': true,
+    'month': true,
+    'quarter': true,
+    'year': true,
+    'decade' : true,
+    'century' : true,
+    'millennium' : true
+};
+
+/**
+    date_histogram: {
+        type: 'histogram',
+        options: {
+            column: 'date', // column data type: date
+            aggregation: 'day' // MANDATORY
+            offset: -7200 // OPTIONAL (UTC offset in seconds)
+        }
+    }
+*/
+module.exports = class DateHistogram extends BaseHistogram {
+    constructor (query, options, queries) {
+        super(query, options, queries);
+
+        this.aggregation = options.aggregation;
+        this.offset = options.offset;
+    }
+
+    _buildQueryTpl (ctx) {
+        return `
+${offsetNameQueryTpl(ctx)}
+${dataBucketsQuery(ctx)}
+${allBucketsArrayQuery(ctx)}
+SELECT
+    array_position(__wd_all_buckets.bins, __wd_buckets.timestamp) - 1 as bin,
+    date_part('epoch', timezone(__wd_tz.name, __wd_buckets.timestamp)) AS timestamp,
+    __wd_buckets.freq as freq,
+    date_part('epoch', timezone(__wd_tz.name, (__wd_all_buckets.bins)[1])) as timestamp_start,
+    array_length(__wd_all_buckets.bins, 1) as bins_number,
+    date_part('epoch', interval '${ctx.interval}') as bin_width,
+    __wd_buckets.nulls_count as nulls_count
+FROM __wd_buckets, __wd_all_buckets, __wd_tz
+GROUP BY __wd_tz.name, __wd_all_buckets.bins, __wd_buckets.timestamp, __wd_buckets.nulls_count, __wd_buckets.freq
+ORDER BY bin ASC;
+`;
+    }
+
+    _buildQuery (psql, override, callback) {
+        if (!this._isValidAggregation(override)) {
+            return callback(new Error('Invalid aggregation value. Valid ones: ' +
+                Object.keys(DATE_AGGREGATIONS).join(', ')
+            ));
+        }
+
+        if (this._getAggregation(override) === 'auto') {
+            this._getAutomaticAggregation(psql, function (err, aggregation) {
+                if (err || aggregation === 'none') {
+                    this.aggregation = 'day';
+                } else {
+                    this.aggregation = aggregation;
+                }
+                override.aggregation = this.aggregation;
+                this._buildQuery(psql, override, callback);
+            }.bind(this));
+            return null;
+        }
+
+        var interval = this._getAggregation(override) === 'quarter' ?
+            '3 months' : '1 ' + this._getAggregation(override);
+
+        const histogramSql = this._buildQueryTpl({
+            override: override,
+            query: this.query,
+            column: this.column,
+            aggregation: this._getAggregation(override),
+            start: this._getBinStart(override),
+            end: this._getBinEnd(override),
+            offset: this._parseOffset(override),
+            interval: interval
+        });
+
+        debug(histogramSql);
+
+        return callback(null, histogramSql);
+    }
+
+    _isValidAggregation (override) {
+        return DATE_AGGREGATIONS.hasOwnProperty(this._getAggregation(override));
+    }
+
+    _getAutomaticAggregation (psql, callback) {
+        const dateIntervalQuery = dateIntervalQueryTpl({
+            query: this.query,
+            column: this.column
+        });
+
+        psql.query(dateIntervalQuery, function (err, result) {
+            if (err) {
+                return callback(err);
+            }
+
+            const aggregations = result.rows[0];
+            const aggregation = Object.keys(aggregations)
+                .map(key => ({ name: key, value: aggregations[key] }))
+                .reduce((closer, current) => {
+                    if (current.value > MAX_INTERVAL_VALUE) {
+                        return closer;
+                    }
+
+                    const closerDiff = MAX_INTERVAL_VALUE - closer.value;
+                    const currentDiff = MAX_INTERVAL_VALUE - current.value;
+
+                    if (Number.isFinite(current.value) && closerDiff > currentDiff) {
+                        return current;
+                    }
+
+                    return closer;
+                }, { name: 'none', value: -1 });
+
+            callback(null, aggregation.name);
+        });
+    }
+
+    _getSummary (result, override) {
+        const firstRow = result.rows[0] || {};
+
+        return {
+            aggregation: this._getAggregation(override),
+            offset: this._getOffset(override),
+            timestamp_start: firstRow.timestamp_start,
+
+            bin_width: firstRow.bin_width || 0,
+            bins_count: firstRow.bins_number || 0,
+            bins_start: firstRow.timestamp,
+            nulls: firstRow.nulls_count,
+            infinities: firstRow.infinities_count,
+            nans: firstRow.nans_count,
+            avg: firstRow.avg_val
+        };
+    }
+
+    _getBuckets (result) {
+        result.rows.forEach(function(row) {
+            row.min = row.max = row.avg = row.timestamp;
+        });
+
+        return result.rows.map(({ bin, min, max, avg, freq, timestamp }) => ({ bin, min, max, avg, freq, timestamp }));
+    }
+
+    _getAggregation (override = {}) {
+        return override.aggregation ? override.aggregation : this.aggregation;
+    }
+
+    _getOffset (override = {}) {
+        return Number.isFinite(override.offset) ? override.offset : (this.offset || 0);
+    }
+
+    _parseOffset (override) {
+        if (this._shouldIgnoreOffset(override)) {
+            return '0';
+        }
+
+        const offsetInHours = Math.ceil(this._getOffset(override) / 3600);
+
+        return '' + offsetInHours;
+    }
+
+    _shouldIgnoreOffset (override) {
+        return (this._getAggregation(override) === 'hour' || this._getAggregation(override) === 'minute');
+    }
+};

lib/cartodb/models/dataview/histograms/numeric-histogram.js

@@ -0,0 +1,195 @@
+const BaseHistogram = require('./base-histogram');
+const debug = require('debug')('windshaft:dataview:numeric-histogram');
+const utils = require('../../../utils/query-utils');
+
+/** Query to get min and max values from the query */
+const irqQueryTpl = ctx => `
+    __cdb_filtered_source AS (
+        SELECT *
+        FROM (${ctx.query}) __cdb_filtered_source_query
+        WHERE ${utils.handleFloatColumn(ctx)} IS NOT NULL
+    ),
+    __cdb_basics AS (
+        SELECT
+            max(${ctx.column}) AS __cdb_max_val,
+            min(${ctx.column}) AS __cdb_min_val,
+            count(1) AS __cdb_total_rows
+        FROM __cdb_filtered_source
+    )
+`;
+
+/* Query to calculate the number of bins (needs irqQueryTpl before it*/
+const binsQueryTpl = ctx => `
+    __cdb_iqrange AS (
+        SELECT max(quartile_max) - min(quartile_max) AS __cdb_iqr
+        FROM (
+            SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (
+                SELECT ${ctx.column} AS _cdb_iqr_column, ntile(4) over (order by ${ctx.column}
+            ) AS quartile
+            FROM __cdb_filtered_source) _cdb_quartiles
+            WHERE quartile = 1 or quartile = 3
+            GROUP BY 1
+        ) __cdb_iqr
+    ),
+    __cdb_bins AS (
+        SELECT
+            CASE WHEN __cdb_total_rows = 0 OR __cdb_iqr = 0
+            THEN 1
+            ELSE GREATEST(
+                LEAST(${ctx.minBins}, CAST(__cdb_total_rows AS INT)),
+                LEAST(
+                    CAST(((__cdb_max_val - __cdb_min_val) / (2 * __cdb_iqr * power(__cdb_total_rows, 1/3))) AS INT),
+                    ${ctx.maxBins}
+                )
+            )
+            END AS __cdb_bins_number
+        FROM __cdb_basics, __cdb_iqrange, __cdb_filtered_source
+        LIMIT 1
+    )
+`;
+
+const BIN_MIN_NUMBER = 6;
+const BIN_MAX_NUMBER = 48;
+
+/**
+Numeric histogram:
+{
+    type: 'histogram',
+    options: {
+        column: 'name', // column data type: numeric
+        bins: 10 // OPTIONAL
+    }
+}
+*/
+module.exports = class NumericHistogram extends BaseHistogram {
+    constructor (query, options, queries) {
+        super(query, options, queries);
+    }
+
+    _buildQuery (psql, override, callback) {
+        const histogramSql = this._buildQueryTpl({
+            column: this._columnType === 'date' ? utils.columnCastTpl({ column: this.column }) : this.column,
+            isFloatColumn: this._columnType === 'float',
+            query: this.query,
+            start: this._getBinStart(override),
+            end: this._getBinEnd(override),
+            bins: this._getBinsCount(override),
+            minBins: BIN_MIN_NUMBER,
+            maxBins: BIN_MAX_NUMBER
+        });
+
+        debug(histogramSql);
+
+        return callback(null, histogramSql);
+    }
+
+
+/**
+ * ctx: Object with the following values
+ * ctx.column -- Column for the histogram
+ * ctx.isFloatColumn - Whether the column is float or not
+ * ctx.query -- Subquery to extract data
+ * ctx.start -- Start value for the bins. [>= end to force calculation]
+ * ctx.end -- End value for the bins.
+ * ctx.bins -- Numbers of bins to generate [<0 to force calculation]
+ * ctx.minBins - If !full min bins to calculate [Optional]
+ * ctx.maxBins - If !full max bins to calculate [Optional]
+ */
+    _buildQueryTpl (ctx) {
+        var extra_tables = ``;
+        var extra_queries = ``;
+        var extra_groupby = ``;
+
+        if (ctx.start >= ctx.end) {
+            ctx.end = `__cdb_basics.__cdb_max_val`;
+            ctx.start = `__cdb_basics.__cdb_min_val`;
+            extra_groupby = `, __cdb_basics.__cdb_max_val, __cdb_basics.__cdb_min_val`;
+            extra_tables = `, __cdb_basics`;
+            extra_queries = `WITH ${irqQueryTpl(ctx)}`;
+        }
+
+        if (ctx.bins <= 0) {
+            ctx.bins = `__cdb_bins.__cdb_bins_number`;
+            extra_groupby += `, __cdb_bins.__cdb_bins_number`;
+            extra_tables += `, __cdb_bins`;
+            extra_queries = `WITH ${irqQueryTpl(ctx)}, ${binsQueryTpl(ctx)}`;
+        }
+
+        return `
+${extra_queries}
+SELECT
+    (${ctx.end} - ${ctx.start}) / ${ctx.bins}::float AS bin_width,
+    ${ctx.bins} as bins_number,
+    ${utils.countNULLs(ctx)} AS nulls_count,
+    ${utils.countInfinites(ctx)} AS infinities_count,
+    ${utils.countNaNs(ctx)} AS nans_count,
+    min(${utils.handleFloatColumn(ctx)}) AS min,
+    max(${utils.handleFloatColumn(ctx)}) AS max,
+    avg(${utils.handleFloatColumn(ctx)}) AS avg,
+    sum(CASE WHEN (${utils.handleFloatColumn(ctx)} is not NULL) THEN 1 ELSE 0 END) as freq,
+    CASE WHEN ${ctx.start} = ${ctx.end}
+        THEN 0
+        ELSE GREATEST(1, LEAST(
+                    ${ctx.bins},
+                    WIDTH_BUCKET(${utils.handleFloatColumn(ctx)}, ${ctx.start}, ${ctx.end}, ${ctx.bins}))) - 1
+    END AS bin
+FROM
+(
+    ${ctx.query}
+) __cdb_filtered_source_query${extra_tables}
+GROUP BY 10${extra_groupby}
+ORDER BY 10;`;
+    }
+
+    _hasOverridenBins (override) {
+        return override && override.hasOwnProperty('bins');
+    }
+
+    _getSummary (result, override) {
+        const firstRow = result.rows[0] || {};
+
+        var total_nulls = 0;
+        var total_infinities = 0;
+        var total_nans = 0;
+        var total_avg = 0;
+        var total_count = 0;
+
+        result.rows.forEach(function(row) {
+            total_nulls += row.nulls_count;
+            total_infinities += row.infinities_count;
+            total_nans += row.nans_count;
+            total_avg += row.avg * row.freq;
+            total_count += row.freq;
+        });
+        if (total_count !== 0) {
+            total_avg /= total_count;
+        }
+
+        return {
+            bin_width: firstRow.bin_width,
+            bins_count: firstRow.bins_number,
+            bins_start: this._populateBinStart(firstRow, override),
+            nulls: total_nulls,
+            infinities: total_infinities,
+            nans: total_nans,
+            avg: total_avg
+        };
+    }
+
+    _getBuckets (result) {
+        return result.rows.map(({ bin, min, max, avg, freq }) => ({ bin, min, max, avg, freq }));
+    }
+
+    _populateBinStart (firstRow, override = {}) {
+        let binStart;
+
+        if (override.hasOwnProperty('start')) {
+            binStart = this._getBinStart(override);
+        } else {
+            binStart = firstRow.min;
+        }
+
+        return binStart;
+    }
+
+};

lib/cartodb/models/dataview/index.js

@@ -1,6 +1,5 @@
 module.exports = {
     Aggregation: require('./aggregation'),
     Formula: require('./formula'),
-    Histogram: require('./histogram'),
-    List: require('./list')
+    Histogram: require('./histogram')
 };

lib/cartodb/models/dataview/list.js

@@ -1,66 +0,0 @@
-var dot = require('dot');
-dot.templateSettings.strip = false;
-
-var BaseWidget = require('./base');
-
-var TYPE = 'list';
-
-var listSqlTpl = dot.template('select {{=it._columns}} from ({{=it._query}}) as _cdb_list');
-
-/**
-{
-    type: 'list',
-    options: {
-        columns: ['name', 'description']
-    }
-}
-*/
-
-function List(query, options) {
-    options = options || {};
-
-    if (!Array.isArray(options.columns)) {
-        throw new Error('List expects `columns` array in widget options');
-    }
-
-    BaseWidget.apply(this);
-
-    this.query = query;
-    this.columns = options.columns;
-}
-
-List.prototype = new BaseWidget();
-List.prototype.constructor = List;
-
-module.exports = List;
-
-List.prototype.sql = function(psql, override, callback) {
-    if (!callback) {
-        callback = override;
-    }
-
-    var listSql = listSqlTpl({
-        _query: this.query,
-        _columns: this.columns.join(', ')
-    });
-
-    return callback(null, listSql);
-};
-
-List.prototype.format = function(result) {
-    return {
-        rows: result.rows
-    };
-};
-
-List.prototype.getType = function() {
-    return TYPE;
-};
-
-List.prototype.toString = function() {
-    return JSON.stringify({
-        _type: TYPE,
-        _query: this.query,
-        _columns: this.columns.join(', ')
-    });
-};

lib/cartodb/models/dataview/overviews/formula.js

@@ -1,55 +1,38 @@
 var BaseOverviewsDataview = require('./base');
 var BaseDataview = require('../formula');
 var debug = require('debug')('windshaft:widget:formula:overview');
+const utils = require('../../../utils/query-utils');
 
 var dot = require('dot');
 dot.templateSettings.strip = false;
 
-var formulaQueryTpls = {
-    'count': dot.template([
-        'SELECT',
-        'sum(_feature_count) AS result,',
-        '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
-        '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities',
-        '  WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count,',
-        '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans',
-        '  WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}',
-        'FROM ({{=it._query}}) _cdb_formula'
-    ].join('\n')),
-    'sum': dot.template([
-        'SELECT',
-        'sum({{=it._column}}*_feature_count) AS result,',
-        '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
-        '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities',
-        '  WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count',
-        ',(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans',
-        '  WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}',
-        'FROM ({{=it._query}}) _cdb_formula',
-        '{{?it._isFloatColumn}}WHERE',
-        '  {{=it._column}} != \'infinity\'::float',
-        'AND',
-        '  {{=it._column}} != \'-infinity\'::float',
-        'AND',
-        '  {{=it._column}} != \'NaN\'::float{{?}}'
-    ].join('\n')),
-    'avg': dot.template([
-        'SELECT',
-        'sum({{=it._column}}*_feature_count)/sum(_feature_count) AS result,',
-        '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
-        '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities',
-        '  WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count',
-        ',(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans',
-        '  WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}',
-        'FROM ({{=it._query}}) _cdb_formula',
-        '{{?it._isFloatColumn}}WHERE',
-        '  {{=it._column}} != \'infinity\'::float',
-        'AND',
-        '  {{=it._column}} != \'-infinity\'::float',
-        'AND',
-        '  {{=it._column}} != \'NaN\'::float{{?}}'
-    ].join('\n')),
+const VALID_OPERATIONS = {
+    count: true,
+    sum: true,
+    avg: true
 };
 
+/** Formulae to calculate the end result using _feature_count from the overview table*/
+function dataviewResult(ctx) {
+    switch (ctx.operation) {
+        case 'count':
+            return `sum(_feature_count)`;
+        case 'sum':
+            return `sum(${utils.handleFloatColumn(ctx)}*_feature_count)`;
+        case 'avg':
+            return `sum(${utils.handleFloatColumn(ctx)}*_feature_count)/sum(_feature_count) `;
+    }
+    return `${ctx.operation}(${utils.handleFloatColumn(ctx)})`;
+}
+
+const formulaQueryTpl = ctx =>
+`SELECT
+    ${dataviewResult(ctx)} AS result,
+    ${utils.countNULLs(ctx)} AS nulls_count
+    ${ctx.isFloatColumn ? `,${utils.countInfinites(ctx)} AS infinities_count,` : ``}
+    ${ctx.isFloatColumn ? `${utils.countNaNs(ctx)} AS nans_count` : ``}
+FROM (${ctx.query}) __cdb_formula`;
+
 function Formula(query, options, queryRewriter, queryRewriteData, params, queries) {
     BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries);
     this.column = options.column || '1';
@@ -65,36 +48,31 @@ module.exports = Formula;
 
 Formula.prototype.sql = function (psql, override, callback) {
     var self = this;
-    var formulaQueryTpl = formulaQueryTpls[this.operation];
-
-    if (formulaQueryTpl) {
-        // supported formula for use with overviews
-        if (this._isFloatColumn === null) {
-            this._isFloatColumn = false;
-            this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) {
-                if (!err && !!type) {
-                    self._isFloatColumn = type.float;
-                }
-                self.sql(psql, override, callback);
-            });
-            return null;
-        }
-
-        var formulaSql = formulaQueryTpl({
-            _isFloatColumn: this._isFloatColumn,
-            _query: this.rewrittenQuery(this.query),
-            _operation: this.operation,
-            _column: this.column
-        });
-
-        callback = callback || override;
-
-        debug(formulaSql);
-
-        return callback(null, formulaSql);
+    if (!VALID_OPERATIONS[this.operation]) {
+        return this.defaultSql(psql, override, callback);
     }
 
+    if (this._isFloatColumn === null) {
+        this._isFloatColumn = false;
+        this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) {
+            if (!err && !!type) {
+                self._isFloatColumn = type.float;
+            }
+            self.sql(psql, override, callback);
+        });
+        return null;
+    }
 
-    // default behaviour
-    return this.defaultSql(psql, override, callback);
+    var formulaSql = formulaQueryTpl({
+        isFloatColumn: this._isFloatColumn,
+        query: this.rewrittenQuery(this.query),
+        operation: this.operation,
+        column: this.column
+    });
+
+    callback = callback || override;
+
+    debug(formulaSql);
+
+    return callback(null, formulaSql);
 };

lib/cartodb/models/dataview/overviews/index.js

@@ -1,6 +1,5 @@
 module.exports = {
   Aggregation: require('./aggregation'),
   Formula: require('./formula'),
-  Histogram: require('./histogram'),
-  List: require('./list')
+  Histogram: require('./histogram')
 };

lib/cartodb/models/dataview/overviews/list.js

@@ -1,11 +0,0 @@
-var BaseOverviewsDataview = require('./base');
-var BaseDataview = require('../list');
-
-function List(query, options, queryRewriter, queryRewriteData, params, queries) {
-    BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries);
-}
-
-List.prototype = Object.create(BaseOverviewsDataview.prototype);
-List.prototype.constructor = List;
-
-module.exports = List;

lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js

@@ -0,0 +1,172 @@
+const AggregationMapConfig = require('../../aggregation/aggregation-mapconfig');
+const queryUtils = require('../../../utils/query-utils');
+
+const unsupportedGeometryTypeErrorMessage = ctx =>
+`Unsupported geometry type: ${ctx.geometryType}. ` +
+`Aggregation is available only for geometry type: ${AggregationMapConfig.SUPPORTED_GEOMETRY_TYPES}`;
+
+const invalidAggregationParamValueErrorMessage = ctx =>
+`Invalid value for 'aggregation' query param: ${ctx.value}. Valid ones are 'true' or 'false'`;
+
+module.exports = class AggregationMapConfigAdapter {
+    constructor (pgConnection) {
+        this.pgConnection = pgConnection;
+    }
+
+    getMapConfig (user, requestMapConfig, params, context, callback) {
+        if (!this._isValidAggregationQueryParam(params)) {
+            return callback(new Error(invalidAggregationParamValueErrorMessage({ value: params.aggregation })));
+        }
+
+        let mapConfig;
+        try {
+            mapConfig = new AggregationMapConfig(user, requestMapConfig, this.pgConnection);
+        } catch (err) {
+            return callback(err);
+        }
+
+
+        if (!this._shouldAdapt(mapConfig, params)) {
+            return callback(null, requestMapConfig);
+        }
+
+        this.pgConnection.getConnection(user, (err, connection) => {
+            if (err) {
+                return callback(err);
+            }
+
+            this._adaptLayers(connection, mapConfig, requestMapConfig, context, callback);
+        });
+    }
+
+    _isValidAggregationQueryParam (params) {
+        const { aggregation } = params;
+        return aggregation === undefined || aggregation === 'true' || aggregation === 'false';
+    }
+
+    _shouldAdapt (mapConfig, params) {
+        const { aggregation } = params;
+
+        if (aggregation === 'false') {
+            return false;
+        }
+
+        if (aggregation === 'true' || mapConfig.isAggregationMapConfig()) {
+            return true;
+        }
+
+        return false;
+    }
+
+    _adaptLayers (connection, mapConfig, requestMapConfig, context, callback) {
+        const adaptLayerPromises = requestMapConfig.layers.map((layer, index) => {
+            return this._adaptLayer(connection, mapConfig, layer, index);
+        });
+
+        Promise.all(adaptLayerPromises)
+            .then(results => {
+                context.aggregation = {
+                    layers: []
+                };
+
+                results.forEach(({ layer, index, adapted }) => {
+                    if (adapted) {
+                        requestMapConfig.layers[index] = layer;
+                    }
+                    const aggregatedFormats = this._getAggregationMetadata(mapConfig, layer, adapted);
+                    context.aggregation.layers.push(aggregatedFormats);
+                });
+
+                callback(null, requestMapConfig);
+            })
+            .catch(err => callback(err));
+    }
+
+    _adaptLayer (connection, mapConfig, layer, index) {
+        return new Promise((resolve, reject) => {
+            this._shouldAdaptLayer(connection, mapConfig, layer, index, (err, shouldAdapt) => {
+                if (err) {
+                    return reject(err);
+                }
+
+                if (!shouldAdapt) {
+                    return resolve({ layer, index, adapted: shouldAdapt });
+                }
+
+                const sqlQueryWrap = layer.options.sql_wrap;
+
+                let aggregationSql = mapConfig.getAggregatedQuery(index);
+
+                if (sqlQueryWrap) {
+                    aggregationSql = sqlQueryWrap.replace(/<%=\s*sql\s*%>/g, aggregationSql);
+                }
+
+                layer.options.sql = aggregationSql;
+
+                mapConfig.getLayerAggregationColumns(index, (err, columns) => {
+                    if (err) {
+                        return reject(err);
+                    }
+
+                    layer.options.columns = columns;
+
+                    return resolve({ layer, index, adapted: shouldAdapt });
+                });
+            });
+        });
+    }
+
+    _shouldAdaptLayer (connection, mapConfig, layer, index, callback) {
+        if (!mapConfig.isAggregationLayer(index)) {
+            return callback(null, false);
+        }
+
+        const aggregationMetadata = queryUtils.getAggregationMetadata({
+            query: layer.options.sql_raw ? layer.options.sql_raw : layer.options.sql,
+            geometryColumn: AggregationMapConfig.getAggregationGeometryColumn()
+        });
+
+        connection.query(aggregationMetadata, (err, res) => {
+            if (err) {
+                return callback(null, false);
+            }
+
+            const result = res.rows[0] || {};
+
+            if (!mapConfig.isVectorOnlyMapConfig() && !AggregationMapConfig.supportsGeometryType(result.type)) {
+                const message = unsupportedGeometryTypeErrorMessage({ geometryType: result.type });
+                const error = new Error(message);
+                error.type = 'layer';
+                error.layer = {
+                    id: mapConfig.getLayerId(index),
+                    index: index,
+                    type: mapConfig.layerType(index)
+                };
+
+                return callback(error);
+            }
+
+            if (mapConfig.isVectorOnlyMapConfig() && !AggregationMapConfig.supportsGeometryType(result.type)) {
+                return callback(null, false);
+            }
+
+            if (!mapConfig.doesLayerReachThreshold(index, result.count)) {
+                return callback(null, false);
+            }
+
+            callback(null, true);
+        });
+    }
+
+    _getAggregationMetadata (mapConfig, layer, adapted) {
+        if (!adapted) {
+            return { png: false, mvt: false };
+        }
+
+        if (mapConfig.isVectorOnlyMapConfig()) {
+            return { png: false, mvt: true };
+        }
+
+        return { png: true, mvt: true };
+    }
+};

lib/cartodb/models/mapconfig/adapter/analysis-mapconfig-adapter.js

@@ -58,6 +58,13 @@ AnalysisMapConfigAdapter.prototype.getMapConfig = function(user, requestMapConfi
 
     requestMapConfig = appendFiltersToNodes(requestMapConfig, dataviewsFiltersBySourceId);
 
+    // Expected format for analyses filters
+    // filters = {analyses: {
+    //     a1: [{min, max}, {accept, reject}],
+    //     b1: [{range, column, min, max}, {category, column, accept, reject}]
+    // }}
+    requestMapConfig = appendFiltersToNodes(requestMapConfig, filters.analyses);
+
     function createAnalysis(analysisDefinition, done) {
         self.analysisBackend.create(analysisConfiguration, analysisDefinition, function (err, analysis) {
             if (err) {
@@ -200,6 +207,7 @@ function dataviewQuery(node, dataviewName, ownFilter) {
 
 function appendFiltersToNodes(requestMapConfig, dataviewsFiltersBySourceId) {
     var analyses = requestMapConfig.analyses || [];
+    dataviewsFiltersBySourceId = dataviewsFiltersBySourceId || {};
 
     requestMapConfig.analyses = analyses.map(function(analysisDefinition) {
         var analysisGraph = new camshaft.reference.AnalysisGraph(analysisDefinition);

lib/cartodb/models/resource-locator.js

@@ -21,69 +21,208 @@ function ResourceLocator(environment) {
 
 module.exports = ResourceLocator;
 
-ResourceLocator.prototype.getUrls = function(username, resource) {
+ResourceLocator.prototype.getTileUrls = function(username, resourcePath) {
     if (this.resourcesUrlTemplates) {
-        return this.getUrlsFromTemplate(username, resource);
-    }
-    var cdnDomain = getCdnDomain(this.environment.serverMetadata, resource);
-    if (cdnDomain) {
+        const urls = this.getUrlsFromTemplate(username, new TileResource(resourcePath));
         return {
-            http: 'http://' + cdnDomain.http + '/' + username + '/api/v1/map/' + resource,
-            https: 'https://' + cdnDomain.https + '/' + username + '/api/v1/map/' + resource
+            http: Array.isArray(urls.http) ? urls.http : [urls.http],
+            https: Array.isArray(urls.https) ? urls.https : [urls.https]
         };
+    }
+    var cdnUrls = getCdnUrls(this.environment.serverMetadata, username, new TileResource(resourcePath));
+    if (cdnUrls) {
+        return cdnUrls;
     } else {
         var port = this.environment.port;
         return {
-            http: 'http://' + username + '.' + 'localhost.lan:' + port +  '/api/v1/map/' + resource
+            http: [`http://${username}.localhost.lan:${port}/api/v1/map/${resourcePath}`]
         };
     }
 };
 
+ResourceLocator.prototype.getTemplateUrls = function(username, resourcePath) {
+    if (this.resourcesUrlTemplates) {
+        return this.getUrlsFromTemplate(username, new TemplateResource(resourcePath), true);
+    }
+    var cdnUrls = getCdnUrls(this.environment.serverMetadata, username, new TemplateResource(resourcePath));
+    if (cdnUrls) {
+        return cdnUrls;
+    } else {
+        var port = this.environment.port;
+        return {
+            http: {
+                urlTemplate: `http://${username}.localhost.lan:${port}/api/v1/map/${resourcePath}`,
+                subdomains: []
+            }
+        };
+    }
+};
 
-ResourceLocator.prototype.getUrlsFromTemplate = function(username, resource) {
-    var urls = {};
-    var cdnDomain = getCdnDomain(this.environment.serverMetadata, resource) || {};
+ResourceLocator.prototype.getUrls = function(username, resourcePath) {
+    if (this.resourcesUrlTemplates) {
+        return this.getUrlsFromTemplate(username, new Resource(resourcePath));
+    }
+    var cdnUrls = getCdnUrls(this.environment.serverMetadata, username, new Resource(resourcePath));
+    if (cdnUrls) {
+        return cdnUrls;
+    } else {
+        var port = this.environment.port;
+        return {
+            http: `http://${username}.localhost.lan:${port}/api/v1/map/${resourcePath}`
+        };
+    }
+};
 
-    if (this.resourcesUrlTemplates.http) {
-        urls.http = this.resourcesUrlTemplates.http({
-            cdn_url: cdnDomain.http,
+function urlForTemplate(tpl, username, cdnDomain, resource, templated) {
+    cdnDomain = cdnDomain || {};
+    if (templated) {
+        return {
+            urlTemplate: tpl({
+                cdn_url: (cdnDomain.hasOwnProperty('urlTemplate') ? cdnDomain.urlTemplate : cdnDomain),
+                user: username,
+                port: this.environment.port,
+                resource: resource.getPath()
+            }),
+            subdomains: cdnDomain.subdomains || []
+        };
+    }
+    if (Array.isArray(cdnDomain)) {
+        return cdnDomain.map(d => tpl({
+            cdn_url: d,
             user: username,
             port: this.environment.port,
-            resource: resource
+            resource: resource.getPath()
+        }));
+    } else {
+        return tpl({
+            cdn_url: cdnDomain,
+            user: username,
+            port: this.environment.port,
+            resource: resource.getPath()
         });
     }
+}
 
+ResourceLocator.prototype.getUrlsFromTemplate = function(username, resource, templated) {
+    var urls = {};
+    var cdnDomain = getCdnDomain(this.environment.serverMetadata, resource) || {};
+    if (this.resourcesUrlTemplates.http) {
+        urls.http = urlForTemplate(this.resourcesUrlTemplates.http, username, cdnDomain.http, resource, templated);
+    }
     if (this.resourcesUrlTemplates.https) {
-        urls.https = this.resourcesUrlTemplates.https({
-            cdn_url: cdnDomain.https,
-            user: username,
-            port: this.environment.port,
-            resource: resource
-        });
+        urls.https = urlForTemplate(this.resourcesUrlTemplates.https, username, cdnDomain.https, resource, templated);
     }
 
     return urls;
 };
 
+class Resource {
+    constructor (resourcePath) {
+        this.resourcePath = resourcePath;
+    }
+
+    getPath () {
+        return this.resourcePath;
+    }
+
+    getDomain (domain, subdomains) {
+        if (!subdomains) {
+            return domain;
+        }
+        return domain.replace('{s}', subdomain(subdomains, this.resourcePath));
+    }
+
+    getUrl (baseUrl, username, subdomains) {
+        let urls = getUrl(baseUrl, username, this.resourcePath);
+        if (subdomains) {
+            urls = urls.replace('{s}', subdomain(subdomains, this.resourcePath));
+        }
+        return urls;
+    }
+}
+
+class TileResource extends Resource {
+    constructor (resourcePath) {
+        super(resourcePath);
+    }
+
+    getDomain (domain, subdomains) {
+        if (!subdomains) {
+            return domain;
+        }
+        return subdomains.map(s => domain.replace('{s}', s));
+    }
+
+    getUrl (baseUrl, username, subdomains) {
+        if (!subdomains) {
+            return [super.getUrl(baseUrl, username)];
+        }
+        return subdomains.map(subdomain => {
+            return getUrl(baseUrl, username, this.resourcePath)
+                .replace('{s}', subdomain);
+        });
+    }
+}
+
+class TemplateResource extends Resource {
+    constructor (resourcePath) {
+        super(resourcePath);
+    }
+
+    getDomain (domain, subdomains) {
+        return {
+            urlTemplate: domain,
+            subdomains: subdomains || []
+        };
+    }
+
+    getUrl (baseUrl, username, subdomains) {
+        return {
+            urlTemplate: getUrl(baseUrl, username, this.resourcePath),
+            subdomains: subdomains || []
+        };
+    }
+}
+
+function getUrl(baseUrl, username, path) {
+    return `${baseUrl}/${username}/api/v1/map/${path}`;
+}
+
+function getCdnUrls(serverMetadata, username, resource) {
+    if (serverMetadata && serverMetadata.cdn_url) {
+        var cdnUrl = serverMetadata.cdn_url;
+        var httpUrls = resource.getUrl(`http://${cdnUrl.http}`, username);
+        var httpsUrls = resource.getUrl(`https://${cdnUrl.https}`, username);
+        if (cdnUrl.templates) {
+            var templates = cdnUrl.templates;
+            httpUrls = resource.getUrl(templates.http.url, username, templates.http.subdomains);
+            httpsUrls = resource.getUrl(templates.https.url, username, templates.https.subdomains);
+        }
+        return {
+            http: httpUrls,
+            https: httpsUrls,
+        };
+    }
+    return null;
+}
+
 function getCdnDomain(serverMetadata, resource) {
     if (serverMetadata && serverMetadata.cdn_url) {
         var cdnUrl = serverMetadata.cdn_url;
-        var http = cdnUrl.http;
-        var https = cdnUrl.https;
+        var httpDomain = resource.getDomain(cdnUrl.http);
+        var httpsDomain = resource.getDomain(cdnUrl.https);
         if (cdnUrl.templates) {
             var templates = cdnUrl.templates;
             var httpUrlTemplate = templates.http.url;
             var httpsUrlTemplate = templates.https.url;
-            http = httpUrlTemplate
-                .replace(/^(http[s]*:\/\/)/, '')
-                .replace('{s}', subdomain(templates.http.subdomains, resource));
-            https = httpsUrlTemplate
-                .replace(/^(http[s]*:\/\/)/, '')
-                .replace('{s}', subdomain(templates.https.subdomains, resource));
+            httpDomain = httpUrlTemplate.replace(/^(http[s]*:\/\/)/, '');
+            httpDomain = resource.getDomain(httpDomain, templates.http.subdomains);
+            httpsDomain = httpsUrlTemplate.replace(/^(http[s]*:\/\/)/, '');
+            httpsDomain = resource.getDomain(httpsDomain, templates.https.subdomains);
         }
         return {
-            http: http,
-            https: https,
+            http: httpDomain,
+            https: httpsDomain,
         };
     }
     return null;

lib/cartodb/server.js

@@ -12,7 +12,8 @@ var VarnishHttpCacheBackend = require('./cache/backend/varnish_http');
 var FastlyCacheBackend = require('./cache/backend/fastly');
 
 var StatsClient = require('./stats/client');
-var Profiler = require('./stats/profiler_proxy');
+const stats = require('./middleware/stats');
+
 var RendererStatsReporter = require('./stats/reporter/renderer');
 
 var windshaft = require('windshaft');
@@ -40,10 +41,16 @@ var AnalysisMapConfigAdapter = require('./models/mapconfig/adapter/analysis-mapc
 var MapConfigOverviewsAdapter = require('./models/mapconfig/adapter/mapconfig-overviews-adapter');
 var TurboCartoAdapter = require('./models/mapconfig/adapter/turbo-carto-adapter');
 var DataviewsWidgetsAdapter = require('./models/mapconfig/adapter/dataviews-widgets-adapter');
+var AggregationMapConfigAdapter = require('./models/mapconfig/adapter/aggregation-mapconfig-adapter');
 var MapConfigAdapter = require('./models/mapconfig/adapter');
 
 var StatsBackend = require('./backends/stats');
 
+const lzmaMiddleware = require('./middleware/lzma');
+const errorMiddleware = require('./middleware/error-middleware');
+
+const prepareContextMiddleware = require('./middleware/context');
+
 module.exports = function(serverOptions) {
     // Make stats client globally accessible
     global.statsClient = StatsClient.getInstance(serverOptions.statsd);
@@ -152,7 +159,8 @@ module.exports = function(serverOptions) {
             grainstore: serverOptions.grainstore,
             mapnik: serverOptions.renderer.mapnik
         },
-        http: serverOptions.renderer.http
+        http: serverOptions.renderer.http,
+        mvt: serverOptions.renderer.mvt
     });
 
     // initialize render cache
@@ -183,6 +191,7 @@ module.exports = function(serverOptions) {
         new SqlWrapMapConfigAdapter(),
         new DataviewsWidgetsAdapter(),
         new AnalysisMapConfigAdapter(analysisBackend),
+        new AggregationMapConfigAdapter(pgConnection),
         new MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi),
         new TurboCartoAdapter()
     );
@@ -206,12 +215,16 @@ module.exports = function(serverOptions) {
 
     var versions = getAndValidateVersions(serverOptions);
 
+    const prepareContext = typeof serverOptions.req2params === 'function' ?
+        serverOptions.req2params :
+        prepareContextMiddleware(authApi, pgConnection);
+
     /*******************************************************************************************************************
      * Routing
      ******************************************************************************************************************/
 
     new controller.Layergroup(
-        authApi,
+        prepareContext,
         pgConnection,
         mapStore,
         tileBackend,
@@ -224,7 +237,7 @@ module.exports = function(serverOptions) {
     ).register(app);
 
     new controller.Map(
-        authApi,
+        prepareContext,
         pgConnection,
         templateMaps,
         mapBackend,
@@ -237,8 +250,7 @@ module.exports = function(serverOptions) {
     ).register(app);
 
     new controller.NamedMaps(
-        authApi,
-        pgConnection,
+        prepareContext,
         namedMapProviderCache,
         tileBackend,
         previewBackend,
@@ -247,9 +259,9 @@ module.exports = function(serverOptions) {
         metadataBackend
     ).register(app);
 
-    new controller.NamedMapsAdmin(authApi, pgConnection, templateMaps).register(app);
+    new controller.NamedMapsAdmin(authApi, templateMaps).register(app);
 
-    new controller.Analyses(authApi, pgConnection).register(app);
+    new controller.Analyses(prepareContext).register(app);
 
     new controller.ServerInfo(versions).register(app);
 
@@ -257,6 +269,8 @@ module.exports = function(serverOptions) {
      * END Routing
      ******************************************************************************************************************/
 
+    app.use(errorMiddleware());
+
     return app;
 };
 
@@ -351,12 +365,6 @@ function bootstrap(opts) {
     app.use(bodyParser.json());
 
     app.use(function bootstrap$prepareRequestResponse(req, res, next) {
-        req.context = req.context || {};
-        req.profiler = new Profiler({
-            statsd_client: global.statsClient,
-            profile: opts.useProfiler
-        });
-
         if (global.environment && global.environment.api_hostname) {
             res.set('X-Served-By-Host', global.environment.api_hostname);
         }
@@ -364,6 +372,13 @@ function bootstrap(opts) {
         next();
     });
 
+    app.use(stats({
+        enabled: opts.useProfiler,
+        statsClient: global.statsClient
+    }));
+
+    app.use(lzmaMiddleware);
+
     // temporary measure until we upgrade to newer version expressjs so we can check err.status
     app.use(function(err, req, res, next) {
         if (err) {

lib/cartodb/server_options.js

@@ -81,6 +81,7 @@ module.exports = {
         statsInterval: rendererConfig.statsInterval
     },
     renderer: {
+        mvt: rendererConfig.mvt,
         mapnik: _.defaults(rendererConfig.mapnik, {
             geojson: {
                 dbPoolParams: {

lib/cartodb/utils/icu_data_env_setter.js

@@ -0,0 +1,19 @@
+'use strict';
+
+const glob = require('glob');
+const path = require('path');
+
+// See https://github.com/CartoDB/support/issues/984
+// CartoCSS properties text-wrap-width/text-wrap-character not working
+function setICUEnvVariable() {
+    if (process.env.ICU_DATA === undefined) {
+        const regexedPath = '/node_modules/mapnik/lib/binding/*/share/mapnik/icu/';
+        const directory = glob.sync(path.join(__dirname, '../../..', regexedPath));
+
+        if (directory && directory.length > 0) {
+            process.env.ICU_DATA = directory[0];
+        }
+    }
+}
+
+module.exports = setICUEnvVariable;

lib/cartodb/utils/query-utils.js

@@ -21,6 +21,53 @@ module.exports.extractTableNames = function extractTableNames(query) {
     ].join('');
 };
 
-module.exports.getQueryRowCount = function getQueryRowEstimation(query) {
-    return 'select CDB_EstimateRowCount(\'' + query + '\') as rows';
+function getQueryRowEstimation(query) {
+    return 'select CDB_EstimateRowCount($windshaft$' + query + '$windshaft$) as rows';
+}
+module.exports.getQueryRowCount = getQueryRowEstimation;
+
+module.exports.getAggregationMetadata = ctx => `
+    WITH
+    rowEstimation AS (
+        ${getQueryRowEstimation(ctx.query)}
+    ),
+    geometryType AS (
+        SELECT ST_GeometryType(${ctx.geometryColumn}) as geom_type
+        FROM (${ctx.query}) AS __cdb_query WHERE ${ctx.geometryColumn} IS NOT NULL LIMIT 1
+    )
+    SELECT
+        rows AS count,
+        geom_type AS type
+    FROM rowEstimation, geometryType;
+`;
+
+/** Cast the column to epoch */
+module.exports.columnCastTpl = function columnCastTpl(ctx) {
+    return `date_part('epoch', ${ctx.column})`;
+};
+
+/** If the column type is float, ignore any non numeric result (infinity / NaN) */
+module.exports.handleFloatColumn = function handleFloatColumn(ctx) {
+    return `${!ctx.isFloatColumn ? `${ctx.column}` :
+        `nullif(nullif(nullif(${ctx.column}, 'infinity'::float), '-infinity'::float), 'NaN'::float)`
+    }`;
+};
+
+/** Count NULL appearances */
+module.exports.countNULLs= function countNULLs(ctx) {
+    return `sum(CASE WHEN (${ctx.column} IS NULL) THEN 1 ELSE 0 END)`;
+};
+
+/** Count only infinity (positive and negative) appearances */
+module.exports.countInfinites = function countInfinites(ctx) {
+    return `${!ctx.isFloatColumn ? `0` :
+        `sum(CASE WHEN (${ctx.column} = 'infinity'::float OR ${ctx.column} = '-infinity'::float) THEN 1 ELSE 0 END)`
+    }`;
+};
+
+/** Count only NaNs appearances*/
+module.exports.countNaNs = function countNaNs(ctx) {
+    return `${!ctx.isFloatColumn ? `0` :
+        `sum(CASE WHEN (${ctx.column} = 'NaN'::float) THEN 1 ELSE 0 END)`
+    }`;
 };

package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "name": "windshaft-cartodb",
-  "version": "3.12.2",
+  "version": "5.2.0",
   "description": "A map tile server for CartoDB",
   "keywords": [
     "cartodb"
@@ -17,30 +17,37 @@
     "Simon Tokumine <simon@vizzuality.com>",
     "Javi Santana <jsantana@vizzuality.com>",
     "Sandro Santilli <strk@vizzuality.com>",
-    "Carlos Matallín <matallo@carto.com>"
+    "Carlos Matallín <matallo@carto.com>",
+    "Daniel Garcia Aubert <dgaubert@carto.com>",
+    "Mario de Frutos <mario.defrutos@carto.com>",
+    "Ivan Malagon <ivan@carto.com>",
+    "Simon Martin <simon@carto.com>"
   ],
   "dependencies": {
-    "body-parser": "~1.14.0",
-    "camshaft": "0.55.7",
-    "cartodb-psql": "0.10.1",
-    "cartodb-query-tables": "0.2.0",
+    "body-parser": "^1.18.2",
+    "camshaft": "0.60.0",
+    "cartodb-psql": "0.10.2",
+    "cartodb-query-tables": "0.3.0",
     "cartodb-redis": "0.14.0",
-    "debug": "~2.2.0",
+    "debug": "^3.1.0",
     "dot": "~1.0.2",
-    "express": "~4.13.3",
+    "express": "~4.16.0",
     "fastly-purge": "~1.0.1",
+    "glob": "^7.1.2",
     "log4js": "cartodb/log4js-node#cdb",
     "lru-cache": "2.6.5",
     "lzma": "~2.3.2",
     "node-statsd": "~0.0.7",
+    "on-headers": "^1.0.1",
     "queue-async": "~1.0.7",
     "redis-mpool": "0.4.1",
-    "request": "~2.79.0",
+    "request": "^2.83.0",
+    "semver": "~5.3.0",
     "step": "~0.0.6",
     "step-profiler": "~0.3.0",
-    "turbo-carto": "0.19.2",
+    "turbo-carto": "0.20.2",
     "underscore": "~1.6.0",
-    "windshaft": "3.3.1",
+    "windshaft": "4.3.3",
     "yargs": "~5.0.0"
   },
   "devDependencies": {
@@ -50,16 +57,20 @@
     "moment": "~2.18.1",
     "nock": "~2.11.0",
     "redis": "~0.12.1",
-    "semver": "~1.1.4",
     "strftime": "~0.8.2"
   },
   "scripts": {
     "lint": "jshint lib test",
     "preinstall": "make pre-install",
-    "test": "make test-all"
+    "test": "make test-all",
+    "update-internal-deps": "rm -rf node_modules && rm -f yarn.lock && yarn",
+    "docker-install": "sudo apt install docker.io && sudo usermod -aG docker $(whoami)",
+    "docker-pull": "docker pull cartoimages/windshaft-testing",
+    "docker-test": "docker run -v `pwd`:/srv cartoimages/windshaft-testing bash docker-test.sh && docker ps --filter status=dead --filter status=exited -aq | xargs -r docker rm -v",
+    "docker-bash": "docker run  -it -v `pwd`:/srv cartoimages/windshaft-testing bash"
   },
   "engines": {
     "node": ">=6.9",
-    "yarn": "^0.21.3"
+    "yarn": ">=0.27.5 <1.0.0"
   }
 }

run_tests.sh

@@ -137,7 +137,7 @@ if test x"$OPT_COVERAGE" = xyes; then
   ./node_modules/.bin/istanbul cover node_modules/.bin/_mocha -- -u tdd -t 5000 ${TESTS}
 else
   echo "Running tests"
-  mocha -u tdd -t 5000 ${TESTS}
+  ./node_modules/.bin/_mocha -c -u tdd -t 5000 ${TESTS}
 fi
 ret=$?
 

scripts/mvt-timeout-error.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+
+import mapbox_vector_tile
+
+lines_list = []
+
+# main diagonal line
+lines_list.append({ "geometry":"LINESTRING (0 0, 4096 4096)"})
+
+# diagonal lines 
+for i in range(4096/32, 4096, 4096/32):
+    start = i
+    end = 4096 - i
+    
+    lines_list.append({ "geometry":"LINESTRING (0 " + str(start) + ", " + str(end) + " 4096)" })
+    lines_list.append({ "geometry":"LINESTRING (" + str(start) + " 0, 4096 " + str(end) + ")" })
+
+# box lines
+lines_list.append({ "geometry":"LINESTRING (0 0, 0 4096)"})
+lines_list.append({ "geometry":"LINESTRING (0 4096, 4096 4096)"})
+lines_list.append({ "geometry":"LINESTRING (4096 4096, 4096 0)"})
+lines_list.append({ "geometry":"LINESTRING (4096 0, 0 0)"})
+
+
+tile = mapbox_vector_tile.encode([
+    {
+        "name": "errorTileSquareLayer",
+        "features": [{ "geometry":"POLYGON ((0 0, 0 4096, 4096 4096, 4096 0, 0 0))" }]
+    },
+    {
+        "name": "errorTileStripesLayer",
+        "features": lines_list
+    }
+])
+
+with open('./assets/render-timeout-fallback.mvt', 'w+') as f:
+    f.write(tile)

test/acceptance/analysis/analyses-controller.js

@@ -0,0 +1,114 @@
+require('../../support/test_helper');
+
+var assert = require('../../support/assert');
+var TestClient = require('../../support/test-client');
+
+
+describe('analyses controller', function () {
+    const mapConfig = {
+        version: '1.5.0',
+        layers:
+            [{
+                type: 'cartodb',
+                options:
+                    {
+                        source: { id: 'a1' },
+                        cartocss: TestClient.CARTOCSS.POLYGONS,
+                        cartocss_version: '2.3.0'
+                    }
+            }],
+        dataviews: {},
+        analyses:
+            [{
+                id: 'a1',
+                type: 'buffer',
+                params: {
+                    source: {
+                        type: 'source',
+                        params: {
+                            query: 'select * from analysis_banks limit 1'
+                        }
+                    },
+                    radius: 250
+                }
+            }]
+    };
+
+    beforeEach(function () {
+        this.testClient = new TestClient(mapConfig, 1234);
+    });
+
+    it('should get an array of analyses from catalog', function (done) {
+        this.testClient.getAnalysesCatalog({}, (err, result) => {
+            if (err) {
+                return done(err);
+            }
+
+            assert.ok(Array.isArray(result.catalog));
+            done();
+        });
+    });
+
+    it('should support jsonp responses', function (done) {
+        this.testClient.getAnalysesCatalog({ jsonp: 'jsonp_test' }, (err, result) => {
+            if (err) {
+                return done(err);
+            }
+
+            assert.ok(result);
+
+            let didRunJsonCallback = false;
+            // jshint ignore:start
+            function jsonp_test(body) {
+                assert.ok(Array.isArray(body.catalog));
+                didRunJsonCallback = true;
+            }
+
+            eval(result);
+            // jshint ignore:end
+
+            assert.ok(didRunJsonCallback);
+
+            done();
+        });
+    });
+
+    it('should respond "unauthorized" when missing api_key', function (done) {
+        const apiKey = this.testClient.apiKey;
+        this.testClient.apiKey = null;
+
+        this.testClient.getAnalysesCatalog({ status: 401 }, (err, result) => {
+            if (err) {
+                return done(err);
+            }
+
+            assert.deepEqual(result.errors[0], 'Unauthorized');
+            this.testClient.apiKey = apiKey;
+            done();
+        });
+    });
+
+    it('should get an array of analyses from catalog', function (done) {
+        this.testClient.getTile(0, 0, 0, (err) => {
+            if (err) {
+                return done(err);
+            }
+
+            this.testClient.getAnalysesCatalog({}, (err, result) => {
+                if (err) {
+                    return done(err);
+                }
+
+                assert.ok(Array.isArray(result.catalog));
+                assert.ok(result.catalog.length >=  2); // buffer & source at least
+
+                result.catalog
+                    .filter(analysis => analysis.node_id === '0a215e1f3405381cf0ea6b3b0deb6fdcfdc2fcaa')
+                    .forEach(analysis => assert.equal(analysis.type, 'buffer'));
+
+                this.testClient.drain(done);
+            });
+
+        });
+    });
+});

test/acceptance/analysis/analyses-filters-params.js

@@ -0,0 +1,150 @@
+require('../../support/test_helper');
+
+const assert = require('../../support/assert');
+const TestClient = require('../../support/test-client');
+
+describe('analysis-filters-params', () => {
+
+    const CARTOCSS = `#layer {
+        marker-fill-opacity: 1;
+        marker-line-color: white;
+        marker-line-width: 0.5;
+        marker-line-opacity: 1;
+        marker-placement: point;
+        marker-type: ellipse;
+        marker-width: 8;
+        marker-fill: red;
+        marker-allow-overlap: true;
+      }`;
+
+    const mapConfig = {
+        version: '1.6.0',
+        layers: [
+            {
+                "type": "cartodb",
+                "options": {
+                    "source": {
+                        "id": "a1"
+                    },
+                    "cartocss": CARTOCSS,
+                    "cartocss_version": "2.3.0"
+                }
+            }
+        ],
+        dataviews: {
+            pop_max_histogram: {
+                source: {
+                    id: 'a1'
+                },
+                type: 'histogram',
+                options: {
+                    column: 'pop_max'
+                }
+            },
+            pop_min_histogram: {
+                source: {
+                    id: 'a1'
+                },
+                type: 'histogram',
+                options: {
+                    column: 'pop_min'
+                }
+            }
+        },
+        analyses: [
+            {
+                "id": "a1",
+                "type": "source",
+                "params": {
+                    "query": "select * from populated_places_simple_reduced"
+                }
+            }
+        ]
+    };
+
+    var params = {
+        filters: {
+            dataviews: {
+                pop_max_histogram: {
+                    min: 2e6
+                },
+                pop_min_histogram: {
+                    max: 2e6
+                }
+            }
+        }
+    };
+
+
+    it('should get a filtered histogram dataview with all filters', function(done) {
+        const testClient = new TestClient(mapConfig, 1234);
+        const testParams = Object.assign({}, params, {
+            own_filter: 1
+        });
+
+        testClient.getDataview('pop_max_histogram', testParams, (err, dataview) => {
+            assert.ok(!err, err);
+
+            assert.equal(dataview.type, 'histogram');
+            assert.equal(dataview.bins_count, 6);
+
+            testClient.drain(done);
+        });
+    });
+
+    it('should get a filtered histogram dataview with all filters except my own filter', function(done) {
+        const testClient = new TestClient(mapConfig, 1234);
+        const testParams = Object.assign({}, params, {
+            own_filter: 0
+        });
+
+        testClient.getDataview('pop_max_histogram', testParams, (err, dataview) => {
+            assert.ok(!err, err);
+
+            assert.equal(dataview.type, 'histogram');
+            assert.equal(dataview.bins_count, 24);
+
+            testClient.drain(done);
+        });
+    });
+
+    it('should get a filtered histogram dataview without filters', function(done) {
+        const testClient = new TestClient(mapConfig, 1234);
+        const testParams = Object.assign({}, params, {
+            no_filters: 1
+        });
+
+        testClient.getDataview('pop_max_histogram', testParams, (err, dataview) => {
+            assert.ok(!err, err);
+
+            assert.equal(dataview.type, 'histogram');
+            assert.equal(dataview.bins_count, 48);
+
+            testClient.drain(done);
+        });
+    });
+
+    it('should return an error if both no_filters and own_filter params are present', function (done) {
+        const testClient = new TestClient(mapConfig, 1234);
+        const expectedError = {
+            errors: ['Both own_filter and no_filters cannot be sent in the same request'],
+            errors_with_context: [{
+                type: 'dataview',
+                message: 'Both own_filter and no_filters cannot be sent in the same request'
+            }]
+        };
+        const testParams = Object.assign({}, params, {
+            no_filters: 1,
+            own_filter: 0,
+            response: {
+                status: 400
+            }
+        });
+
+        testClient.getDataview('pop_max_histogram', testParams, (err, dataview) => {
+            assert.deepEqual(dataview, expectedError);
+
+            testClient.drain(done);
+        });
+    });
+});

test/acceptance/analysis/analyses-filters.js

@@ -0,0 +1,84 @@
+require('../../support/test_helper');
+
+const assert = require('../../support/assert');
+const TestClient = require('../../support/test-client');
+
+describe('analysis-layers-dataviews', () => {
+
+    const CARTOCSS = `#layer {
+        marker-fill-opacity: 1;
+        marker-line-color: white;
+        marker-line-width: 0.5;
+        marker-line-opacity: 1;
+        marker-placement: point;
+        marker-type: ellipse;
+        marker-width: 8;
+        marker-fill: red;
+        marker-allow-overlap: true;
+      }`;
+
+    const mapConfig = {
+        version: '1.6.0',
+        layers: [
+            {
+                "type": "cartodb",
+                "options": {
+                    "source": {
+                        "id": "a1"
+                    },
+                    "cartocss": CARTOCSS,
+                    "cartocss_version": "2.3.0"
+                }
+            }
+        ],
+        dataviews: {
+            pop_max_histogram: {
+                source: {
+                    id: 'a1'
+                },
+                type: 'histogram',
+                options: {
+                    column: 'pop_max'
+                }
+            }
+        },
+        analyses: [
+            {
+                "id": "a1",
+                "type": "source",
+                "params": {
+                    "query": "select * from populated_places_simple_reduced"
+                }
+            }
+        ]
+    };
+
+    it('should get a filtered histogram dataview', function(done) {
+        const testClient = new TestClient(mapConfig, 1234);
+
+        const params = {
+            filters: {
+                analyses: {
+                    'a1': [
+                        {
+                            type: 'range',
+                            column: 'pop_max',
+                            params: {
+                                min: 2e6
+                            }
+                        }
+                    ]
+                }
+            }
+        };
+
+        testClient.getDataview('pop_max_histogram', params, (err, dataview) => {
+            assert.ok(!err, err);
+
+            assert.equal(dataview.type, 'histogram');
+            assert.equal(dataview.bins_start, 2008000);
+
+            testClient.drain(done);
+        });
+    });
+});

test/acceptance/analysis/analysis-layers-dataviews.js

@@ -109,7 +109,8 @@ describe('analysis-layers-dataviews', function() {
                         min: 2e6
                     }
                 }
-            }
+            },
+            own_filter: 1
         };
 
         testClient.getDataview('pop_max_histogram', params, function(err, dataview) {

test/acceptance/analysis/analysis-layers-geojson.js

@@ -1,81 +0,0 @@
-require('../../support/test_helper');
-
-var assert = require('../../support/assert');
-var TestClient = require('../../support/test-client');
-
-describe('analysis-layers-dataviews-geojson', function() {
-
-    function createMapConfig(layers, dataviews, analysis) {
-        return {
-            version: '1.5.0',
-            layers: layers,
-            dataviews: dataviews || {},
-            analyses: analysis || []
-        };
-    }
-
-    var CARTOCSS = [
-        "#points {",
-        "  marker-fill-opacity: 1.0;",
-        "  marker-line-color: #FFF;",
-        "  marker-line-width: 0.5;",
-        "  marker-line-opacity: 1.0;",
-        "  marker-placement: point;",
-        "  marker-type: ellipse;",
-        "  marker-width: 8;",
-        "  marker-fill: red;",
-        "  marker-allow-overlap: true;",
-        "}"
-    ].join('\n');
-
-    var mapConfig = createMapConfig(
-        [
-            {
-                "type": "cartodb",
-                "options": {
-                    "source": {
-                        "id": "2570e105-7b37-40d2-bdf4-1af889598745"
-                    },
-                    "cartocss": CARTOCSS,
-                    "cartocss_version": "2.3.0"
-                }
-            }
-        ],
-        {
-            pop_max_histogram: {
-                source: {
-                    id: '2570e105-7b37-40d2-bdf4-1af889598745'
-                },
-                type: 'histogram',
-                options: {
-                    column: 'pop_max'
-                }
-            }
-        },
-        [
-            {
-                "id": "2570e105-7b37-40d2-bdf4-1af889598745",
-                "type": "source",
-                "params": {
-                    "query": "select * from populated_places_simple_reduced"
-                }
-            }
-        ]
-    );
-
-    it('should get pop_max column from dataview', function(done) {
-        var testClient = new TestClient(mapConfig, 1234);
-
-        testClient.getTile(0, 0, 0, {format: 'geojson', layers: 0}, function(err, res, geojson) {
-            assert.ok(!err, err);
-
-            assert.ok(Array.isArray(geojson.features));
-            assert.ok(geojson.features.length > 0);
-            var feature = geojson.features[0];
-            assert.ok(feature.properties.hasOwnProperty('pop_max'), 'Missing pop_max property');
-
-            testClient.drain(done);
-        });
-    });
-
-});

test/acceptance/analysis/analysis-layers-use-cases.js

@@ -5,34 +5,34 @@ var TestClient = require('../../support/test-client');
 var dot = require('dot');
 var debug = require('debug')('windshaft:cartodb:test');
 
-describe('analysis-layers use cases', function() {
+describe('analysis-layers use cases', function () {
 
 
-    var multitypeStyleTemplate = dot.template([
-        "#points['mapnik::geometry_type'=1] {",
-        "  marker-fill-opacity: {{=it._opacity}};",
-        "  marker-line-color: #FFF;",
-        "  marker-line-width: 0.5;",
-        "  marker-line-opacity: {{=it._opacity}};",
-        "  marker-placement: point;",
-        "  marker-type: ellipse;",
-        "  marker-width: 8;",
-        "  marker-fill: {{=it._color}};",
-        "  marker-allow-overlap: true;",
-        "}",
-        "#lines['mapnik::geometry_type'=2] {",
-        "  line-color: {{=it._color}};",
-        "  line-width: 2;",
-        "  line-opacity: {{=it._opacity}};",
-        "}",
-        "#polygons['mapnik::geometry_type'=3] {",
-        "  polygon-fill: {{=it._color}};",
-        "  polygon-opacity: {{=it._opacity}};",
-        "  line-color: #FFF;",
-        "  line-width: 0.5;",
-        "  line-opacity: {{=it._opacity}};",
-        "}"
-    ].join('\n'));
+    var multitypeStyleTemplate = dot.template(
+        `#points['mapnik::geometry_type'=1] {
+          marker-fill-opacity: {{=it._opacity}};
+          marker-line-color: #FFF;
+          marker-line-width: 0.5;
+          marker-line-opacity: {{=it._opacity}};
+          marker-placement: point;
+          marker-type: ellipse;
+          marker-width: 8;
+          marker-fill: {{=it._color}};
+          marker-allow-overlap: true;
+        }
+        #lines['mapnik::geometry_type'=2] {
+          line-color: {{=it._color}};
+          line-width: 2;
+          line-opacity: {{=it._opacity}};
+        }
+        #polygons['mapnik::geometry_type'=3] {
+          polygon-fill: {{=it._color}};
+          polygon-opacity: {{=it._opacity}};
+          line-color: #FFF;
+          line-width: 0.5;
+          line-opacity: {{=it._opacity}};
+        }`
+    );
 
 
     function cartocss(color, opacity) {
@@ -47,18 +47,53 @@ describe('analysis-layers use cases', function() {
             version: '1.5.0',
             layers: layers,
             dataviews: dataviews || {},
-            analysis: analysis || []
+            analyses: analysis || []
         };
     }
 
-    function analysisDef(analysis) {
-        return JSON.stringify(analysis);
-    }
-
     var DEFAULT_MULTITYPE_STYLE = cartocss();
 
     var TILE_ANALYSIS_TABLES = { z: 14, x: 8023, y: 6177 };
 
+    var pointInPolygonDef = {
+        id: 'a1',
+        type: 'point-in-polygon',
+        params: {
+            points_source: {
+                type: 'source',
+                params: {
+                    query: 'select * from analysis_rent_listings'
+                }
+            },
+            polygons_source: {
+                type: 'buffer',
+                params: {
+                    source: {
+                        type: 'source',
+                        params: {
+                            query: 'select * from analysis_banks'
+                        }
+                    },
+                    radius: 250
+                }
+            }
+        }
+    };
+
+    var bufferDef = {
+        id: 'b1',
+        type: 'buffer',
+        params: {
+            source: {
+                type: 'source',
+                params: {
+                    query: 'select * from analysis_banks'
+                }
+            },
+            radius: 250
+        }
+    };
+
     var useCases = [
         {
             desc: '1 mapnik layer',
@@ -68,7 +103,7 @@ describe('analysis-layers use cases', function() {
                     {
                         type: 'cartodb',
                         options: {
-                            sql: "select * from analysis_rent_listings",
+                            sql: 'select * from analysis_rent_listings',
                             cartocss: DEFAULT_MULTITYPE_STYLE,
                             cartocss_version: '2.3.0'
                         }
@@ -83,7 +118,7 @@ describe('analysis-layers use cases', function() {
                 {
                     type: 'cartodb',
                     options: {
-                        sql: "select * from analysis_banks",
+                        sql: 'select * from analysis_banks',
                         cartocss: cartocss('#2167AB'),
                         cartocss_version: '2.3.0'
                     }
@@ -91,7 +126,7 @@ describe('analysis-layers use cases', function() {
                 {
                     type: 'cartodb',
                     options: {
-                        sql: "select * from analysis_rent_listings",
+                        sql: 'select * from analysis_rent_listings',
                         cartocss: DEFAULT_MULTITYPE_STYLE,
                         cartocss_version: '2.3.0'
                     }
@@ -105,30 +140,27 @@ describe('analysis-layers use cases', function() {
                 {
                     type: 'cartodb',
                     options: {
-                        sql: "select * from analysis_rent_listings",
+                        sql: 'select * from analysis_rent_listings',
                         cartocss: DEFAULT_MULTITYPE_STYLE,
                         cartocss_version: '2.3.0'
                     }
                 },
                 {
-                    type: 'analysis',
+                    type: 'cartodb',
                     options: {
-                        def: analysisDef({
-                            "type": "buffer",
-                            "params": {
-                                "source": {
-                                    "type": "source",
-                                    "params": {
-                                        "query": "select * from analysis_banks"
-                                    }
-                                },
-                                "radius": 250
-                            }
-                        }),
-                        cartocss: cartocss('black', 0.5)
+                        source: {
+                            id: 'b1'
+                        },
+                        cartocss: DEFAULT_MULTITYPE_STYLE,
+                        cartocss_version: '2.3.0'
                     }
                 }
-            ])
+            ],
+                {},
+                [
+                    bufferDef
+                ]
+            )
         },
 
         {
@@ -137,531 +169,115 @@ describe('analysis-layers use cases', function() {
                 {
                     type: 'cartodb',
                     options: {
-                        sql: "select * from analysis_rent_listings",
+                        sql: 'select * from analysis_rent_listings',
                         cartocss: DEFAULT_MULTITYPE_STYLE,
                         cartocss_version: '2.3.0'
                     }
                 },
                 {
-                    type: 'analysis',
+                    type: 'cartodb',
                     options: {
-                        def: analysisDef({
-                            "type": "point-in-polygon",
-                            "params": {
-                                "pointsSource": {
-                                    "type": "source",
-                                    "params": {
-                                        "query": "select * from analysis_rent_listings"
-                                    }
-                                },
-                                "polygonsSource": {
-                                    "type": "buffer",
-                                    "params": {
-                                        "source": {
-                                            "type": "source",
-                                            "params": {
-                                                "query": "select * from analysis_banks"
-                                            }
-                                        },
-                                        "radius": 250
-                                    }
-                                }
-                            }
-                        }),
-                        cartocss: cartocss('green', 1.0)
+                        source: {
+                            id: 'a1'
+                        },
+                        cartocss: DEFAULT_MULTITYPE_STYLE,
+                        cartocss_version: '2.3.0'
                     }
                 }
-            ])
+            ],
+                {},
+                [
+                    pointInPolygonDef
+                ]
+            )
         },
 
         {
             desc: 'point-in-polygon from buffer atm-machines and rent listings + rent listings',
-            mapConfig: mapConfig([
-                {
-                    type: 'analysis',
-                    options: {
-                        def: analysisDef({
-                            "type": "point-in-polygon",
-                            "params": {
-                                "pointsSource": {
-                                    "type": "source",
-                                    "params": {
-                                        "query": "select * from analysis_rent_listings"
-                                    }
-                                },
-                                "polygonsSource": {
-                                    "type": "buffer",
-                                    "params": {
-                                        "source": {
-                                            "type": "source",
-                                            "params": {
-                                                "query": "select * from analysis_banks"
-                                            }
-                                        },
-                                        "radius": 250
-                                    }
-                                }
-                            }
-                        }),
-                        cartocss: cartocss('green', 1.0)
-                    }
-                },
-                {
-                    type: 'cartodb',
-                    options: {
-                        sql: "select * from analysis_rent_listings",
-                        cartocss: DEFAULT_MULTITYPE_STYLE,
-                        cartocss_version: '2.3.0'
-                    }
-                }
-            ])
-        },
-
-        {
-            desc: 'buffer + point-in-polygon from buffer atm-machines and rent listings + rent listings',
-            mapConfig: mapConfig([
-                {
-                    type: 'cartodb',
-                    options: {
-                        sql: "select * from analysis_rent_listings",
-                        cartocss: DEFAULT_MULTITYPE_STYLE,
-                        cartocss_version: '2.3.0'
-                    }
-                },
-                {
-                    type: 'analysis',
-                    options: {
-                        def: analysisDef({
-                            "type": "buffer",
-                            "params": {
-                                "source": {
-                                    "type": "source",
-                                    "params": {
-                                        "query": "select * from analysis_banks"
-                                    }
-                                },
-                                "radius": 300
-                            }
-                        }),
-                        cartocss: cartocss('magenta', 0.5)
-                    }
-                },
-                {
-                    type: 'analysis',
-                    options: {
-                        def: analysisDef({
-                            "type": "point-in-polygon",
-                            "params": {
-                                "pointsSource": {
-                                    "type": "source",
-                                    "params": {
-                                        "query": "select * from analysis_rent_listings"
-                                    }
-                                },
-                                "polygonsSource": {
-                                    "type": "buffer",
-                                    "params": {
-                                        "source": {
-                                            "type": "source",
-                                            "params": {
-                                                "query": "select * from analysis_banks"
-                                            }
-                                        },
-                                        "radius": 300
-                                    }
-                                }
-                            }
-                        }),
-                        cartocss: cartocss('green', 1.0)
-                    }
-                }
-            ])
-        },
-
-        {
-            skip: true,
-            desc: 'buffer + point-in-polygon from buffer atm-machines and rent listings + rent listings',
-            mapConfig: mapConfig([
-                {
-                    type: 'cartodb',
-                    options: {
-                        "source": { id: "a" },
-                        "cartocss": DEFAULT_MULTITYPE_STYLE,
-                        "cartocss_version": "2.3.0"
-                    }
-                },
-                {
-                    type: 'cartodb',
-                    options: {
-                        "source": { id: "b1" },
-                        "cartocss": cartocss('green', 1.0),
-                        "cartocss_version": "2.3.0"
-                    }
-                },
-                {
-                    type: 'cartodb',
-                    options: {
-                        "source": { id: "b2" },
-                        "cartocss": cartocss('magenta', 0.5),
-                        "cartocss_version": "2.3.0"
-                    }
-                }
-            ],
-            [
-                {
-                    id: "b2",
-                    options: {
-                        def: analysisDef({
-                            "type": "count-in-polygon",
-                            "id": "a0",
-                            "params": {
-                                "columnName": 'count_airbnb',
-                                "pointsSource": {
-                                    "type": "source",
-                                    "params": {
-                                        query: "select * from analysis_rent_listings"
-                                    },
-                                    dataviews: {
-                                        price_histogram: {
-                                            type: 'histogram',
-                                            options: {
-                                                column: 'price'
-                                            }
-                                        }
-                                    }
-                                },
-                                "polygonsSource": {
-                                    "id": "b1",
-                                    "type": "buffer",
-                                    "params": {
-                                        "source": {
-                                            "id": "b0",
-                                            "type": "source",
-                                            "params": {
-                                                query: "select * from analysis_banks"
-                                            }
-                                        },
-                                        "radius": 250
-                                    },
-                                    dataviews: {
-                                        bank_category: {
-                                            type: 'aggregation',
-                                            options: {
-                                                column: 'bank'
-                                            }
-                                        }
-                                    }
-                                }
-                            },
-                            dataviews: {
-                                count_histogram: {
-                                    type: 'histogram',
-                                    options: {
-                                        column: 'count_airbnb'
-                                    }
-                                }
-                            }
-                        }),
-                        cartocss: cartocss('green', 1.0)
-                    }
-                }
-            ])
-        },
-
-        {
-            skip: true,
-            desc: 'I. Distribution centers',
             mapConfig: mapConfig(
-                // layers
                 [
                     {
                         type: 'cartodb',
                         options: {
-                            "source": { id: "b0" },
-                            "cartocss": [
-                                "#distribution_centers {",
-                                "  marker-fill-opacity: 1.0;",
-                                "  marker-line-color: #FFF;",
-                                "  marker-line-width: 0.5;",
-                                "  marker-line-opacity: 0.7;",
-                                "  marker-placement: point;",
-                                "  marker-type: ellipse;",
-                                "  marker-width: 8;",
-                                "  marker-fill: blue;",
-                                "  marker-allow-overlap: true;",
-                                "}"
-                            ].join('\n'),
-                            "cartocss_version": "2.3.0"
+                            source: {
+                                id: 'a1'
+                            },
+                            cartocss: DEFAULT_MULTITYPE_STYLE,
+                            cartocss_version: '2.3.0'
                         }
                     },
                     {
                         type: 'cartodb',
                         options: {
-                            "source": { id: "a0" },
-                            "cartocss": [
-                                "#shops {",
-                                "  marker-fill-opacity: 1.0;",
-                                "  marker-line-color: #FFF;",
-                                "  marker-line-width: 0.5;",
-                                "  marker-line-opacity: 0.7;",
-                                "  marker-placement: point;",
-                                "  marker-type: ellipse;",
-                                "  marker-width: 8;",
-                                "  marker-fill: red;",
-                                "  marker-allow-overlap: true;",
-                                "}"
-                            ].join('\n'),
-                            "cartocss_version": "2.3.0"
-                        }
-                    },
-                    {
-                        type: 'cartodb',
-                        options: {
-                            "source": { id: "a1" },
-                            "cartocss": [
-                                "#routing {",
-                                "  line-color: ramp([routing_time], colorbrewer(Reds));",
-                                "  line-width: ramp([routing_time], 2, 8);",
-                                "  line-opacity: 1.0;",
-                                "}"
-                            ].join('\n'),
-                            "cartocss_version": "2.3.0"
+                            sql: 'select * from analysis_rent_listings',
+                            cartocss: DEFAULT_MULTITYPE_STYLE,
+                            cartocss_version: '2.3.0'
                         }
                     }
                 ],
-                // dataviews
-                {
-                    distribution_center_name_category: {
-                        source: { id: 'b0' },
-                        type: 'aggregation',
-                        options: {
-                            column: 'name'
-                        }
-                    },
-                    time_histogram: {
-                        source: { id: 'a1' },
-                        type: 'histogram',
-                        options: {
-                            column: 'routing_time'
-                        }
-                    },
-                    distance_histogram: {
-                        source: { id: 'a1' },
-                        type: 'histogram',
-                        options: {
-                            column: 'routing_distance'
-                        }
-                    }
-                },
-                // analysis
+                {},
                 [
-                    {
-                        id: 'a1',
-                        type: 'routing-n-to-n',
-                        params: {
-                            // distanceColumn: 'routing_distance',
-                            // timeColumn: 'routing_time',
-                            originSource: {
-                                id: 'b0',
-                                type: 'source',
-                                params: {
-                                    query: 'select * from distribution_centers'
-                                }
-                            },
-                            destinationSource: {
-                                id: 'a0',
-                                type: 'source',
-                                params: {
-                                    query: 'select * from shops'
-                                }
-                            }
-                        }
-                    }
+                    pointInPolygonDef
                 ]
             )
         },
 
         {
-            skip: true,
-            desc: 'II. Population analysis',
+            desc: 'buffer + point-in-polygon from buffer atm-machines and rent listings + rent listings',
             mapConfig: mapConfig(
-            // layers
-            [
-                {
-                    type: 'cartodb',
-                    options: {
-                        "source": { id: "a2" },
-                        "cartocss": [
-                            "#count_in_polygon {",
-                            "  polygon-opacity: 1.0",
-                            "  line-color: #FFF;",
-                            "  line-width: 0.5;",
-                            "  line-opacity: 0.7",
-                            "  polygon-fill: ramp([estimated_people], colorbrewer(Reds));",
-                            "}"
-                        ].join('\n'),
-                        "cartocss_version": "2.3.0"
-                    }
-                },
-                {
-                    type: 'cartodb',
-                    options: {
-                        "source": { id: "a0" },
-                        "cartocss": DEFAULT_MULTITYPE_STYLE,
-                        "cartocss_version": "2.3.0"
-                    }
-                }
-            ],
-            // dataviews
-            {
-                total_population_formula: {
-                    "source": { id: "a3" },
-                    type: 'formula',
-                    options: {
-                        column: 'total_population',
-                        operation: 'sum'
-                    }
-                },
-                people_histogram: { // this injects a range filter at `a2` node output
-                    "source": { id: "a2" },
-                    type: 'histogram',
-                    options: {
-                        column: 'estimated_people'
-                    }
-                },
-                subway_line_category: { // this injects a category filter at `a0` node output
-                    "source": { id: "a0" },
-                    type: 'aggregation',
-                    options: {
-                        column: 'subway_line'
-                    }
-                }
-            },
-            // analysis
-            [
-                {
-                    id: 'a3',
-                    // this will union the polygons, produce just one polygon, and calculate the total population for it
-                    type: 'total-population',
-                    params: {
-                        columnName: 'total_population',
-                        source: {
-                            id: 'a2',
-                            type: 'estimated-population',
-                            params: {
-                                columnName: 'estimated_people',
-                                source: {
-                                    id: 'a1',
-                                    type: 'trade-area',
-                                    params: {
-                                        source: {
-                                            "id": "a0",
-                                            "type": "source",
-                                            "params": {
-                                                query: "select * from subway_stops"
-                                            }
-                                        },
-                                        kind: 'walk',
-                                        time: 300
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
-            ])
-        },
-
-        {
-            skip: true,
-            desc: 'III. Point in polygon',
-            mapConfig: mapConfig(
-                // layers
                 [
                     {
                         type: 'cartodb',
                         options: {
-                            "source": { id: "a1" },
-                            "cartocss": [
-                                "#count_in_polygon {",
-                                "  polygon-opacity: 1.0",
-                                "  line-color: #FFF;",
-                                "  line-width: 0.5;",
-                                "  line-opacity: 0.7",
-                                "  polygon-fill: ramp([count_people], colorbrewer(Reds));",
-                                "}"
-                            ].join('\n'),
-                            "cartocss_version": "2.3.0"
+                            sql: 'select * from analysis_rent_listings',
+                            cartocss: DEFAULT_MULTITYPE_STYLE,
+                            cartocss_version: '2.3.0'
+                        }
+                    },
+                    {
+                        type: 'cartodb',
+                        options: {
+                            source: {
+                                id: 'a1'
+                            },
+                            cartocss: DEFAULT_MULTITYPE_STYLE,
+                            cartocss_version: '2.3.0'
+                        }
+                    },
+                    {
+                        type: 'cartodb',
+                        options: {
+                            source: {
+                                id: 'b1'
+                            },
+                            cartocss: DEFAULT_MULTITYPE_STYLE,
+                            cartocss_version: '2.3.0'
                         }
                     }
                 ],
-                // dataviews
-                {
-                    age_histogram: {
-                        "source": { id: "a0" },
-                        type: 'histogram',
-                        options: {
-                            column: 'age'
-                        }
-                    },
-                    income_histogram: {
-                        "source": { id: "a0" },
-                        type: 'histogram',
-                        options: {
-                            column: 'income'
-                        }
-                    },
-                    gender_category: {
-                        "source": { id: "a0" },
-                        type: 'aggregation',
-                        options: {
-                            column: 'gender'
-                        }
-                    }
-                },
-                // analysis
+                {},
                 [
-                    {
-                        "id": "a1",
-                        "type": "count-in-polygon",
-                        "params": {
-                            "columnName": 'count_people',
-                            "pointsSource": {
-                                "id": 'a0',
-                                "type": "source",
-                                "params": {
-                                    query: "select the_geom, age, gender, income from people"
-                                }
-                            },
-                            "polygonsSource": {
-                                "id": "b0",
-                                "type": "source",
-                                "params": {
-                                    query: "select * from postal_codes"
-                                }
-                            }
-                        }
-                    }
+                    bufferDef,
+                    pointInPolygonDef
                 ]
             )
         }
-
     ];
 
-    useCases.forEach(function(useCase, imageIdx) {
+    useCases.forEach(function (useCase) {
         if (!!useCase.skip) {
-            debug(JSON.stringify(useCase.mapConfig, null, 4));
+            return debug(JSON.stringify(useCase.mapConfig, null, 4));
         }
-        it.skip('should implement use case: "' + useCase.desc + '"', function(done) {
+        it(`should implement use case: '${useCase.desc}'`, function (done) {
 
             var testClient = new TestClient(useCase.mapConfig, 1234);
 
             var tile = useCase.tile || TILE_ANALYSIS_TABLES;
 
-            testClient.getTile(tile.z, tile.x, tile.y, function(err, res, image) {
+            testClient.getTile(tile.z, tile.x, tile.y, function (err, res, image) {
                 assert.ok(!err, err);
 
-                image.save('/tmp/tests/' + imageIdx + '---' + useCase.desc.replace(/\s/g, '-') + '.png');
+                //image.save('/tmp/tests/' + imageIdx + '---' + useCase.desc.replace(/\s/g, '-') + '.png');
 
                 assert.equal(image.width(), 256);
 

test/acceptance/analysis/analysis-layers.js

@@ -174,7 +174,9 @@ describe('analysis-layers', function() {
             }
         };
 
-        testClient.getLayergroup(PERMISSION_DENIED_RESPONSE, function(err, layergroupResult) {
+
+
+        testClient.getLayergroup({ response: PERMISSION_DENIED_RESPONSE }, function(err, layergroupResult) {
             assert.ok(!err, err);
             assert.deepEqual(
                 layergroupResult.errors,

test/acceptance/analysis/error-cases.js

@@ -62,7 +62,7 @@ describe('analysis-layers error cases', function() {
 
         var testClient = new TestClient(mapConfig, 1234);
 
-        testClient.getLayergroup(ERROR_RESPONSE, function(err, layergroupResult) {
+        testClient.getLayergroup({ response: ERROR_RESPONSE }, function(err, layergroupResult) {
             assert.ok(!err, err);
 
             assert.equal(layergroupResult.errors.length, 1);
@@ -97,7 +97,7 @@ describe('analysis-layers error cases', function() {
 
         var testClient = new TestClient(mapConfig, 1234);
 
-        testClient.getLayergroup(ERROR_RESPONSE, function(err, layergroupResult) {
+        testClient.getLayergroup({ response: ERROR_RESPONSE }, function(err, layergroupResult) {
             assert.ok(!err, err);
 
             assert.equal(layergroupResult.errors.length, 1);
@@ -144,7 +144,7 @@ describe('analysis-layers error cases', function() {
 
         var testClient = new TestClient(mapConfig, 1234);
 
-        testClient.getLayergroup(ERROR_RESPONSE, function(err, layergroupResult) {
+        testClient.getLayergroup({ response: ERROR_RESPONSE }, function(err, layergroupResult) {
             assert.ok(!err, err);
 
             assert.equal(layergroupResult.errors.length, 1);
@@ -190,7 +190,7 @@ describe('analysis-layers error cases', function() {
 
         var testClient = new TestClient(mapConfig, 11111);
 
-        testClient.getLayergroup(AUTH_ERROR_RESPONSE, function(err, layergroupResult) {
+        testClient.getLayergroup({ response: AUTH_ERROR_RESPONSE }, function(err, layergroupResult) {
             assert.ok(!err, err);
 
             assert.equal(layergroupResult.errors.length, 1);
@@ -246,7 +246,7 @@ describe('analysis-layers error cases', function() {
 
         var testClient = new TestClient(mapConfig, 1234);
 
-        testClient.getLayergroup(ERROR_RESPONSE, function(err, layergroupResult) {
+        testClient.getLayergroup({ response: ERROR_RESPONSE }, function(err, layergroupResult) {
             assert.ok(!err, err);
 
             assert.equal(layergroupResult.errors.length, 1);
@@ -298,7 +298,7 @@ describe('analysis-layers error cases', function() {
 
         var testClient = new TestClient(mapConfig, 1234);
 
-        testClient.getLayergroup(ERROR_RESPONSE, function(err, layergroupResult) {
+        testClient.getLayergroup({ response: ERROR_RESPONSE }, function(err, layergroupResult) {
             assert.ok(!err, err);
 
             assert.equal(layergroupResult.errors.length, 1);
@@ -351,7 +351,7 @@ describe('analysis-layers error cases', function() {
 
         var testClient = new TestClient(mapConfig, 1234);
 
-        testClient.getLayergroup(ERROR_RESPONSE, function(err, layergroupResult) {
+        testClient.getLayergroup({ response: ERROR_RESPONSE }, function(err, layergroupResult) {
             assert.ok(!err, err);
 
             assert.equal(layergroupResult.errors.length, 1);
@@ -415,7 +415,7 @@ describe('analysis-layers error cases', function() {
 
         var testClient = new TestClient(mapConfig, 1234);
 
-        testClient.getLayergroup(ERROR_RESPONSE, function(err, layergroupResult) {
+        testClient.getLayergroup({ response: ERROR_RESPONSE }, function(err, layergroupResult) {
             assert.ok(!err, err);
 
             assert.equal(layergroupResult.errors.length, 1);

test/acceptance/analysis/named-maps.js

@@ -7,7 +7,7 @@ var serverOptions = require('../../../lib/cartodb/server_options');
 var server = new CartodbWindshaft(serverOptions);
 var TestClient = require('../../support/test-client');
 
-var LayergroupToken = require('../../support/layergroup-token');
+var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token');
 
 describe('named-maps analysis', function() {
 

test/acceptance/buffer-size-format.js

@@ -3,6 +3,7 @@ require('../support/test_helper');
 var fs = require('fs');
 var assert = require('../support/assert');
 var TestClient = require('../support/test-client');
+var serverOptions = require('../../lib/cartodb/server_options');
 var mapnik = require('windshaft').mapnik;
 var IMAGE_TOLERANCE_PER_MIL = 5;
 
@@ -124,24 +125,37 @@ describe('buffer size per format', function () {
         }
     ];
 
+    afterEach(function(done) {
+        if (this.testClient) {
+            return this.testClient.drain(done);
+        }
+        return done();
+    });
+
+    const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS;
     testCases.forEach(function (test) {
-        it(test.desc, function (done) {
-            var testClient = new TestClient(test.mapConfig, 1234);
-            var coords = test.coords;
-            var options = {
-                format: test.format,
-                layers: test.layers
-            };
-            testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) {
-                assert.ifError(err);
-                // To generate images use:
-                // tile.save(test.fixturePath);
-                test.assert(tile, function (err) {
+        var testFn = (usePostGIS) => {
+            it(test.desc, function (done) {
+                serverOptions.renderer.mvt.usePostGIS = usePostGIS;
+                this.testClient = new TestClient(test.mapConfig, 1234);
+                serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS;
+                var coords = test.coords;
+                var options = {
+                    format: test.format,
+                    layers: test.layers
+                };
+                this.testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) {
                     assert.ifError(err);
-                    testClient.drain(done);
+                    // To generate images use:
+                    // tile.save(test.fixturePath);
+                    test.assert(tile, done);
                 });
             });
-        });
+        };
+        if (process.env.POSTGIS_VERSION === '2.4' && test.format === 'mvt'){
+            testFn(true);
+        }
+        testFn(false);
     });
 });
 
@@ -260,23 +274,27 @@ describe('buffer size per format for named maps', function () {
         }
     ];
 
+    afterEach(function(done) {
+        if (this.testClient) {
+            return this.testClient.drain(done);
+        }
+        return done();
+    });
+
     testCases.forEach(function (test) {
         it(test.desc, function (done) {
-            var testClient = new TestClient(test.template, 1234);
+            this.testClient = new TestClient(test.template, 1234);
             var coords = test.coords;
             var options = {
                 format: test.format,
                 placeholders: test.placeholders,
                 layers: test.layers
             };
-            testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) {
+            this.testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) {
                 assert.ifError(err);
                 // To generate images use:
                 //tile.save('./test/fixtures/buffer-size/tile-7.64.48-buffer-size-0-test.png');
-                test.assert(tile, function (err) {
-                    assert.ifError(err);
-                    testClient.drain(done);
-                });
+                test.assert(tile, done);
             });
         });
     });
@@ -416,26 +434,40 @@ describe('buffer size per format for named maps w/o placeholders', function () {
 
     ];
 
+    afterEach(function(done) {
+        if (this.testClient) {
+            return this.testClient.drain(done);
+        }
+        return done();
+    });
+
+    const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS;
     testCases.forEach(function (test) {
-        it(test.desc, function (done) {
-            var testClient = new TestClient(test.template, 1234);
-            var coords = test.coords;
-            var options = {
-                format: test.format,
-                placeholders: test.placeholders,
-                layers: test.layers
-            };
-            testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) {
-                assert.ifError(err);
-                // To generate images use:
-                //tile.save(test.fixturePath);
-                // require('fs').writeFileSync(test.fixturePath, JSON.stringify(tile));
-                // require('fs').writeFileSync(test.fixturePath, tile.getDataSync());
-                test.assert(tile, function (err) {
-                    assert.ifError(err);
-                    testClient.drain(done);
+        var testFn = (usePostGIS) => {
+                it(test.desc + `(${usePostGIS? 'PostGIS':'mapnik'})`, function (done) {
+                    serverOptions.renderer.mvt.usePostGIS = usePostGIS;
+                    test.template.name += '_1';
+                    this.testClient = new TestClient(test.template, 1234);
+                    serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS;
+                    var coords = test.coords;
+                    var options = {
+                        format: test.format,
+                        placeholders: test.placeholders,
+                        layers: test.layers
+                    };
+                    this.testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) {
+                        assert.ifError(err);
+                        // To generate images use:
+                        //tile.save(test.fixturePath);
+                        // require('fs').writeFileSync(test.fixturePath, JSON.stringify(tile));
+                        // require('fs').writeFileSync(test.fixturePath, tile.getDataSync());
+                        test.assert(tile, done);
+                    });
                 });
-            });
-        });
+        };
+        if (process.env.POSTGIS_VERSION === '2.4' && test.format === 'mvt'){
+            testFn(true);
+        }
+        testFn(false);
     });
 });

test/acceptance/cache/cache_headers.js

@@ -8,7 +8,7 @@ var serverOptions = require('../../../lib/cartodb/server_options');
 var server = new CartodbWindshaft(serverOptions);
 server.setMaxListeners(0);
 
-var LayergroupToken = require('../../support/layergroup-token');
+var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token');
 
 describe('get requests with cache headers', function() {
 

test/acceptance/dataviews/aggregation.js

@@ -324,3 +324,104 @@ describe('aggregation-dataview: special float values', function() {
         });
     });
 });
+
+describe('aggregation dataview tuned by categories query param', function () {
+    const mapConfig = {
+        version: '1.5.0',
+        layers: [
+            {
+                type: "cartodb",
+                options: {
+                    source: {
+                        "id": "a0"
+                    },
+                    cartocss: "#points { marker-width: 10; marker-fill: red; }",
+                    cartocss_version: "2.3.0"
+                }
+            }
+        ],
+        dataviews: {
+            categories: {
+                source: {
+                    id: 'a0'
+                },
+                type: 'aggregation',
+                options: {
+                    column: 'cat',
+                    aggregation: 'sum',
+                    aggregationColumn: 'val'
+                }
+            }
+        },
+        analyses: [
+            {
+                id: "a0",
+                type: "source",
+                params: {
+                    query: `
+                        SELECT
+                            null::geometry the_geom_webmercator,
+                            CASE
+                                WHEN x % 4 = 0 THEN 1
+                                WHEN x % 4 = 1 THEN 2
+                                WHEN x % 4 = 2 THEN 3
+                                ELSE 4
+                            END AS val,
+                            CASE
+                                WHEN x % 4 = 0 THEN 'category_1'
+                                WHEN x % 4 = 1 THEN 'category_2'
+                                WHEN x % 4 = 2 THEN 'category_3'
+                                ELSE 'category_4'
+                            END AS cat
+                        FROM generate_series(1, 1000) x
+                    `
+                }
+            }
+        ]
+    };
+
+    beforeEach(function () {
+        this.testClient = new TestClient(mapConfig, 1234);
+    });
+
+    afterEach(function (done) {
+        this.testClient.drain(done);
+    });
+
+    var scenarios = [
+        {
+            params: { own_filter: 0, categories: -1 },
+            categoriesExpected: 4
+        },
+        {
+            params: { own_filter: 0, categories: 0 },
+            categoriesExpected: 4
+        },
+        {
+            params: { own_filter: 0, categories: 1 },
+            categoriesExpected: 1
+        },
+        {
+            params: { own_filter: 0, categories: 2 },
+            categoriesExpected: 2
+        },
+        {
+            params: { own_filter: 0, categories: 4 },
+            categoriesExpected: 4
+        },
+        {
+            params: { own_filter: 0, categories: 5 },
+            categoriesExpected: 4
+        }
+    ];
+
+    scenarios.forEach(function (scenario) {
+        it(`should handle cartegories to customize aggregations: ${JSON.stringify(scenario.params)}`, function (done) {
+            this.testClient.getDataview('categories', scenario.params, (err, dataview) => {
+                assert.ifError(err);
+                assert.equal(dataview.categories.length, scenario.categoriesExpected);
+                done();
+            });
+        });
+    });
+});

test/acceptance/dataviews/error-cases.js

@@ -51,7 +51,7 @@ describe('histogram-dataview', function() {
     it('should fail when invalid dataviews object is provided, string case', function(done) {
         var mapConfig = createMapConfig("wadus-string");
         this.testClient = new TestClient(mapConfig, 1234);
-        this.testClient.getLayergroup(ERROR_RESPONSE, function(err, errObj) {
+        this.testClient.getLayergroup({ response: ERROR_RESPONSE }, function(err, errObj) {
             assert.ok(!err, err);
 
             assert.deepEqual(errObj.errors, [ '"dataviews" must be a valid JSON object: "string" type found' ]);
@@ -63,7 +63,7 @@ describe('histogram-dataview', function() {
     it('should fail when invalid dataviews object is provided, array case', function(done) {
         var mapConfig = createMapConfig([]);
         this.testClient = new TestClient(mapConfig, 1234);
-        this.testClient.getLayergroup(ERROR_RESPONSE, function(err, errObj) {
+        this.testClient.getLayergroup({ response: ERROR_RESPONSE }, function(err, errObj) {
             assert.ok(!err, err);
 
             assert.deepEqual(errObj.errors, [ '"dataviews" must be a valid JSON object: "array" type found' ]);

test/acceptance/dataviews/histogram.js

@@ -124,7 +124,7 @@ describe('histogram-dataview for date column type', function() {
                 "type": "cartodb",
                 "options": {
                     "source": {
-                        "id": "datetime-histogram-source"
+                        "id": "datetime-histogram-source-week"
                     },
                     "cartocss": "#points { marker-width: 10; marker-fill: red; }",
                     "cartocss_version": "2.3.0"
@@ -134,7 +134,7 @@ describe('histogram-dataview for date column type', function() {
         {
             datetime_histogram: {
                 source: {
-                    id: 'datetime-histogram-source'
+                    id: 'datetime-histogram-source-week'
                 },
                 type: 'histogram',
                 options: {
@@ -154,9 +154,109 @@ describe('histogram-dataview for date column type', function() {
                     offset: -14400 // EDT Eastern Daylight Time (GMT-4) in seconds
                 }
             },
-            datetime_histogram_automatic: {
+            datetime_histogram_automatic_second: {
                 source: {
-                    id: 'datetime-histogram-source'
+                    id: 'datetime-histogram-source-second'
+                },
+                type: 'histogram',
+                options: {
+                    column: 'd',
+                    aggregation: 'auto'
+                }
+            },
+            datetime_histogram_automatic_minute: {
+                source: {
+                    id: 'datetime-histogram-source-minute'
+                },
+                type: 'histogram',
+                options: {
+                    column: 'd',
+                    aggregation: 'auto'
+                }
+            },
+            datetime_histogram_automatic_hour: {
+                source: {
+                    id: 'datetime-histogram-source-hour'
+                },
+                type: 'histogram',
+                options: {
+                    column: 'd',
+                    aggregation: 'auto'
+                }
+            },
+            datetime_histogram_automatic_day: {
+                source: {
+                    id: 'datetime-histogram-source-day'
+                },
+                type: 'histogram',
+                options: {
+                    column: 'd',
+                    aggregation: 'auto'
+                }
+            },
+            datetime_histogram_automatic_week: {
+                source: {
+                    id: 'datetime-histogram-source-week'
+                },
+                type: 'histogram',
+                options: {
+                    column: 'd',
+                    aggregation: 'auto'
+                }
+            },
+            datetime_histogram_automatic_month: {
+                source: {
+                    id: 'datetime-histogram-source-month'
+                },
+                type: 'histogram',
+                options: {
+                    column: 'd',
+                    aggregation: 'auto'
+                }
+            },
+            datetime_histogram_automatic_quarter: {
+                source: {
+                    id: 'datetime-histogram-source-quarter'
+                },
+                type: 'histogram',
+                options: {
+                    column: 'd',
+                    aggregation: 'auto'
+                }
+            },
+            datetime_histogram_automatic_year: {
+                source: {
+                    id: 'datetime-histogram-source-year'
+                },
+                type: 'histogram',
+                options: {
+                    column: 'd',
+                    aggregation: 'auto'
+                }
+            },
+            datetime_histogram_automatic_decade: {
+                source: {
+                    id: 'datetime-histogram-source-decade'
+                },
+                type: 'histogram',
+                options: {
+                    column: 'd',
+                    aggregation: 'auto'
+                }
+            },
+            datetime_histogram_automatic_century: {
+                source: {
+                    id: 'datetime-histogram-source-century'
+                },
+                type: 'histogram',
+                options: {
+                    column: 'd',
+                    aggregation: 'auto'
+                }
+            },
+            datetime_histogram_automatic_millennium: {
+                source: {
+                    id: 'datetime-histogram-source-millennium'
                 },
                 type: 'histogram',
                 options: {
@@ -186,7 +286,7 @@ describe('histogram-dataview for date column type', function() {
             },
             minute_histogram: {
                 source: {
-                    id: 'minute-histogram-source'
+                    id: 'minute-histogram-source-tz'
                 },
                 type: 'histogram',
                 options: {
@@ -197,13 +297,144 @@ describe('histogram-dataview for date column type', function() {
         },
         [
             {
-                "id": "datetime-histogram-source",
+                "id": "datetime-histogram-source-second",
                 "type": "source",
                 "params": {
                     "query": [
                         "select null::geometry the_geom_webmercator, date AS d",
                         "from generate_series(",
-                            "'2007-02-15 01:00:00'::timestamp, '2008-04-09 01:00:00'::timestamp, '1 day'::interval",
+                            "'2007-02-15 01:00:00'::timestamp, '2007-02-15 01:00:57'::timestamp,",
+                             "'0.9 second'::interval",
+                        ") date"
+                    ].join(' ')
+                }
+            },
+            {
+                "id": "datetime-histogram-source-minute",
+                "type": "source",
+                "params": {
+                    "query": [
+                        "select null::geometry the_geom_webmercator, date AS d",
+                        "from generate_series(",
+                            "'2007-02-15 01:00:00'::timestamp, '2007-02-15 02:00:57'::timestamp,",
+                            "'75 second'::interval",
+                        ") date"
+                    ].join(' ')
+                }
+            },
+            {
+                "id": "datetime-histogram-source-hour",
+                "type": "source",
+                "params": {
+                    "query": [
+                        "select null::geometry the_geom_webmercator, date AS d",
+                        "from generate_series(",
+                            "'2007-02-15 01:00:00'::timestamp, '2007-02-18 02:00:57'::timestamp,",
+                            "'47 minutes'::interval",
+                        ") date"
+                    ].join(' ')
+                }
+            },
+            {
+                "id": "datetime-histogram-source-day",
+                "type": "source",
+                "params": {
+                    "query": [
+                        "select null::geometry the_geom_webmercator, date AS d",
+                        "from generate_series(",
+                            "'2007-02-15 01:00:00'::timestamp, '2007-04-18 02:00:57'::timestamp,",
+                            "'24 hours'::interval",
+                        ") date"
+                    ].join(' ')
+                }
+            },
+            {
+                "id": "datetime-histogram-source-week",
+                "type": "source",
+                "params": {
+                    "query": [
+                        "select null::geometry the_geom_webmercator, date AS d",
+                        "from generate_series(",
+                            "'2007-02-15 01:00:00'::timestamp, '2008-04-09 01:00:00'::timestamp,",
+                            "'1 day'::interval",
+                        ") date"
+                    ].join(' ')
+                }
+            },
+            {
+                "id": "datetime-histogram-source-month",
+                "type": "source",
+                "params": {
+                    "query": [
+                        "select null::geometry the_geom_webmercator, date AS d",
+                        "from generate_series(",
+                            "'2007-02-15 01:00:00'::timestamp, '2010-04-09 01:00:00'::timestamp,",
+                            "'30 day'::interval",
+                        ") date"
+                    ].join(' ')
+                }
+            },
+            {
+                "id": "datetime-histogram-source-quarter",
+                "type": "source",
+                "params": {
+                    "query": [
+                        "select null::geometry the_geom_webmercator, date AS d",
+                        "from generate_series(",
+                            "'2007-02-15 01:00:00'::timestamp, '2020-04-09 01:00:00'::timestamp,",
+                            "'30 day'::interval",
+                        ") date"
+                    ].join(' ')
+                }
+            },
+            {
+                "id": "datetime-histogram-source-year",
+                "type": "source",
+                "params": {
+                    "query": [
+                        "select null::geometry the_geom_webmercator, date AS d",
+                        "from generate_series(",
+                            "'1990-02-15 01:00:00'::timestamp, '2018-04-09 01:00:00'::timestamp,",
+                            "'30 day'::interval",
+                        ") date"
+                    ].join(' ')
+                }
+            },
+            {
+                "id": "datetime-histogram-source-decade",
+                "type": "source",
+                "params": {
+                    "query": [
+                        "select null::geometry the_geom_webmercator, date AS d",
+                        "from generate_series(",
+                            "'1850-02-15 01:00:00'::timestamp, '2018-04-09 01:00:00'::timestamp,",
+                            "'30 day'::interval",
+                        ") date"
+                    ].join(' ')
+                }
+            },
+            {
+                "id": "datetime-histogram-source-century",
+                "type": "source",
+                "params": {
+                    "query": [
+                        "select null::geometry the_geom_webmercator, date AS d",
+                        "from generate_series(",
+                            "'0650-02-15 01:00:00'::timestamp, '1918-04-09 01:00:00'::timestamp,",
+                            "'6 years'::interval",
+                        ") date"
+                    ].join(' ')
+                }
+            },
+            {
+                "id": "datetime-histogram-source-millennium",
+                "type": "source",
+                "params": {
+                    "query": [
+                        "select null::geometry the_geom_webmercator, date AS d",
+                        "from generate_series(",
+                            "'0005-02-15 01:00:00'::timestamp, '12000-04-09 01:00:00'::timestamp,",
+                            "'72 years'::interval",
                         ") date"
                     ].join(' ')
                 }
@@ -214,8 +445,8 @@ describe('histogram-dataview for date column type', function() {
                 "params": {
                     "query": [
                         "select null::geometry the_geom_webmercator, date AS d",
-                        "from generate_series(",
-                            "'2007-02-15 01:00:00'::timestamptz, '2008-04-09 01:00:00'::timestamptz, '1 day'::interval",
+                        "from generate_series('2007-02-15 01:00:00+00'::timestamptz,",
+                        "'2008-04-09 01:00:00+00'::timestamptz, '1 day'::interval",
                         ") date"
                     ].join(' ')
                 }
@@ -233,13 +464,13 @@ describe('histogram-dataview for date column type', function() {
                 }
             },
             {
-                "id": "minute-histogram-source",
+                "id": "minute-histogram-source-tz",
                 "type": "source",
                 "params": {
                     "query": [
                         "select null::geometry the_geom_webmercator, date AS d",
-                        "from generate_series(",
-                            "'2007-02-15 23:50:00'::timestamp, '2007-02-16 00:10:00'::timestamp, '1 minute'::interval",
+                        "from generate_series('2007-02-15 23:50:00+00'::timestamptz,",
+                        "'2007-02-16 00:10:00+00'::timestamptz, '1 minute'::interval",
                         ") date"
                     ].join(' ')
                 }
@@ -256,6 +487,7 @@ describe('histogram-dataview for date column type', function() {
     }];
 
     dateHistogramsUseCases.forEach(function (test) {
+
         it('should create a date histogram aggregated in months (EDT) ' + test.desc, function (done) {
             var OFFSET_EDT_IN_MINUTES = -4 * 60; // EDT Eastern Daylight Time (GMT-4) in minutes
 
@@ -323,7 +555,7 @@ describe('histogram-dataview for date column type', function() {
                 assert.ok(!err, err);
                 assert.equal(dataview.type, 'histogram');
                 assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width);
-                assert.equal(dataview.bins.length, 6);
+                assert.equal(dataview.bins_count, 6);
                 dataview.bins.forEach(function (bin) {
                     assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin));
                 });
@@ -332,28 +564,22 @@ describe('histogram-dataview for date column type', function() {
             });
         });
 
-
-        it('should return same histogram ' + test.desc, function (done) {
+        it('should cast overridden start and end to float to avoid out of range errors ' + test.desc, function (done) {
             var params = {
-                start: 1171501200, // 2007-02-15 01:00:00 = min(date_colum)
-                end: 1207702800 // 2008-04-09 01:00:00 = max(date_colum)
+                start: -2145916800,
+                end: 1193792400
             };
 
             this.testClient = new TestClient(mapConfig, 1234);
-            this.testClient.getDataview(test.dataviewId, {}, function (err, dataview) {
+            this.testClient.getDataview(test.dataviewId, params, function (err, dataview) {
                 assert.ok(!err, err);
+                assert.equal(dataview.type, 'histogram');
+                assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width);
 
-                this.testClient = new TestClient(mapConfig, 1234);
-                this.testClient.getDataview(test.dataviewId, params, function (err, filteredDataview) {
-                    assert.ok(!err, err);
-
-                    assert.deepEqual(dataview, filteredDataview);
-                    done();
-                });
+                done();
             });
         });
 
-
         it('should aggregate histogram overriding default offset to CEST ' + test.desc, function (done) {
             var OFFSET_CEST_IN_SECONDS = 2 * 3600; // Central European Summer Time (Daylight Saving Time)
             var OFFSET_CEST_IN_MINUTES = 2 * 60; // Central European Summer Time (Daylight Saving Time)
@@ -434,6 +660,47 @@ describe('histogram-dataview for date column type', function() {
             });
         });
 
+        it('should aggregate histogram using "second" aggregation ' + test.desc, function (done) {
+            var OFFSET_UTC_IN_SECONDS = 0 * 3600; // UTC
+            var OFFSET_UTC_IN_MINUTES = 0 * 60; // UTC
+            var params = {
+                offset: OFFSET_UTC_IN_SECONDS,
+                aggregation: 'second'
+            };
+
+            this.testClient = new TestClient(mapConfig, 1234);
+            this.testClient.getDataview('datetime_histogram_automatic_second', params, function (err, dataview) {
+                assert.ok(!err, err);
+                assert.equal(dataview.type, 'histogram');
+                assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width);
+                assert.equal(dataview.bins.length, 57);
+
+                var initialTimestamp = '2007-02-15T01:00:00Z';
+                var binsStartInMilliseconds = dataview.bins_start * 1000;
+                var binsStartFormatted = moment.utc(binsStartInMilliseconds)
+                    .utcOffset(OFFSET_UTC_IN_MINUTES)
+                    .format();
+                assert.equal(binsStartFormatted, initialTimestamp);
+
+                dataview.bins.forEach(function (bin, index) {
+                    var binTimestampExpected = moment.utc(initialTimestamp)
+                        .utcOffset(OFFSET_UTC_IN_MINUTES)
+                        .add(index, 'second')
+                        .format();
+                    var binsTimestampInMilliseconds = bin.timestamp * 1000;
+                    var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds)
+                        .utcOffset(OFFSET_UTC_IN_MINUTES)
+                        .format();
+
+                    assert.equal(binTimestampFormatted, binTimestampExpected);
+                    assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin));
+                    assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin));
+                });
+
+                done();
+            });
+        });
+
         it('should aggregate histogram using "quarter" aggregation ' + test.desc, function (done) {
             var OFFSET_UTC_IN_SECONDS = 0 * 3600; // UTC
             var OFFSET_UTC_IN_MINUTES = 0 * 60; // UTC
@@ -475,6 +742,132 @@ describe('histogram-dataview for date column type', function() {
             });
         });
 
+        it('should aggregate histogram using "decade" aggregation ' + test.desc, function (done) {
+            var OFFSET_UTC_IN_SECONDS = 0 * 3600; // UTC
+            var OFFSET_UTC_IN_MINUTES = 0 * 60; // UTC
+            var params = {
+                offset: OFFSET_UTC_IN_SECONDS,
+                aggregation: 'decade'
+            };
+
+            this.testClient = new TestClient(mapConfig, 1234);
+            this.testClient.getDataview('datetime_histogram_automatic_decade', params, function (err, dataview) {
+                assert.ok(!err, err);
+                assert.equal(dataview.type, 'histogram');
+                assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width);
+                assert.equal(dataview.bins.length, 17);
+
+                var initialTimestamp = '1850-01-01T00:00:00Z';
+                var binsStartInMilliseconds = dataview.bins_start * 1000;
+                var binsStartFormatted = moment.utc(binsStartInMilliseconds)
+                    .utcOffset(OFFSET_UTC_IN_MINUTES)
+                    .format();
+                assert.equal(binsStartFormatted, initialTimestamp);
+
+                dataview.bins.forEach(function (bin, index) {
+                    var binTimestampExpected = moment.utc(initialTimestamp)
+                        .utcOffset(OFFSET_UTC_IN_MINUTES)
+                        .add(index * 10, 'year')
+                        .format();
+                    var binsTimestampInMilliseconds = bin.timestamp * 1000;
+                    var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds)
+                        .utcOffset(OFFSET_UTC_IN_MINUTES)
+                        .format();
+
+                    assert.equal(binTimestampFormatted, binTimestampExpected);
+                    assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin));
+                    assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin));
+                });
+
+                done();
+            });
+        });
+
+        it('should aggregate histogram using "century" aggregation ' + test.desc, function (done) {
+            var OFFSET_UTC_IN_SECONDS = 0 * 3600; // UTC
+            var OFFSET_UTC_IN_MINUTES = 0 * 60; // UTC
+            var params = {
+                offset: OFFSET_UTC_IN_SECONDS,
+                aggregation: 'century'
+            };
+
+            this.testClient = new TestClient(mapConfig, 1234);
+            this.testClient.getDataview('datetime_histogram_automatic_century', params, function (err, dataview) {
+                assert.ok(!err, err);
+                assert.equal(dataview.type, 'histogram');
+                assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width);
+                assert.equal(dataview.bins.length, 14);
+
+                var initialTimestamp = '0601-01-01T00:00:00Z';
+                var binsStartInMilliseconds = dataview.bins_start * 1000;
+                var binsStartFormatted = moment.utc(binsStartInMilliseconds)
+                    .utcOffset(OFFSET_UTC_IN_MINUTES)
+                    .format();
+                assert.equal(binsStartFormatted, initialTimestamp);
+
+                dataview.bins.forEach(function (bin, index) {
+                    var binTimestampExpected = moment.utc(initialTimestamp)
+                        .utcOffset(OFFSET_UTC_IN_MINUTES)
+                        .add(index * 100, 'year')
+                        .format();
+                    var binsTimestampInMilliseconds = bin.timestamp * 1000;
+                    var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds)
+                        .utcOffset(OFFSET_UTC_IN_MINUTES)
+                        .format();
+
+                    assert.equal(binTimestampFormatted, binTimestampExpected);
+                    assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin));
+                    assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin));
+                });
+
+                done();
+            });
+        });
+
+        it('should aggregate histogram using "millennium" aggregation ' + test.desc, function (done) {
+            var OFFSET_UTC_IN_SECONDS = 0 * 3600; // UTC
+            var OFFSET_UTC_IN_MINUTES = 0 * 60; // UTC
+            var params = {
+                offset: OFFSET_UTC_IN_SECONDS,
+                aggregation: 'millennium'
+            };
+
+            this.testClient = new TestClient(mapConfig, 1234);
+            this.testClient.getDataview('datetime_histogram_automatic_millennium', params, function (err, dataview) {
+                assert.ok(!err, err);
+                assert.equal(dataview.type, 'histogram');
+                assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width);
+                assert.equal(dataview.bins.length, 12);
+
+                var initialTimestamp = '0001-01-01T00:00:00Z';
+                var binsStartInMilliseconds = dataview.bins_start * 1000;
+                var binsStartFormatted = moment.utc(binsStartInMilliseconds)
+                    .utcOffset(OFFSET_UTC_IN_MINUTES)
+                    .format();
+                assert.equal(binsStartFormatted, initialTimestamp);
+
+                dataview.bins.forEach(function (bin, index) {
+                    var binTimestampExpected = moment.utc(initialTimestamp)
+                        .utcOffset(OFFSET_UTC_IN_MINUTES)
+                        .add(index * 1000, 'year')
+                        .format();
+                    var binsTimestampInMilliseconds = bin.timestamp * 1000;
+                    var binTimestampFormatted = moment.utc(binsTimestampInMilliseconds)
+                        .utcOffset(OFFSET_UTC_IN_MINUTES)
+                        .format();
+
+                    assert.equal(binTimestampFormatted, binTimestampExpected);
+                    assert.ok(bin.timestamp <= bin.min, 'bin timestamp < bin min: ' + JSON.stringify(bin));
+                    assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin));
+                });
+
+                done();
+            });
+        });
+
+
+
+
         it('bins_count should be equal to bins length filtered by start and end ' + test.desc, function (done) {
             var OFFSET_UTC_IN_SECONDS = 0 * 3600; // UTC
             var params = {
@@ -518,19 +911,149 @@ describe('histogram-dataview for date column type', function() {
         });
     });
 
-    it('should find the best aggregation (automatic mode) to build the histogram', function (done) {
+    it('should return same histogram ', function (done) {
+        var params = {
+            start: 1171501200, // 2007-02-15 01:00:00 = min(date_colum)
+            end: 1207702800 // 2008-04-09 01:00:00 = max(date_colum)
+        };
+
+        this.testClient = new TestClient(mapConfig, 1234);
+        this.testClient.getDataview('datetime_histogram_tz', {}, function (err, dataview) {
+            assert.ok(!err, err);
+
+            this.testClient = new TestClient(mapConfig, 1234);
+            this.testClient.getDataview('datetime_histogram_tz', params, function (err, filteredDataview) {
+                assert.ok(!err, err);
+
+                assert.deepEqual(dataview, filteredDataview);
+                done();
+            });
+        });
+    });
+
+    it('should find the best aggregation (automatic mode) to build the histogram: second', function (done) {
         var params = {};
         this.testClient = new TestClient(mapConfig, 1234);
-        this.testClient.getDataview('datetime_histogram_automatic', params, function (err, dataview) {
+        this.testClient.getDataview('datetime_histogram_automatic_second', params, function (err, dataview) {
             assert.ifError(err);
             assert.equal(dataview.type, 'histogram');
-            assert.equal(dataview.aggregation, 'week');
-            assert.equal(dataview.bins.length, 61);
-            assert.equal(dataview.bins_count, 61);
+            assert.equal(dataview.aggregation, 'second');
             done();
         });
     });
 
+    it('should find the best aggregation (automatic mode) to build the histogram: minute', function (done) {
+        var params = {};
+        this.testClient = new TestClient(mapConfig, 1234);
+        this.testClient.getDataview('datetime_histogram_automatic_minute', params, function (err, dataview) {
+            assert.ifError(err);
+            assert.equal(dataview.type, 'histogram');
+            assert.equal(dataview.aggregation, 'minute');
+            done();
+        });
+    });
+
+    it('should find the best aggregation (automatic mode) to build the histogram: hour', function (done) {
+        var params = {};
+        this.testClient = new TestClient(mapConfig, 1234);
+        this.testClient.getDataview('datetime_histogram_automatic_hour', params, function (err, dataview) {
+            assert.ifError(err);
+            assert.equal(dataview.type, 'histogram');
+            assert.equal(dataview.aggregation, 'hour');
+            done();
+        });
+    });
+
+    it('should find the best aggregation (automatic mode) to build the histogram: day', function (done) {
+        var params = {};
+        this.testClient = new TestClient(mapConfig, 1234);
+        this.testClient.getDataview('datetime_histogram_automatic_day', params, function (err, dataview) {
+            assert.ifError(err);
+            assert.equal(dataview.type, 'histogram');
+            assert.equal(dataview.aggregation, 'day');
+            done();
+        });
+    });
+
+    it('should find the best aggregation (automatic mode) to build the histogram: week', function (done) {
+        var params = {};
+        this.testClient = new TestClient(mapConfig, 1234);
+        this.testClient.getDataview('datetime_histogram_automatic_week', params, function (err, dataview) {
+            assert.ifError(err);
+            assert.equal(dataview.type, 'histogram');
+            assert.equal(dataview.aggregation, 'week');
+            done();
+        });
+    });
+
+    it('should find the best aggregation (automatic mode) to build the histogram: month', function (done) {
+        var params = {};
+        this.testClient = new TestClient(mapConfig, 1234);
+        this.testClient.getDataview('datetime_histogram_automatic_month', params, function (err, dataview) {
+            assert.ifError(err);
+            assert.equal(dataview.type, 'histogram');
+            assert.equal(dataview.aggregation, 'month');
+            done();
+        });
+    });
+
+    it('should find the best aggregation (automatic mode) to build the histogram: quarter', function (done) {
+        var params = {};
+        this.testClient = new TestClient(mapConfig, 1234);
+        this.testClient.getDataview('datetime_histogram_automatic_quarter', params, function (err, dataview) {
+            assert.ifError(err);
+            assert.equal(dataview.type, 'histogram');
+            assert.equal(dataview.aggregation, 'quarter');
+            done();
+        });
+    });
+
+    it('should find the best aggregation (automatic mode) to build the histogram: year', function (done) {
+        var params = {};
+        this.testClient = new TestClient(mapConfig, 1234);
+        this.testClient.getDataview('datetime_histogram_automatic_year', params, function (err, dataview) {
+            assert.ifError(err);
+            assert.equal(dataview.type, 'histogram');
+            assert.equal(dataview.aggregation, 'year');
+            done();
+        });
+    });
+
+    it('should find the best aggregation (automatic mode) to build the histogram: decade', function (done) {
+        var params = {};
+        this.testClient = new TestClient(mapConfig, 1234);
+        this.testClient.getDataview('datetime_histogram_automatic_decade', params, function (err, dataview) {
+            assert.ifError(err);
+            assert.equal(dataview.type, 'histogram');
+            assert.equal(dataview.aggregation, 'decade');
+            done();
+        });
+    });
+
+    it('should find the best aggregation (automatic mode) to build the histogram: century', function (done) {
+        var params = {};
+        this.testClient = new TestClient(mapConfig, 1234);
+        this.testClient.getDataview('datetime_histogram_automatic_century', params, function (err, dataview) {
+            assert.ifError(err);
+            assert.equal(dataview.type, 'histogram');
+            assert.equal(dataview.aggregation, 'century');
+            done();
+        });
+    });
+
+    it('should find the best aggregation (automatic mode) to build the histogram: millennium', function (done) {
+        var params = {};
+        this.testClient = new TestClient(mapConfig, 1234);
+        this.testClient.getDataview('datetime_histogram_automatic_millennium', params, function (err, dataview) {
+            assert.ifError(err);
+            assert.equal(dataview.type, 'histogram');
+            assert.equal(dataview.aggregation, 'millennium');
+            done();
+        });
+    });
+
+
+
     it('should work with dates', function (done) {
         var params = {};
         this.testClient = new TestClient(mapConfig, 1234);
@@ -545,19 +1068,6 @@ describe('histogram-dataview for date column type', function() {
     });
 
 
-    it('should find the best aggregation (automatic mode) to build the histogram with dates', function (done) {
-        var params = {};
-        this.testClient = new TestClient(mapConfig, 1234);
-        this.testClient.getDataview('date_histogram_automatic', params, function (err, dataview) {
-            assert.ifError(err);
-            assert.equal(dataview.type, 'histogram');
-            assert.equal(dataview.aggregation, 'week');
-            assert.equal(dataview.bins.length, 61);
-            assert.equal(dataview.bins_count, 61);
-            done();
-        });
-    });
-
     it('should not apply offset for a histogram aggregated by minutes', function (done) {
         var self = this;
         var params = {
@@ -625,7 +1135,7 @@ describe('histogram-dataview for date column type', function() {
 
         var dataviewWithDailyAggFixture = {
             aggregation: 'day',
-            bin_width: 600,
+            bin_width: 86400,
             bins_count: 2,
             bins_start: 1171497600,
             timestamp_start: 1171497600,
@@ -635,17 +1145,17 @@ describe('histogram-dataview for date column type', function() {
             [{
                 bin: 0,
                 timestamp: 1171497600,
-                min: 1171583400,
-                max: 1171583940,
-                avg: 1171583670,
+                min: 1171497600,
+                max: 1171497600,
+                avg: 1171497600,
                 freq: 10
             },
             {
                 bin: 1,
                 timestamp: 1171584000,
                 min: 1171584000,
-                max: 1171584600,
-                avg: 1171584300,
+                max: 1171584000,
+                avg: 1171584000,
                 freq: 11
             }],
             type: 'histogram'
@@ -672,19 +1182,19 @@ describe('histogram-dataview for date column type', function() {
 
         var dataviewWithDailyAggAndOffsetFixture = {
             aggregation: 'day',
-            bin_width: 1200,
+            bin_width: 86400,
             bins_count: 1,
             bins_start: 1171501200,
-            timestamp_start: 1171497600,
+            timestamp_start: 1171501200,
             nulls: 0,
             offset: -3600,
             bins:
             [{
                 bin: 0,
                 timestamp: 1171501200,
-                min: 1171583400,
-                max: 1171584600,
-                avg: 1171584000,
+                min: 1171501200,
+                max: 1171501200,
+                avg: 1171501200,
                 freq: 21
             }],
             type: 'histogram'
@@ -843,13 +1353,15 @@ describe('histogram-dates: aggregation input value', function() {
 
             assert.deepEqual(dataviewError, {
                 errors: [
-                    'Invalid aggregation value. Valid ones: auto, minute, hour, day, week, month, quarter, year'
+                    'Invalid aggregation value. Valid ones: auto, second, minute, ' +
+                    'hour, day, week, month, quarter, year, decade, century, millennium'
                 ],
                 errors_with_context: [{
                     type: 'unknown',
                     message: [
                         'Invalid aggregation value. ',
-                        'Valid ones: auto, minute, hour, day, week, month, quarter, year'
+                        'Valid ones: auto, second, minute, hour, day, week, month, ' +
+                            'quarter, year, decade, century, millennium'
                     ].join('')
                 }]
             });
@@ -872,13 +1384,15 @@ describe('histogram-dates: aggregation input value', function() {
 
             assert.deepEqual(dataviewError, {
                 errors: [
-                    'Invalid aggregation value. Valid ones: auto, minute, hour, day, week, month, quarter, year'
+                    'Invalid aggregation value. Valid ones: auto, second, minute, ' +
+                    'hour, day, week, month, quarter, year, decade, century, millennium'
                 ],
                 errors_with_context: [{
                     type: 'unknown',
                     message: [
                         'Invalid aggregation value. ',
-                        'Valid ones: auto, minute, hour, day, week, month, quarter, year'
+                        'Valid ones: auto, second, minute, hour, day, week, month, ' +
+                            'quarter, year, decade, century, millennium'
                     ].join('')
                 }]
             });
@@ -951,7 +1465,7 @@ describe('histogram-dates: timestamp starts at epoch', function() {
             const { aggregation, timestamp_start } = dataview;
 
             assert.equal(timestamp_start, 0);
-            assert.equal(aggregation, 'month');
+            assert.equal(aggregation, 'quarter');
 
             done();
         });

test/acceptance/dataviews/overviews.js

@@ -393,7 +393,8 @@ describe('dataviews using tables with overviews', function() {
                 var params = {
                     filters: {
                         dataviews: { test_histogram: { min: 2 } }
-                    }
+                    },
+                    own_filter: 1
                 };
                 var testClient = new TestClient(overviewsMapConfig);
                 testClient.getDataview('test_histogram', params, function (err, histogram) {
@@ -412,7 +413,8 @@ describe('dataviews using tables with overviews', function() {
                 var params = {
                     filters: {
                         dataviews: { test_histogram: { max: -1 } }
-                    }
+                    },
+                    own_filter: 1
                 };
                 var testClient = new TestClient(overviewsMapConfig);
                 testClient.getDataview('test_histogram', params, function (err, histogram) {
@@ -433,7 +435,8 @@ describe('dataviews using tables with overviews', function() {
                 var params = {
                     filters: {
                         dataviews: { test_histogram_date: { max: -1 } }
-                    }
+                    },
+                    own_filter: 1
                 };
                 var testClient = new TestClient(overviewsMapConfig);
                 testClient.getDataview('test_histogram_date', params, function (err, histogram) {

test/acceptance/dynamic-styling-named-maps.js

@@ -1,6 +1,6 @@
 var assert = require('../support/assert');
 var step = require('step');
-var LayergroupToken = require('../support/layergroup-token');
+var LayergroupToken = require('../../lib/cartodb/models/layergroup-token');
 var testHelper = require(__dirname + '/../support/test_helper');
 var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server');
 var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');

test/acceptance/error-middleware.js

@@ -0,0 +1,42 @@
+const assert = require('../support/assert');
+const TestClient = require('../support/test-client');
+
+describe('error middleware', function () {
+    it('should returns a errors header', function (done) {
+        const mapConfig = {
+            version: '1.6.0',
+            layers: [{
+                type: 'mapnik',
+                options: {}
+            }]
+        };
+
+        const errorHeader = {
+            mainError: {
+                statusCode: 400,
+                message: "Missing cartocss for layer 0 options",
+                name: "Error",
+                label: "ANONYMOUS LAYERGROUP",
+                type: "layer",
+            },
+            moreErrors: []
+        };
+
+        this.testClient = new TestClient(mapConfig, 1234);
+
+        const params = {
+            response: {
+                status: 400,
+                headers: {
+                    'Content-Type': 'application/json; charset=utf-8',
+                    'X-Tiler-Errors': JSON.stringify(errorHeader)
+                }
+            }
+        };
+
+        this.testClient.getLayergroup(params, (err) => {
+            assert.ifError(err);
+            done();
+        });
+    });
+});

test/acceptance/geojson-renderer.js

@@ -1,343 +0,0 @@
-require('../support/test_helper');
-
-var assert = require('../support/assert');
-var TestClient = require('../support/test-client');
-
-
-describe('use only needed columns', function() {
-
-    function getFeatureByCartodbId(features, cartodbId) {
-        for (var i = 0, len = features.length; i < len; i++) {
-            if (features[i].properties.cartodb_id === cartodbId) {
-                return features[i];
-            }
-        }
-        return {};
-    }
-
-    var options = { format: 'geojson', layer: 0 };
-
-    afterEach(function(done) {
-        if (this.testClient) {
-            this.testClient.drain(done);
-        } else {
-            done();
-        }
-    });
-
-    it('with aggregation widget, interactivity and cartocss columns', function(done) {
-        var widgetMapConfig = {
-            version: '1.5.0',
-            layers: [{
-                type: 'mapnik',
-                options: {
-                    sql: 'select * from populated_places_simple_reduced',
-                    cartocss: '#layer0 { marker-fill: red; marker-width: 10; [name="Madrid"] { marker-fill: green; } }',
-                    cartocss_version: '2.0.1',
-                    widgets: {
-                        adm0name: {
-                            type: 'aggregation',
-                            options: {
-                                column: 'adm0name',
-                                aggregation: 'sum',
-                                aggregationColumn: 'pop_max'
-                            }
-                        }
-                    },
-                    interactivity: "cartodb_id,pop_min"
-                }
-            }]
-        };
-
-        this.testClient = new TestClient(widgetMapConfig);
-        this.testClient.getTile(0, 0, 0, options, function (err, res, geojsonTile) {
-            assert.ok(!err, err);
-            assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, {
-                cartodb_id: 1109,
-                name: 'Mardin',
-                adm0name: 'Turkey',
-                pop_max: 71373,
-                pop_min: 57586
-            });
-            done();
-        });
-    });
-
-    it('should not duplicate columns', function(done) {
-        var widgetMapConfig = {
-            version: '1.5.0',
-            layers: [{
-                type: 'mapnik',
-                options: {
-                    sql: 'select * from populated_places_simple_reduced',
-                    cartocss: ['#layer0 {',
-                        'marker-fill: red;',
-                        'marker-width: 10;',
-                        '[name="Madrid"] { marker-fill: green; } ',
-                        '[pop_max>100000] { marker-fill: black; } ',
-                        '}'].join('\n'),
-                    cartocss_version: '2.3.0',
-                    widgets: {
-                        adm0name: {
-                            type: 'aggregation',
-                            options: {
-                                column: 'adm0name',
-                                aggregation: 'sum',
-                                aggregationColumn: 'pop_max'
-                            }
-                        }
-                    },
-                    interactivity: "cartodb_id,pop_max"
-                }
-            }]
-        };
-
-        this.testClient = new TestClient(widgetMapConfig);
-        this.testClient.getTile(0, 0, 0, options, function (err, res, geojsonTile) {
-            assert.ok(!err, err);
-            assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, {
-                cartodb_id: 1109,
-                name: 'Mardin',
-                adm0name: 'Turkey',
-                pop_max: 71373
-            });
-            done();
-        });
-    });
-
-    it('with formula widget, no interactivity and no cartocss columns', function(done) {
-        var formulaWidgetMapConfig = {
-            version: '1.5.0',
-            layers: [{
-                type: 'mapnik',
-                options: {
-                    sql: 'select * from populated_places_simple_reduced where pop_max > 0 and pop_max < 600000',
-                    cartocss: '#layer0 { marker-fill: red; marker-width: 10; }',
-                    cartocss_version: '2.0.1',
-                    interactivity: 'cartodb_id',
-                    widgets: {
-                        pop_max_f: {
-                            type: 'formula',
-                            options: {
-                                column: 'pop_max',
-                                operation: 'count'
-                            }
-                        }
-                    }
-                }
-            }]
-        };
-
-        this.testClient = new TestClient(formulaWidgetMapConfig);
-        this.testClient.getTile(0, 0, 0, options, function (err, res, geojsonTile) {
-            assert.ok(!err, err);
-            assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, {
-                cartodb_id: 1109,
-                pop_max: 71373
-            });
-            done();
-        });
-    });
-    it('with cartocss with multiple expressions', function(done) {
-        var formulaWidgetMapConfig = {
-            version: '1.5.0',
-            layers: [{
-                type: 'mapnik',
-                options: {
-                    sql: 'select * from populated_places_simple_reduced where pop_max > 0 and pop_max < 600000',
-                    cartocss: '#layer0 { marker-fill: red; marker-width: 10; }' +
-                        '#layer0 { marker-width: 14; [name="Madrid"] { marker-width: 20; } }' +
-                        '#layer0[pop_max>1000] { marker-width: 14; [name="Madrid"] { marker-width: 20; } }' +
-                        '#layer0[adm0name=~".*Turkey*"] { marker-width: 14; [name="Madrid"] { marker-width: 20; } }',
-                    cartocss_version: '2.0.1',
-                    interactivity: 'cartodb_id'
-                }
-            }]
-        };
-
-        this.testClient = new TestClient(formulaWidgetMapConfig);
-        this.testClient.getTile(0, 0, 0, options, function (err, res, geojsonTile) {
-            assert.ok(!err, err);
-            assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, {
-                cartodb_id: 1109,
-                pop_max:71373,
-                name:"Mardin",
-                adm0name:"Turkey"
-            });
-            done();
-        });
-    });
-
-    it('should work with mapnik substitution tokens', function(done) {
-        var cartocss = [
-            "#layer {",
-            "  line-width: 2;",
-            "  line-color: #3B3B58;",
-            "  line-opacity: 1;",
-            "  polygon-opacity: 0.7;",
-            "  polygon-fill: ramp([points_count], (#E5F5F9,#99D8C9,#2CA25F))",
-            "}"
-        ].join('\n');
-
-        var sql = [
-            'WITH hgrid AS (',
-            '  SELECT CDB_HexagonGrid(',
-            '    ST_Expand(!bbox!, greatest(!pixel_width!,!pixel_height!) * 100),',
-            '    greatest(!pixel_width!,!pixel_height!) * 100',
-            '  ) as cell',
-            ')',
-            'SELECT',
-            '  hgrid.cell as the_geom_webmercator,',
-            '  count(1) as points_count,',
-            '  count(1)/power(100 * CDB_XYZ_Resolution(CDB_ZoomFromScale(!scale_denominator!)), 2) as points_density,',
-            '  1 as cartodb_id',
-            'FROM hgrid, (SELECT * FROM populated_places_simple_reduced) i',
-            'where ST_Intersects(i.the_geom_webmercator, hgrid.cell)',
-            'GROUP BY hgrid.cell'
-        ].join('\n');
-
-        var mapConfig = {
-            "version": "1.4.0",
-            "layers": [
-                {
-                    "type": 'mapnik',
-                    "options": {
-                        "cartocss_version": '2.3.0',
-                        "sql": sql,
-                        "cartocss": cartocss
-                    }
-                }
-            ]
-        };
-
-        this.testClient = new TestClient(mapConfig);
-        this.testClient.getTile(0, 0, 0, { format: 'geojson', layer: 0 }, function(err, res, geojson) {
-            assert.ok(!err, err);
-
-            assert.ok(geojson);
-            assert.equal(geojson.features.length, 5);
-
-            done();
-        });
-    });
-
-    it('should skip empty and null columns for geojson tiles', function(done) {
-
-        var mapConfig = {
-            "analyses": [
-                {
-                    "id": "a0",
-                    "params": {
-                        "query": "SELECT * FROM test_table"
-                    },
-                    "type": "source"
-                }
-            ],
-            "dataviews": {
-                "4e7b0e07-6d21-4b83-9adb-6d7e17eea6ca": {
-                    "options": {
-                        "aggregationColumn": null,
-                        "column": "cartodb_id",
-                        "operation": "avg"
-                    },
-                    "source": {
-                        "id": "a0"
-                    },
-                    "type": "formula"
-                },
-                "74f590f8-625c-4e95-922f-34ad3e9919c0": {
-                    "options": {
-                        "aggregation": "sum",
-                        "aggregationColumn": "cartodb_id",
-                        "column": "name"
-                    },
-                    "source": {
-                        "id": "a0"
-                    },
-                    "type": "aggregation"
-                },
-                "98a75757-3006-400a-b028-fb613a6c0b69": {
-                    "options": {
-                        "aggregationColumn": null,
-                        "column": "cartodb_id",
-                        "operation": "sum"
-                    },
-                    "source": {
-                        "id": "a0"
-                    },
-                    "type": "formula"
-                },
-                "ebbc97b2-87d2-4895-9e1f-2f012df3679d": {
-                    "options": {
-                        "aggregationColumn": null,
-                        "bins": "12",
-                        "column": "cartodb_id"
-                    },
-                    "source": {
-                        "id": "a0"
-                    },
-                    "type": "histogram"
-                },
-                "ebc0653f-3581-469c-8b31-c969e440a865": {
-                    "options": {
-                        "aggregationColumn": null,
-                        "column": "cartodb_id",
-                        "operation": "avg"
-                    },
-                    "source": {
-                        "id": "a0"
-                    },
-                    "type": "formula"
-                }
-            },
-            "layers": [
-                {
-                    "options": {
-                        "subdomains": "abcd",
-                        "urlTemplate": "http://{s}.basemaps.cartocdn.com/light_nolabels/{z}/{x}/{y}.png"
-                    },
-                    "type": "http"
-                },
-                {
-                    "options": {
-                        "attributes": {
-                            "columns": [
-                                "name",
-                                "address"
-                            ],
-                            "id": "cartodb_id"
-                        },
-                        "cartocss": "#layer { marker-width: 10; marker-fill: red; }",
-                        "cartocss_version": "2.3.0",
-                        "interactivity": "cartodb_id",
-                        "layer_name": "wadus",
-                        "source": {
-                            "id": "a0"
-                        }
-                    },
-                    "type": "cartodb"
-                },
-                {
-                    "options": {
-                        "subdomains": "abcd",
-                        "urlTemplate": "http://{s}.basemaps.cartocdn.com/light_only_labels/{z}/{x}/{y}.png"
-                    },
-                    "type": "http"
-                }
-            ]
-        };
-
-        this.testClient = new TestClient(mapConfig);
-        this.testClient.getTile(0, 0, 0, { format: 'geojson', layer: 0 }, function(err, res, geojson) {
-            assert.ok(!err, err);
-
-            assert.ok(geojson);
-            assert.equal(geojson.features.length, 5);
-
-            assert.deepEqual(Object.keys(geojson.features[0].properties), ['cartodb_id', 'name']);
-
-            done();
-        });
-    });
-
-});

test/acceptance/label-wrap.js

@@ -0,0 +1,69 @@
+require('../support/test_helper');
+var TestClient = require('../support/test-client');
+
+var assert = require('../support/assert');
+var IMAGE_TOLERANCE = 5;
+
+describe('CartoCSS wrap', function () {
+    const options = {
+        sql: `
+            SELECT 
+                5 as cartodb_id, 
+                ST_Transform(ST_SetSRID(ST_MakePoint(-57.65625,-15.6230368),4326),3857) as the_geom_webmercator,  
+                ST_SetSRID(ST_MakePoint(-57.65625,-15.62303683),4326) as the_geom, 
+                'South America' as continent
+        `,
+        cartocss: `
+            #continent_points::labels {
+                text-name: [continent];
+                text-face-name: 'DejaVu Sans Book';
+                text-size: 10;
+                text-fill: lighten(#000,40);
+                text-transform: uppercase;
+                text-wrap-width: 30;
+                text-character-spacing: 2;
+                text-placement: point;
+                text-placement-type: dummy;
+                [zoom >= 3]{
+                    text-character-spacing: 2;
+                    text-size: 11;
+                }
+            }
+        `,
+        cartocss_version: '3.0.12'
+    };
+
+    const type = 'mapnik';
+
+    const mapConfig = {
+        version: '1.6.0',
+        layers: [
+            {
+                type,
+                id: 'layerLabel',
+                options
+            }
+        ]
+    };
+
+
+    var keysToDelete;
+
+    beforeEach(function () {
+        keysToDelete = {};
+    });
+
+    afterEach(function (done) {
+        if (this.testClient) {
+            this.testClient.drain(done);
+        }
+    });
+
+    it("Label should be text-wrapped", function (done) {
+        this.testClient = new TestClient(mapConfig);
+        this.testClient.getTile(1, 0, 1, { layers: [0] }, (err, res, body) => {
+            var textWrapPath = './test/fixtures/text_wrap.png';
+            assert.imageIsSimilarToFile(body, textWrapPath, IMAGE_TOLERANCE, done);
+        });
+    });
+});

test/acceptance/layers-filters.js

@@ -0,0 +1,54 @@
+require('../support/test_helper');
+var TestClient = require('../support/test-client');
+
+describe('layers filters', function() {
+    const type = 'mapnik';
+    const sql = 'select * from populated_places_simple_reduced';
+    const cartocss = `#points {
+        marker-fill-opacity: 1.0;
+        marker-line-color: #FFF;
+        marker-line-width: 0.5;
+        marker-line-opacity: 1.0;
+        marker-placement: point;
+        marker-type: ellipse;
+        marker-width: 8;
+        marker-fill: red;
+        marker-allow-overlap: true;
+    }`;
+    const cartocss_version = '3.0.12';
+    const options = {
+        sql,
+        cartocss,
+        cartocss_version
+    };
+
+    const mapConfig = {
+        version: '1.6.0',
+        layers: [
+            {
+                type,
+                id: 'layerA',
+                options
+            },
+            {
+                type,
+                id: 'layerB',
+                options
+            }
+        ]
+    };
+
+    afterEach(function(done) {
+        if (this.testClient) {
+            this.testClient.drain(done);
+        }
+    });
+
+    ['layerA', 'layerB'].forEach(layer => {
+        it(`should work for individual layer ids: ${layer}`, function (done) {
+            this.testClient = new TestClient(mapConfig);
+            this.testClient.getTile(0, 0, 0, { layers: layer }, done);
+        });
+    });
+
+});

test/acceptance/multilayer.js

@@ -9,7 +9,7 @@ var mapnik = require('windshaft').mapnik;
 var semver = require('semver');
 
 var helper = require(__dirname + '/../support/test_helper');
-var LayergroupToken = require('../support/layergroup-token');
+var LayergroupToken = require('../../lib/cartodb/models/layergroup-token');
 
 var windshaft_fixtures = __dirname + '/../../node_modules/windshaft/test/fixtures';
 
@@ -1163,8 +1163,12 @@ describe(suiteName, function() {
       );
     });
 
+    // WARN: MapConfig with mapnik layer and no cartocss it's valid since
+    // vector & raster aggregation project, now we can request MVT format w/o defining styles
+    // for the layer.
+
     // See https://github.com/CartoDB/Windshaft-cartodb/issues/133
-    it("MapConfig with mapnik layer and no cartocss", function(done) {
+    it.skip("MapConfig with mapnik layer and no cartocss", function(done) {
 
       var layergroup =  {
         version: '1.0.0',

test/acceptance/multilayer_server.js

@@ -4,7 +4,7 @@ var assert = require('../support/assert');
 
 var _ = require('underscore');
 
-var LayergroupToken = require('../support/layergroup-token');
+var LayergroupToken = require('../../lib/cartodb/models/layergroup-token');
 
 var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner');
 var QueryTables = require('cartodb-query-tables');

test/acceptance/mvt.js

@@ -1,9 +1,10 @@
 require('../support/test_helper');
 
-const assert = require('../support/assert');
-const TestClient = require('../support/test-client');
+var assert = require('../support/assert');
+var TestClient = require('../support/test-client');
+var serverOptions = require('../../lib/cartodb/server_options');
 
-function createMapConfig (sql = TestClient.SQL.ONE_POINT) {
+function createMapConfig(sql = TestClient.SQL.ONE_POINT) {
     return {
         version: '1.6.0',
         layers: [{
@@ -18,7 +19,100 @@ function createMapConfig (sql = TestClient.SQL.ONE_POINT) {
     };
 }
 
-describe('mvt', function () {
+describe('mvt (mapnik)', mvt(false));
+if (process.env.POSTGIS_VERSION === '2.4') {
+    describe('mvt (postgis)', mvt(true));
+}
+
+function mvt(usePostGIS) {
+return function () {
+    const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS;
+    before(function () {
+        serverOptions.renderer.mvt.usePostGIS = usePostGIS;
+    });
+    after(function (){
+        serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS;
+    });
+
+    describe('analysis-layers-dataviews-mvt', function () {
+
+        function createMapConfig(layers, dataviews, analysis) {
+            return {
+                version: '1.5.0',
+                layers: layers,
+                dataviews: dataviews || {},
+                analyses: analysis || []
+            };
+        }
+
+        var CARTOCSS = [
+            "#points {",
+            "  marker-fill-opacity: 1.0;",
+            "  marker-line-color: #FFF;",
+            "  marker-line-width: 0.5;",
+            "  marker-line-opacity: 1.0;",
+            "  marker-placement: point;",
+            "  marker-type: ellipse;",
+            "  marker-width: 8;",
+            "  marker-fill: red;",
+            "  marker-allow-overlap: true;",
+            "}"
+        ].join('\n');
+
+        var mapConfig = createMapConfig(
+            [
+                {
+                    "type": "cartodb",
+                    "options": {
+                        "source": {
+                            "id": "2570e105-7b37-40d2-bdf4-1af889598745"
+                        },
+                        "cartocss": CARTOCSS,
+                        "cartocss_version": "2.3.0"
+                    }
+                }
+            ],
+            {
+                pop_max_histogram: {
+                    source: {
+                        id: '2570e105-7b37-40d2-bdf4-1af889598745'
+                    },
+                    type: 'histogram',
+                    options: {
+                        column: 'pop_max'
+                    }
+                }
+            },
+            [
+                {
+                    "id": "2570e105-7b37-40d2-bdf4-1af889598745",
+                    "type": "source",
+                    "params": {
+                        "query": "select * from populated_places_simple_reduced"
+                    }
+                }
+            ]
+        );
+
+        it('should get pop_max column from dataview', function (done) {
+            var testClient = new TestClient(mapConfig);
+
+            testClient.getTile(0, 0, 0, { format: 'mvt', layers: 0 }, function (err, res, MVT) {
+                var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0));
+                assert.ok(!err, err);
+
+                assert.ok(Array.isArray(geojsonTile.features));
+                assert.ok(geojsonTile.features.length > 0);
+                var feature = geojsonTile.features[0];
+                assert.ok(feature.properties.hasOwnProperty('pop_max'), 'Missing pop_max property');
+
+                testClient.drain(done);
+            });
+        });
+
+    });
+
+
     const testCases = [
         {
             desc: 'should get empty mvt with code 204 (no content)',
@@ -48,16 +142,366 @@ describe('mvt', function () {
 
     testCases.forEach(function (test) {
         it(test.desc, done => {
-            const testClient = new TestClient(test.mapConfig, 1234);
+            var testClient = new TestClient(test.mapConfig);
             const { z, x, y } = test.coords;
             const { format, response } = test;
 
-            testClient.getTile(z, x, y, { format, response }, (err, res) => {
+            testClient.getTile(z, x, y, { format, response }, err => {
                 assert.ifError(err);
-
-                assert.equal(res.statusCode, test.response.status);
                 testClient.drain(done);
             });
         });
     });
-});
+
+
+    if (usePostGIS){
+        describe('use only needed columns', onlyNeededColumns);
+    }else{
+        describe.skip('use only needed columns', onlyNeededColumns);
+    }
+
+    function onlyNeededColumns() {
+        function getFeatureByCartodbId(features, cartodbId) {
+            for (var i = 0, len = features.length; i < len; i++) {
+                if (features[i].properties.cartodb_id === cartodbId) {
+                    return features[i];
+                }
+            }
+            return {};
+        }
+
+        var options = { format: 'mvt', layer: 0 };
+
+        afterEach(function (done) {
+            if (this.testClient) {
+                this.testClient.drain(done);
+            } else {
+                done();
+            }
+        });
+
+        it('with aggregation widget, interactivity and cartocss columns', function (done) {
+            var widgetMapConfig = {
+                version: '1.5.0',
+                layers: [{
+                    type: 'mapnik',
+                    options: {
+                        sql: 'select * from populated_places_simple_reduced',
+                        cartocss:
+                            '#layer0 { marker-fill: red; marker-width: 10; [name="Madrid"] { marker-fill: green; } }',
+                        cartocss_version: '2.0.1',
+                        widgets: {
+                            adm0name: {
+                                type: 'aggregation',
+                                options: {
+                                    column: 'adm0name',
+                                    aggregation: 'sum',
+                                    aggregationColumn: 'pop_max'
+                                }
+                            }
+                        },
+                        interactivity: "cartodb_id,pop_min"
+                    }
+                }]
+            };
+
+            this.testClient = new TestClient(widgetMapConfig);
+            this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) {
+                var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0));
+                assert.ok(!err, err);
+                assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, {
+                    cartodb_id: 1109,
+                    name: 'Mardin',
+                    adm0name: 'Turkey',
+                    pop_max: 71373,
+                    pop_min: 57586
+                });
+                done();
+            });
+        });
+
+        it('should not duplicate columns', function (done) {
+            var widgetMapConfig = {
+                version: '1.5.0',
+                layers: [{
+                    type: 'mapnik',
+                    options: {
+                        sql: 'select * from populated_places_simple_reduced',
+                        cartocss: ['#layer0 {',
+                            'marker-fill: red;',
+                            'marker-width: 10;',
+                            '[name="Madrid"] { marker-fill: green; } ',
+                            '[pop_max>100000] { marker-fill: black; } ',
+                            '}'].join('\n'),
+                        cartocss_version: '2.3.0',
+                        widgets: {
+                            adm0name: {
+                                type: 'aggregation',
+                                options: {
+                                    column: 'adm0name',
+                                    aggregation: 'sum',
+                                    aggregationColumn: 'pop_max'
+                                }
+                            }
+                        },
+                        interactivity: "cartodb_id,pop_max"
+                    }
+                }]
+            };
+
+            this.testClient = new TestClient(widgetMapConfig);
+            this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) {
+                var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0));
+                assert.ok(!err, err);
+                assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, {
+                    cartodb_id: 1109,
+                    name: 'Mardin',
+                    adm0name: 'Turkey',
+                    pop_max: 71373
+                });
+                done();
+            });
+        });
+
+        it('with formula widget, no interactivity and no cartocss columns', function (done) {
+            var formulaWidgetMapConfig = {
+                version: '1.5.0',
+                layers: [{
+                    type: 'mapnik',
+                    options: {
+                        sql: 'select * from populated_places_simple_reduced where pop_max > 0 and pop_max < 600000',
+                        cartocss: '#layer0 { marker-fill: red; marker-width: 10; }',
+                        cartocss_version: '2.0.1',
+                        interactivity: 'cartodb_id',
+                        widgets: {
+                            pop_max_f: {
+                                type: 'formula',
+                                options: {
+                                    column: 'pop_max',
+                                    operation: 'count'
+                                }
+                            }
+                        }
+                    }
+                }]
+            };
+            this.testClient = new TestClient(formulaWidgetMapConfig);
+            this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) {
+                var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0));
+                assert.ok(!err, err);
+                assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, {
+                    cartodb_id: 1109,
+                    pop_max: 71373
+                });
+                done();
+            });
+        });
+        it('with cartocss with multiple expressions', function (done) {
+            var formulaWidgetMapConfig = {
+                version: '1.5.0',
+                layers: [{
+                    type: 'mapnik',
+                    options: {
+                        sql: 'select * from populated_places_simple_reduced where pop_max > 0 and pop_max < 600000',
+                        cartocss: '#layer0 { marker-fill: red; marker-width: 10; }' +
+                        '#layer0 { marker-width: 14; [name="Madrid"] { marker-width: 20; } }' +
+                        '#layer0[pop_max>1000] { marker-width: 14; [name="Madrid"] { marker-width: 20; } }' +
+                        '#layer0[adm0name=~".*Turkey*"] { marker-width: 14; [name="Madrid"] { marker-width: 20; } }',
+                        cartocss_version: '2.0.1',
+                        interactivity: 'cartodb_id'
+                    }
+                }]
+            };
+
+            this.testClient = new TestClient(formulaWidgetMapConfig);
+            this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) {
+                var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0));
+                assert.ok(!err, err);
+                assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, {
+                    cartodb_id: 1109,
+                    pop_max: 71373,
+                    name: "Mardin",
+                    adm0name: "Turkey"
+                });
+                done();
+            });
+        });
+
+        var skipOnPostGIS = usePostGIS ? it.skip: it;
+        skipOnPostGIS('should work with mapnik substitution tokens', function (done) {
+            var cartocss = [
+                "#layer {",
+                "  line-width: 2;",
+                "  line-color: #3B3B58;",
+                "  line-opacity: 1;",
+                "  polygon-opacity: 0.7;",
+                "  polygon-fill: ramp([points_count], (#E5F5F9,#99D8C9,#2CA25F))",
+                "}"
+            ].join('\n');
+
+            var sql = [
+            'WITH hgrid AS (',
+            '  SELECT CDB_HexagonGrid(',
+            '    ST_Expand(!bbox!, greatest(!pixel_width!,!pixel_height!) * 100),',
+            '    greatest(!pixel_width!,!pixel_height!) * 100',
+            '  ) as cell',
+            ')',
+            'SELECT',
+            '  hgrid.cell as the_geom_webmercator,',
+            '  count(1) as points_count,',
+            '  count(1)/power(100 * CDB_XYZ_Resolution(CDB_ZoomFromScale(!scale_denominator!)), 2) as points_density,',
+            '  1 as cartodb_id',
+            'FROM hgrid, (SELECT * FROM populated_places_simple_reduced) i',
+            'where ST_Intersects(i.the_geom_webmercator, hgrid.cell)',
+            'GROUP BY hgrid.cell'
+            ].join('\n');
+
+            var mapConfig = {
+                "version": "1.4.0",
+                "layers": [
+                    {
+                        "type": 'mapnik',
+                        "options": {
+                            "cartocss_version": '2.3.0',
+                            "sql": sql,
+                            "cartocss": cartocss
+                        }
+                    }
+                ]
+            };
+
+            this.testClient = new TestClient(mapConfig);
+            this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) {
+                var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0));
+                assert.ok(!err, err);
+
+                assert.ok(geojsonTile);
+                assert.equal(geojsonTile.features.length, 5);
+
+                done();
+            });
+        });
+
+        it('should skip empty and null columns for geojson tiles', function (done) {
+
+            var mapConfig = {
+                "analyses": [
+                    {
+                        "id": "a0",
+                        "params": {
+                            "query": "SELECT * FROM test_table"
+                        },
+                        "type": "source"
+                    }
+                ],
+                "dataviews": {
+                    "4e7b0e07-6d21-4b83-9adb-6d7e17eea6ca": {
+                        "options": {
+                            "aggregationColumn": null,
+                            "column": "cartodb_id",
+                            "operation": "avg"
+                        },
+                        "source": {
+                            "id": "a0"
+                        },
+                        "type": "formula"
+                    },
+                    "74f590f8-625c-4e95-922f-34ad3e9919c0": {
+                        "options": {
+                            "aggregation": "sum",
+                            "aggregationColumn": "cartodb_id",
+                            "column": "name"
+                        },
+                        "source": {
+                            "id": "a0"
+                        },
+                        "type": "aggregation"
+                    },
+                    "98a75757-3006-400a-b028-fb613a6c0b69": {
+                        "options": {
+                            "aggregationColumn": null,
+                            "column": "cartodb_id",
+                            "operation": "sum"
+                        },
+                        "source": {
+                            "id": "a0"
+                        },
+                        "type": "formula"
+                    },
+                    "ebbc97b2-87d2-4895-9e1f-2f012df3679d": {
+                        "options": {
+                            "aggregationColumn": null,
+                            "bins": "12",
+                            "column": "cartodb_id"
+                        },
+                        "source": {
+                            "id": "a0"
+                        },
+                        "type": "histogram"
+                    },
+                    "ebc0653f-3581-469c-8b31-c969e440a865": {
+                        "options": {
+                            "aggregationColumn": null,
+                            "column": "cartodb_id",
+                            "operation": "avg"
+                        },
+                        "source": {
+                            "id": "a0"
+                        },
+                        "type": "formula"
+                    }
+                },
+                "layers": [
+                    {
+                        "options": {
+                            "subdomains": "abcd",
+                            "urlTemplate": "http://{s}.basemaps.cartocdn.com/light_nolabels/{z}/{x}/{y}.png"
+                        },
+                        "type": "http"
+                    },
+                    {
+                        "options": {
+                            "attributes": {
+                                "columns": [
+                                    "name",
+                                    "address"
+                                ],
+                                "id": "cartodb_id"
+                            },
+                            "cartocss": "#layer { marker-width: 10; marker-fill: red; }",
+                            "cartocss_version": "2.3.0",
+                            "interactivity": "cartodb_id",
+                            "layer_name": "wadus",
+                            "source": {
+                                "id": "a0"
+                            }
+                        },
+                        "type": "cartodb"
+                    },
+                    {
+                        "options": {
+                            "subdomains": "abcd",
+                            "urlTemplate": "http://{s}.basemaps.cartocdn.com/light_only_labels/{z}/{x}/{y}.png"
+                        },
+                        "type": "http"
+                    }
+                ]
+            };
+
+            this.testClient = new TestClient(mapConfig);
+            this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) {
+                var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0));
+                assert.ok(!err, err);
+
+                assert.ok(geojsonTile);
+                assert.equal(geojsonTile.features.length, 5);
+
+                assert.deepEqual(Object.keys(geojsonTile.features[0].properties), ['cartodb_id', 'name']);
+
+                done();
+            });
+        });
+
+    }
+};
+}

test/acceptance/named_layers.js

@@ -5,7 +5,7 @@ var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server');
 var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
 var server = new CartodbWindshaft(serverOptions);
 
-var LayergroupToken = require('../support/layergroup-token');
+var LayergroupToken = require('../../lib/cartodb/models/layergroup-token');
 
 var RedisPool = require('redis-mpool');
 var TemplateMaps = require('../../lib/cartodb/backends/template_maps.js');

test/acceptance/named_maps_static_view.js

@@ -19,7 +19,8 @@ describe('named maps static view', function() {
     var username = 'localhost';
     var templateName = 'template_with_view';
 
-    var IMAGE_TOLERANCE = 20;
+    var PNG_IMAGE_TOLERANCE = 20;
+    var JPG_IMAGE_TOLERANCE = 100;
 
     function createTemplate(view, layers) {
         return {
@@ -92,8 +93,8 @@ describe('named maps static view', function() {
         });
     }
 
-    function previewFixture(version) {
-        return './test/fixtures/previews/populated_places_simple_reduced-' + version + '.png';
+    function previewFixture(version, format='png') {
+        return './test/fixtures/previews/populated_places_simple_reduced-' + version + '.' + format;
     }
 
     it('should return an image estimating its bounds based on dataset', function (done) {
@@ -103,7 +104,7 @@ describe('named maps static view', function() {
             }
             getStaticMap(function(err, img) {
                 assert.ok(!err);
-                assert.imageIsSimilarToFile(img, previewFixture('estimated'), IMAGE_TOLERANCE, done);
+                assert.imageIsSimilarToFile(img, previewFixture('estimated'), PNG_IMAGE_TOLERANCE, done);
             });
         });
     });
@@ -122,7 +123,7 @@ describe('named maps static view', function() {
             }
             getStaticMap(function(err, img) {
                 assert.ok(!err);
-                assert.imageIsSimilarToFile(img, previewFixture('zoom-center'), IMAGE_TOLERANCE, done);
+                assert.imageIsSimilarToFile(img, previewFixture('zoom-center'), PNG_IMAGE_TOLERANCE, done);
             });
         });
     });
@@ -142,7 +143,7 @@ describe('named maps static view', function() {
             }
             getStaticMap(function(err, img) {
                 assert.ok(!err);
-                assert.imageIsSimilarToFile(img, previewFixture('bounds'), IMAGE_TOLERANCE, done);
+                assert.imageIsSimilarToFile(img, previewFixture('bounds'), PNG_IMAGE_TOLERANCE, done);
             });
         });
     });
@@ -167,7 +168,7 @@ describe('named maps static view', function() {
             }
             getStaticMap(function(err, img) {
                 assert.ok(!err);
-                assert.imageIsSimilarToFile(img, previewFixture('zoom-center'), IMAGE_TOLERANCE, done);
+                assert.imageIsSimilarToFile(img, previewFixture('zoom-center'), PNG_IMAGE_TOLERANCE, done);
             });
         });
     });
@@ -192,8 +193,32 @@ describe('named maps static view', function() {
             }
             getStaticMap({ zoom: 3 }, function(err, img) {
                 assert.ok(!err);
-                img.save('/tmp/static.png');
-                assert.imageIsSimilarToFile(img, previewFixture('override-zoom'), IMAGE_TOLERANCE, done);
+                assert.imageIsSimilarToFile(img, previewFixture('override-zoom'), PNG_IMAGE_TOLERANCE, done);
+            });
+        });
+    });
+
+    it('should return override bbox', function (done) {
+        var view = {
+            bounds: {
+                west: 0,
+                south: 0,
+                east: 45,
+                north: 45
+            },
+            zoom: 4,
+            center: {
+                lng: 40,
+                lat: 20
+            }
+        };
+        templateMaps.addTemplate(username, createTemplate(view), function (err) {
+            if (err) {
+                return done(err);
+            }
+            getStaticMap({ bbox: '0,45,90,45' }, function(err, img) {
+                assert.ok(!err);
+                assert.imageIsSimilarToFile(img, previewFixture('override-bbox'), PNG_IMAGE_TOLERANCE, done);
             });
         });
     });
@@ -232,7 +257,27 @@ describe('named maps static view', function() {
             }
             getStaticMap({ layer: 0 }, function(err, img) {
                 assert.ok(!err);
-                assert.imageIsSimilarToFile(img, previewFixture('bounds'), IMAGE_TOLERANCE, done);
+                assert.imageIsSimilarToFile(img, previewFixture('bounds'), PNG_IMAGE_TOLERANCE, done);
+            });
+        });
+    });
+
+    it('should return jpg static map', function (done) {
+        var view = {
+            zoom: 4,
+            center: {
+                lng: 40,
+                lat: 20
+            }
+        };
+        templateMaps.addTemplate(username, createTemplate(view), function (err) {
+            if (err) {
+                return done(err);
+            }
+            getStaticMap({ format: 'jpeg' }, function(err, img) {
+                assert.ok(!err);
+                assert.imageIsSimilarToFile(img, previewFixture('zoom-center', 'jpeg'),
+                                            JPG_IMAGE_TOLERANCE, done, 'jpeg');
             });
         });
     });

test/acceptance/overviews_metadata.js

@@ -5,7 +5,7 @@ var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server');
 var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
 var server = new CartodbWindshaft(serverOptions);
 
-var LayergroupToken = require('../support/layergroup-token');
+var LayergroupToken = require('../../lib/cartodb/models/layergroup-token');
 
 var RedisPool = require('redis-mpool');
 

test/acceptance/overviews_metadata_named_maps.js

@@ -5,7 +5,7 @@ var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server');
 var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
 var server = new CartodbWindshaft(serverOptions);
 
-var LayergroupToken = require('../support/layergroup-token');
+var LayergroupToken = require('../../lib/cartodb/models/layergroup-token');
 
 var RedisPool = require('redis-mpool');
 

test/acceptance/overviews_queries.js

@@ -4,7 +4,6 @@ var assert = require('../support/assert');
 var cartodbServer = require('../../lib/cartodb/server');
 var ServerOptions = require('./ported/support/ported_server_options');
 var testClient = require('./ported/support/test_client');
-var BaseController = require('../../lib/cartodb/controllers/base');
 
 describe('overviews_queries', function() {
 
@@ -13,15 +12,7 @@ describe('overviews_queries', function() {
 
     var IMAGE_EQUALS_TOLERANCE_PER_MIL = 2;
 
-    var req2paramsFn;
-    before(function() {
-        req2paramsFn = BaseController.prototype.req2params;
-        BaseController.prototype.req2params = ServerOptions.req2params;
-    });
-
     after(function() {
-        BaseController.prototype.req2params = req2paramsFn;
-
         testHelper.rmdirRecursiveSync(global.environment.millstone.cache_basedir);
     });
 

test/acceptance/ported/attributes.js

@@ -4,13 +4,10 @@ var assert        = require('../../support/assert');
 var step          = require('step');
 var cartodbServer = require('../../../lib/cartodb/server');
 var PortedServerOptions = require('./support/ported_server_options');
-var BaseController = require('../../../lib/cartodb/controllers/base');
-
-var LayergroupToken = require('../../support/layergroup-token');
 
+var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token');
 
 describe('attributes', function() {
-
     var server = cartodbServer(PortedServerOptions);
     server.setMaxListeners(0);
 
@@ -49,16 +46,6 @@ describe('attributes', function() {
         testHelper.deleteRedisKeys(keysToDelete, done);
     });
 
-    var req2paramsFn;
-    before(function() {
-        req2paramsFn = BaseController.prototype.req2params;
-        BaseController.prototype.req2params = PortedServerOptions.req2params;
-    });
-
-    after(function() {
-        BaseController.prototype.req2params = req2paramsFn;
-    });
-
     it("can only be fetched from layer having an attributes spec", function(done) {
 
             var expected_token;

test/acceptance/ported/blend.js

@@ -3,21 +3,7 @@ require('../../support/test_helper');
 var assert = require('../../support/assert');
 var testClient = require('./support/test_client');
 
-var PortedServerOptions = require('./support/ported_server_options');
-var BaseController = require('../../../lib/cartodb/controllers/base');
-
 describe('blend png renderer', function() {
-
-    var req2paramsFn;
-    before(function() {
-        req2paramsFn = BaseController.prototype.req2params;
-        BaseController.prototype.req2params = PortedServerOptions.req2params;
-    });
-
-    after(function() {
-        BaseController.prototype.req2params = req2paramsFn;
-    });
-
     var IMAGE_TOLERANCE_PER_MIL = 20;
 
     function plainTorqueMapConfig(plainColor) {

test/acceptance/ported/blend_filtering.js

@@ -5,20 +5,13 @@ var testClient = require('./support/test_client');
 var fs = require('fs');
 var http = require('http');
 
-var PortedServerOptions = require('./support/ported_server_options');
-var BaseController = require('../../../lib/cartodb/controllers/base');
-
 describe('blend layer filtering', function() {
 
     var IMG_TOLERANCE_PER_MIL = 20;
 
     var httpRendererResourcesServer;
 
-    var req2paramsFn;
     before(function(done) {
-        req2paramsFn = BaseController.prototype.req2params;
-        BaseController.prototype.req2params = PortedServerOptions.req2params;
-
         // Start a server to test external resources
         httpRendererResourcesServer = http.createServer( function(request, response) {
             var filename = __dirname + '/../../fixtures/http/light_nolabels-1-0-0.png';
@@ -32,7 +25,6 @@ describe('blend layer filtering', function() {
     });
 
     after(function(done) {
-        BaseController.prototype.req2params = req2paramsFn;
         httpRendererResourcesServer.close(done);
     });
 

test/acceptance/ported/blend_http_fallback.js

@@ -5,19 +5,13 @@ var testClient = require('./support/test_client');
 var fs = require('fs');
 var http = require('http');
 
-var PortedServerOptions = require('./support/ported_server_options');
-var BaseController = require('../../../lib/cartodb/controllers/base');
-
 describe('blend http fallback', function() {
 
     var IMG_TOLERANCE_PER_MIL = 20;
 
     var httpRendererResourcesServer;
 
-    var req2paramsFn;
     before(function(done) {
-        req2paramsFn = BaseController.prototype.req2params;
-        BaseController.prototype.req2params = PortedServerOptions.req2params;
         // Start a server to test external resources
         httpRendererResourcesServer = http.createServer( function(request, response) {
             if (request.url.match(/^\/error404\//)) {
@@ -39,7 +33,6 @@ describe('blend http fallback', function() {
     });
 
     after(function(done) {
-        BaseController.prototype.req2params = req2paramsFn;
         httpRendererResourcesServer.close(done);
     });