Release 4.0.1

Upgrade camshaft to version 0.59.4

.gitignore

@@ -8,6 +8,6 @@ tools/munin/windshaft.conf
 logs/
 pids/
 redis.pid
-test.log
-npm-debug.log
+*.log
 coverage/
+.DS_Store

.jshintignore

@@ -1,4 +1,3 @@
 test/results/
 test/monkey/
 test/benchmark.js
-test/support/

.jshintrc

@@ -40,7 +40,7 @@
     "debug"         : false,     // true: Allow debugger statements e.g. browser breakpoints.
 //    "eqnull"        : false,     // true: Tolerate use of `== null`
 //    "es5"           : false,     // true: Allow ES5 syntax (ex: getters and setters)
-//    "esnext"        : false,     // true: Allow ES.next (ES6) syntax (ex: `const`)
+    "esnext"        : true,     // true: Allow ES.next (ES6) syntax (ex: `const`)
 //    "moz"           : false,     // true: Allow Mozilla specific syntax (extends and overrides esnext features)
 //    // (ex: `for each`, multiple try/catch, function expression…)
 //    "evil"          : false,     // true: Tolerate use of `eval` and `new Function()`

.travis.yml

@@ -1,27 +1,14 @@
+sudo: required
 dist: trusty
-addons:
-  postgresql: "9.5"
-  apt:
-    sources:
-      - ubuntu-toolchain-r-test
-    packages:
-      - postgresql-9.5-postgis-2.3
-      - postgresql-plpython-9.5
-      - pkg-config
-      - libcairo2-dev
-      - libjpeg8-dev
-      - libgif-dev
-      - libpango1.0-dev
-      - g++-4.9
+
+services:
+  - docker
 
 before_install:
-  - createdb template_postgis
-  - createuser publicuser
-  - psql -c "CREATE EXTENSION postgis" template_postgis
+  - docker pull cartoimages/windshaft-carto-testing
 
-env:
-  - NPROCS=1 JOBS=1 PGUSER=postgres CXX=g++-4.9
+script:
+  - docker run -e POSTGIS_VERSION=2.4 -v `pwd`:/srv cartoimages/windshaft-carto-testing
+
+language: generic
 
-language: node_js
-node_js:
-  - "6"

HOWTO_RELEASE

@@ -1,7 +1,7 @@
 1. Test (make clean all check), fix if broken before proceeding
 2. Ensure proper version in package.json
 3. Ensure NEWS section exists for the new version, review it, add release date
-4. Recreate yarn.lock with: `yarn upgrade`
+4. If there are modified dependencies in package.json, update them with `yarn upgrade {{package_name}}@{{version}}`
 5. Commit package.json, yarn.lock, NEWS
 6. git tag -a Major.Minor.Patch # use NEWS section as content
 7. Stub NEWS/package for next version

INSTALL.md

@@ -5,7 +5,7 @@ Make sure that you have the requirements needed. These are
 
 - Core
   - Node.js >=6.9.x
-  - yarn >=0.21.3
+  - yarn >=0.27.5 <1.0.0
   - PostgreSQL >8.3.x, PostGIS >1.5.x
   - Redis >2.4.0 (http://www.redis.io)
   - Mapnik >3.x. See [Installing Mapnik](https://github.com/CartoDB/Windshaft#installing-mapnik).

Makefile

@@ -16,7 +16,7 @@ config.status--test:
 	./configure --environment=test
 
 config/environments/test.js: config.status--test
-	./config.status--test 
+	./config.status--test
 
 TEST_SUITE := $(shell find test/{acceptance,integration,unit} -name "*.js")
 TEST_SUITE_UNIT := $(shell find test/unit -name "*.js")

NEWS.md

@@ -1,10 +1,224 @@
 # Changelog
 
-## 3.6.7
-Released 2017-09-01
+## 4.0.1
+Released 2017-10-18
 
 Announcements:
- - Adding a new cartodb-psql fixing a pg vulnerability.
+ - Upgrades camshaft to [0.59.4](https://github.com/CartoDB/camshaft/releases/tag/0.59.4).
+ - Split and move `req2params` method to multiple middlewares.
+ - Use express error handler middleware to respond in case of something went wrong.
+ - Use `res.locals` object to share info between middlewares and leave `req.params` as an object containing properties mapped to the named route params.
+ - Move `LZMA` decompression to its own middleware.
+ - Implement stats middleware removing some duplicated code while sending response.
+
+
+## 4.0.0
+Released 2017-10-04
+
+Backward incompatible changes:
+ - Removes `list` dataview type.
+
+Announcements:
+ - Upgrades body-parser to 1.18.2.
+ - Upgrades express to 4.16.0.
+ - Upgrades debug to 3.1.0.
+ - Upgrades request to 2.83.0.
+ - Upgrades turbo-carto to [0.20.1](https://github.com/CartoDB/turbo-carto/releases/tag/0.20.1)
+ - Upgrades cartodb-psql to [0.10.2](https://github.com/CartoDB/node-cartodb-psql/releases/tag/0.10.2).
+ - Upgrades camshaft to [0.59.2](https://github.com/CartoDB/camshaft/releases/tag/0.59.2).
+ - Upgrades windshaft to [3.3.3](https://github.com/CartoDB/windshaft/releases/tag/3.3.3).
+ - Upgrades yarn minimum version requirement to v0.27.5
+
+
+## 3.13.0
+Released 2017-10-02
+ - Upgrades camshaft, cartodb-query-tables, and turbo-carto: better support for query variables.
+
+Bugfixes:
+ - Bounding box parameter ignored in static named maps #735.
+ - camhaft 0.59.1 fixes duplicate columns in aggregate-intersection analysis
+
+## 3.12.10
+Released 2017-09-18
+ - Upgrades windshaft to [3.3.2](https://github.com/CartoDB/windshaft/releases/tag/3.3.2).
+
+## 3.12.9
+Released 2017-09-07
+
+Bug fixes:
+- Do not use distinct when calculating quantiles. #743
+
+## 3.12.8
+Released 2017-09-07
+
+Bug fixes:
+- Integer out of range in date histograms. (https://github.com/CartoDB/support/issues/962)
+
+## 3.12.7
+Released 2017-09-01
+
+ - Upgrades camshaft to [0.58.1](https://github.com/CartoDB/camshaft/releases/tag/0.58.1).
+
+
+## 3.12.6
+Released 2017-08-31
+
+ - Upgrades camshaft to [0.58.0](https://github.com/CartoDB/camshaft/releases/tag/0.58.0).
+
+
+## 3.12.5
+Released 2017-08-24
+
+ - Upgrades camshaft to [0.57.0](https://github.com/CartoDB/camshaft/releases/tag/0.57.0).
+
+
+## 3.12.4
+Released 2017-08-23
+
+Announcements:
+ - Upgrades camshaft to [0.56.0](https://github.com/CartoDB/camshaft/releases/tag/0.56.0).
+
+## 3.12.3
+Released 2017-08-22
+
+Announcements:
+ - Upgrades camshaft to [0.55.8](https://github.com/CartoDB/camshaft/releases/tag/0.55.8).
+
+## 3.12.2
+Released 2017-08-16
+
+Bug fixes:
+ - Polygon count problems #725.
+
+
+## 3.12.1
+Released 2017-08-13
+ - Upgrades cartodb-psql to [0.10.1](https://github.com/CartoDB/node-cartodb-psql/releases/tag/0.10.1).
+ - Upgrades windshaft to [3.3.1](https://github.com/CartoDB/windshaft/releases/tag/3.3.1).
+ - Upgrades camshaft to [0.55.7](https://github.com/CartoDB/camshaft/releases/tag/0.55.7).
+
+
+## 3.12.0
+Released 2017-08-10
+
+Announcements:
+ - Apply max tile response time for requests to layergoup, tiles, static maps, attributes and dataviews services #717.
+ - Upgrades windshaft to [3.3.0](https://github.com/CartoDB/windshaft/releases/tag/3.3.0).
+ - Upgrades cartodb-redis to [0.14.0](https://github.com/CartoDB/node-cartodb-redis/releases/tag/0.14.0).
+
+
+## 3.11.0
+Released 2017-08-08
+
+Announcements:
+ - Allow to override with any aggregation for histograms instantiated w/o aggregation.
+
+Bug fixes:
+ - Apply timezone after truncating the minimun date for each bin to calculate timestamps in time-series.
+ - Support timestamp with timezones to calculate the number of bins in time-series.
+ - Fixed issue related to name collision while building time-series query.
+
+
+## 3.10.1
+Released 2017-08-04
+
+Bug fixes:
+ - Exclude Infinities & NaNs from ramps #719.
+ - Fixed issue in time-series when aggregation starts at 1970-01-01 (epoch) #720.
+
+
+## 3.10.0
+Released 2017-08-03
+
+Announcements:
+ - Improve time-series dataview, now supports date aggregations (e.g: daily, weekly, monthly, etc.) and timezones (UTC by default) #698.
+ - Support special numeric values (±Infinity, NaN) for json responses #706
+
+
+## 3.9.8
+Released 2017-07-21
+
+ - Upgrades windshaft to [3.2.2](https://github.com/CartoDB/windshaft/releases/tag/3.2.2).
+
+
+## 3.9.7
+Released 2017-07-20
+
+Bug fixes:
+ - Respond with 204 (No content) when vector tile has no data #712
+
+Announcements:
+ - Upgrades turbo-carto to [0.19.2](https://github.com/CartoDB/turbo-carto/releases/tag/0.19.2)
+
+
+## 3.9.6
+Released 2017-07-11
+
+ - Dataviews: support for aggregation in search results #708
+
+
+## 3.9.5
+Released 2017-06-27
+
+ - Dataviews: support special numeric values (±Infinity, NaN) #700
+
+
+## 3.9.4
+Released 2017-06-22
+
+Announcements:
+ - Upgrades camshaft to [0.55.6](https://github.com/CartoDB/camshaft/releases/tag/0.55.6).
+
+## 3.9.3
+Released 2017-06-16
+
+Announcements:
+ - Upgrades camshaft to [0.55.5](https://github.com/CartoDB/camshaft/releases/tag/0.55.5).
+
+## 3.9.2
+Released 2017-06-16
+
+Announcements:
+ - Upgrades camshaft to [0.55.4](https://github.com/CartoDB/camshaft/releases/tag/0.55.4).
+
+## 3.9.1
+Released 2017-06-06
+
+Announcements:
+ - Upgrades camshaft to [0.55.3](https://github.com/CartoDB/camshaft/releases/tag/0.55.3).
+
+
+## 3.9.0
+Released 2017-05-31
+
+Announcements:
+ - Upgrades windshaft to [3.2.1](https://github.com/CartoDB/windshaft/releases/tag/3.2.1).
+ - Add support to retrieve info about layer stats in map instantiation.
+ - Upgrades camshaft to [0.55.2](https://github.com/CartoDB/camshaft/releases/tag/0.55.2).
+ - Remove promise polyfill from turbo-carto adapter
+
+
+## 3.8.0
+Released 2017-05-22
+
+Announcements:
+ - Upgrades camshaft to [0.55.0](https://github.com/CartoDB/camshaft/releases/tag/0.55.0).
+ - Upgrades turbo-carto to [0.19.1](https://github.com/CartoDB/turbo-carto/releases/tag/0.19.1)
+
+
+## 3.7.1
+Released 2017-05-18
+
+Bug fixes:
+ - Fix buffersize assignment when is not defined in requested mapconfig.
+
+
+## 3.7.0
+Released 2017-05-18
+
+Announcements:
+- Manage multiple values of buffer-size for different formats
+- Upgrades windshaft to [3.2.0](https://github.com/CartoDB/windshaft/releases/tag/3.2.0).
 
 
 ## 3.6.6

app.js

@@ -2,14 +2,20 @@ var http = require('http');
 var https = require('https');
 var path = require('path');
 var fs = require('fs');
-
 var _ = require('underscore');
+var semver = require('semver');
 
 // jshint undef:false
 var log = console.log.bind(console);
 var logError = console.error.bind(console);
 // jshint undef:true
 
+var nodejsVersion = process.versions.node;
+if (!semver.satisfies(nodejsVersion, '>=6.9.0')) {
+    logError(`Node version ${nodejsVersion} is not supported, please use Node.js 6.9 or higher.`);
+    process.exit(1);
+}
+
 var argv = require('yargs')
     .usage('Usage: $0 <environment> [options]')
     .help('h')

config/environments/development.js.example

@@ -107,6 +107,20 @@ var config = {
       // Milliseconds since last access before renderer cache item expires
       cache_ttl: 60000,
       statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
+      mvt: {
+        //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik,
+        //PostGIS 2.4 is required for this to work
+        //If disabled it will use Mapnik MVT generation
+        usePostGIS: false,
+        dbPoolParams: {
+              // maximum number of resources to create at any given time
+              size: 16,
+              // max milliseconds a resource can go unused before it should be destroyed
+              idleTimeout: 3000,
+              // frequency to check for idle resources
+              reapInterval: 1000
+          }
+      },
       mapnik: {
           // The size of the pool of internal mapnik backend
           // This pool size is per mapnik renderer created in Windshaft's RendererFactory
@@ -324,8 +338,7 @@ var config = {
         // whether the affected tables for a given SQL must query directly postgresql or use the SQL API
         cdbQueryTablesFromPostgres: true,
         // whether in mapconfig is available stats & metadata for each layer
-        layerMetadata: true
-
+        layerStats: true
     }
 };
 

config/environments/production.js.example

@@ -101,6 +101,20 @@ var config = {
       // Milliseconds since last access before renderer cache item expires
       cache_ttl: 60000,
       statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
+      mvt: {
+        //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik,
+        //PostGIS 2.4 is required for this to work
+        //If disabled it will use Mapnik MVT generation
+        usePostGIS: false,
+        dbPoolParams: {
+              // maximum number of resources to create at any given time
+              size: 16,
+              // max milliseconds a resource can go unused before it should be destroyed
+              idleTimeout: 3000,
+              // frequency to check for idle resources
+              reapInterval: 1000
+          }
+      },
       mapnik: {
           // The size of the pool of internal mapnik backend
           // This pool size is per mapnik renderer created in Windshaft's RendererFactory
@@ -324,7 +338,7 @@ var config = {
         // whether the affected tables for a given SQL must query directly postgresql or use the SQL API
         cdbQueryTablesFromPostgres: true,
         // whether in mapconfig is available stats & metadata for each layer
-        layerMetadata: false
+        layerStats: false
     }
 };
 

config/environments/staging.js.example

@@ -101,6 +101,20 @@ var config = {
       // Milliseconds since last access before renderer cache item expires
       cache_ttl: 60000,
       statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
+      mvt: {
+        //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik,
+        //PostGIS 2.4 is required for this to work
+        //If disabled it will use Mapnik MVT generation
+        usePostGIS: false,
+        dbPoolParams: {
+              // maximum number of resources to create at any given time
+              size: 16,
+              // max milliseconds a resource can go unused before it should be destroyed
+              idleTimeout: 3000,
+              // frequency to check for idle resources
+              reapInterval: 1000
+          }
+      },
       mapnik: {
           // The size of the pool of internal mapnik backend
           // This pool size is per mapnik renderer created in Windshaft's RendererFactory
@@ -324,7 +338,7 @@ var config = {
         // whether the affected tables for a given SQL must query directly postgresql or use the SQL API
         cdbQueryTablesFromPostgres: true,
         // whether in mapconfig is available stats & metadata for each layer
-        layerMetadata: true
+        layerStats: true
     }
 };
 

config/environments/test.js.example

@@ -100,6 +100,20 @@ var config = {
       // Milliseconds since last access before renderer cache item expires
       cache_ttl: 60000,
       statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
+      mvt: {
+        //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik,
+        //PostGIS 2.4 is required for this to work
+        //If disabled it will use Mapnik MVT generation
+        usePostGIS: false,
+        dbPoolParams: {
+              // maximum number of resources to create at any given time
+              size: 16,
+              // max milliseconds a resource can go unused before it should be destroyed
+              idleTimeout: 3000,
+              // frequency to check for idle resources
+              reapInterval: 1000
+          }
+      },
       mapnik: {
           // The size of the pool of internal mapnik backend
           // This pool size is per mapnik renderer created in Windshaft's RendererFactory
@@ -318,7 +332,7 @@ var config = {
         // whether the affected tables for a given SQL must query directly postgresql or use the SQL API
         cdbQueryTablesFromPostgres: true,
         // whether in mapconfig is available stats & metadata for each layer
-        layerMetadata: true
+        layerStats: true
     }
 };
 

docker-test.sh

@@ -0,0 +1,11 @@
+export NPROCS=1 && export JOBS=1 && export CXX=g++-4.9 && export PGUSER=postgres
+
+npm install -g yarn@0.27.5
+yarn
+
+ /etc/init.d/postgresql start
+
+createdb template_postgis && createuser publicuser
+psql -c "CREATE EXTENSION postgis" template_postgis
+
+POSTGIS_VERSION=2.4 npm test

docs/MapConfig-Dataviews-extension.md

@@ -8,50 +8,13 @@ This specification describes an extension for
 
 This extension depends on Analyses extension. It extends MapConfig with a new attribute: `dataviews`.
 
-It makes possible to get tabular data from analysis nodes: lists, aggregated lists, aggregations, and histograms.
+It makes possible to get tabular data from analysis nodes: aggregated lists, aggregations, and histograms.
 
 ## 2.1. Dataview types
 
-### List
-
-A list is a simple result set per row where is possible to retrieve several columns from the original layer query.
-
-Definition
-```
-{
-    // REQUIRED
-    // string, `type` the list type
-    “type”: “list”,
-    // REQUIRED
-    // object, `options` dataview params
-    “options”: {
-        // REQUIRED
-        // array, `columns` to select for the list
-        “columns”: [“name”, “description”]
-    }
-}
-```
-
-Expected output
-```
-{
-  "type": "list",
-  "rows": [
-    {
-      "{columnName1}": "val1",
-      "{columnName2}": 100
-    },
-    {
-      "{columnName1}": "val2",
-      "{columnName2}": 200
-    }
-  ]
-}
-```
-
 ### Aggregation
 
-An aggregation is very similar to a list but results are aggregated by a column and a given aggregation function.
+An aggregation is a list with aggregated results by a column and a given aggregation function.
 
 Definition
 ```

lib/cartodb/api/auth_api.js

@@ -19,22 +19,22 @@ function AuthApi(pgConnection, metadataBackend, mapStore, templateMaps) {
 
 module.exports = AuthApi;
 
-// Check if a request is authorized by a signer
+// Check if the user is authorized by a signer
 //
-// @param req express request object
+// @param res express response object
 // @param callback function(err, signed_by) signed_by will be
 //                 null if the request is not signed by anyone
 //                 or will be a string cartodb username otherwise.
 //
-AuthApi.prototype.authorizedBySigner = function(req, callback) {
-    if ( ! req.params.token || ! req.params.signer ) {
+AuthApi.prototype.authorizedBySigner = function(res, callback) {
+    if ( ! res.locals.token || ! res.locals.signer ) {
         return callback(null, false); // no signer requested
     }
 
     var self = this;
 
-    var layergroup_id = req.params.token;
-    var auth_token = req.params.auth_token;
+    var layergroup_id = res.locals.token;
+    var auth_token = res.locals.auth_token;
 
     this.mapStore.load(layergroup_id, function(err, mapConfig) {
         if (err) {
@@ -84,11 +84,12 @@ AuthApi.prototype.authorizedByAPIKey = function(user, req, callback) {
  * Check access authorization
  *
  * @param req - standard req object. Importantly contains table and host information
+ * @param res - standard res object. Contains the auth parameters in locals
  * @param callback function(err, allowed) is access allowed not?
  */
-AuthApi.prototype.authorize = function(req, callback) {
+AuthApi.prototype.authorize = function(req, res, callback) {
     var self = this;
-    var user = req.context.user;
+    var user = res.locals.user;
 
     step(
         function () {
@@ -101,11 +102,11 @@ AuthApi.prototype.authorize = function(req, callback) {
             // if not authorized by api_key, continue
             if (!authorized)  {
                 // not authorized by api_key, check if authorized by signer
-                return self.authorizedBySigner(req, this);
+                return self.authorizedBySigner(res, this);
             }
 
             // authorized by api key, login as the given username and stop
-            self.pgConnection.setDBAuth(user, req.params, function(err) {
+            self.pgConnection.setDBAuth(user, res.locals, function(err) {
                 callback(err, true); // authorized (or error)
             });
         },
@@ -120,7 +121,7 @@ AuthApi.prototype.authorize = function(req, callback) {
                 // if no signer name was given, let dbparams and
                 // PostgreSQL do the rest.
                 //
-                if ( ! req.params.signer ) {
+                if ( ! res.locals.signer ) {
                     return callback(null, true); // authorized so far
                 }
 
@@ -128,7 +129,7 @@ AuthApi.prototype.authorize = function(req, callback) {
                 return callback(null, false);
             }
 
-            self.pgConnection.setDBAuth(user, req.params, function(err) {
+            self.pgConnection.setDBAuth(user, res.locals, function(err) {
                 req.profiler.done('setDBAuth');
                 callback(err, true); // authorized (or error)
             });

lib/cartodb/api/user_limits_api.js

@@ -1,3 +1,5 @@
+var step = require('step');
+
 /**
  *
  * @param metadataBackend
@@ -13,16 +15,65 @@ function UserLimitsApi(metadataBackend, options) {
 
 module.exports = UserLimitsApi;
 
-UserLimitsApi.prototype.getRenderLimits = function (username, callback) {
+UserLimitsApi.prototype.getRenderLimits = function (username, apiKey, callback) {
     var self = this;
-    this.metadataBackend.getTilerRenderLimit(username, function handleTilerLimits(err, renderLimit) {
+
+    var limits = {
+        cacheOnTimeout: self.options.limits.cacheOnTimeout || false,
+        render: self.options.limits.render || 0
+    };
+
+    self.getTimeoutRenderLimit(username, apiKey, function (err, timeoutRenderLimit) {
         if (err) {
             return callback(err);
         }
 
-        return callback(null, {
-            cacheOnTimeout: self.options.limits.cacheOnTimeout || false,
-            render: renderLimit || self.options.limits.render || 0
-        });
+        if (timeoutRenderLimit && timeoutRenderLimit.render) {
+            if (Number.isFinite(timeoutRenderLimit.render)) {
+                limits.render = timeoutRenderLimit.render;
+            }
+        }
+
+        return callback(null, limits);
     });
 };
+
+UserLimitsApi.prototype.getTimeoutRenderLimit = function (username, apiKey, callback) {
+    var self = this;
+
+    step(
+        function isAuthorized() {
+            var next = this;
+
+            if (!apiKey) {
+                return next(null, false);
+            }
+
+            self.metadataBackend.getUserMapKey(username, function (err, userApiKey) {
+                if (err) {
+                    return next(err);
+                }
+
+                return next(null, userApiKey === apiKey);
+            });
+        },
+        function getUserTimeoutRenderLimits(err, authorized) {
+            var next = this;
+
+            if (err) {
+                return next(err);
+            }
+
+            self.metadataBackend.getUserTimeoutRenderLimits(username, function (err, timeoutRenderLimit) {
+                if (err) {
+                    return next(err);
+                }
+
+                next(null, {
+                    render: authorized ? timeoutRenderLimit.render : timeoutRenderLimit.renderPublic
+                });
+            });
+        },
+        callback
+    );
+};

lib/cartodb/backends/dataview.js

@@ -24,7 +24,7 @@ module.exports = DataviewBackend;
 
 DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, params, callback) {
 
-    var dataviewName = params.dataviewName;
+    var dataviewName = params.dataviewName;  
     step(
         function getMapConfig() {
             mapConfigProvider.getMapConfig(this);
@@ -43,53 +43,19 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param
             ownFilter = !!ownFilter;
 
             var query = (ownFilter) ? dataviewDefinition.sql.own_filter_on : dataviewDefinition.sql.own_filter_off;
-            var sourceId =  dataviewDefinition.source.id; // node.id
-            var layer = _.find(mapConfig.obj().layers, function(l) {
-                return l.options.source && (l.options.source.id === sourceId);
-            });
-            var queryRewriteData = layer && layer.options.query_rewrite_data;
-            if (queryRewriteData && dataviewDefinition.node.type === 'source') {
-                queryRewriteData = _.extend({}, queryRewriteData, {
-                    filters: dataviewDefinition.node.filters,
-                    unfiltered_query: dataviewDefinition.sql.own_filter_on
-                });
-            }
-
             if (params.bbox) {
                 var bboxFilter = new BBoxFilter({column: 'the_geom_webmercator', srid: 3857}, {bbox: params.bbox});
                 query = bboxFilter.sql(query);
-                if ( queryRewriteData ) {
-                    var bbox_filter_definition = {
-                        type: 'bbox',
-                        options: {
-                            column: 'the_geom_webmercator',
-                            srid: 3857
-                        },
-                        params: {
-                            bbox: params.bbox
-                        }
-                    };
-                    queryRewriteData = _.extend(queryRewriteData, { bbox_filter: bbox_filter_definition });
-                }
             }
 
+            var queryRewriteData = getQueryRewriteData(mapConfig, dataviewDefinition, params);
+
             var dataviewFactory = DataviewFactoryWithOverviews.getFactory(
                 overviewsQueryRewriter, queryRewriteData, { bbox: params.bbox }
             );
 
-            var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins'),
-                function castNumbers(overrides, val, k) {
-                    if (!Number.isFinite(+val)) {
-                        throw new Error('Invalid number format for parameter \'' + k + '\'');
-                    }
-                    overrides[k] = +val;
-                    return overrides;
-                },
-                {ownFilter: ownFilter}
-            );
-
             var dataview = dataviewFactory.getDataview(query, dataviewDefinition);
-            dataview.getResult(pg, overrideParams, this);
+            dataview.getResult(pg, getOverrideParams(params, ownFilter), this);
         },
         function returnCallback(err, result) {
             return callback(err, result);
@@ -97,9 +63,57 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param
     );
 };
 
-DataviewBackend.prototype.search = function (mapConfigProvider, user, params, callback) {
-    var dataviewName = params.dataviewName;
+function getQueryRewriteData(mapConfig, dataviewDefinition, params) {
+    var sourceId = dataviewDefinition.source.id; // node.id
+    var layer = _.find(mapConfig.obj().layers, function(l) {
+        return l.options.source && (l.options.source.id === sourceId);
+    });
+    var queryRewriteData = layer && layer.options.query_rewrite_data;
+    if (queryRewriteData && dataviewDefinition.node.type === 'source') {
+        queryRewriteData = _.extend({}, queryRewriteData, {
+            filters: dataviewDefinition.node.filters,
+            unfiltered_query: dataviewDefinition.sql.own_filter_on
+        });
+    }
 
+    if (params.bbox && queryRewriteData) {
+        var bbox_filter_definition = {
+            type: 'bbox',
+            options: {
+                column: 'the_geom_webmercator',
+                srid: 3857
+            },
+            params: {
+                bbox: params.bbox
+            }
+        };
+        queryRewriteData = _.extend(queryRewriteData, { bbox_filter: bbox_filter_definition });
+    }
+
+    return queryRewriteData;
+}
+
+function getOverrideParams(params, ownFilter) {
+    var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins', 'offset'),
+        function castNumbers(overrides, val, k) {
+            if (!Number.isFinite(+val)) {
+                throw new Error('Invalid number format for parameter \'' + k + '\'');
+            }
+            overrides[k] = +val;
+            return overrides;
+        },
+        {ownFilter: ownFilter}
+    );
+
+    // validation will be delegated to the proper dataview
+    if (params.aggregation !== undefined) {
+        overrideParams.aggregation = params.aggregation;
+    }
+
+    return overrideParams;
+}
+
+DataviewBackend.prototype.search = function (mapConfigProvider, user, dataviewName, params, callback) {
     step(
         function getMapConfig() {
             mapConfigProvider.getMapConfig(this);

lib/cartodb/backends/layer-stats/empty-layer-stats.js

@@ -0,0 +1,16 @@
+function EmptyLayerStats(types) {
+    this._types = types || {};
+}
+
+EmptyLayerStats.prototype.is = function (type) {
+    return this._types[type] ? this._types[type] : false;
+};
+
+EmptyLayerStats.prototype.getStats =
+function (layer, dbConnection, callback) {
+    setImmediate(function() {
+        callback(null, {});
+    });
+};
+
+module.exports = EmptyLayerStats;

lib/cartodb/backends/layer-stats/factory.js

@@ -0,0 +1,23 @@
+var LayerStats = require('./layer-stats');
+var EmptyLayerStats = require('./empty-layer-stats');
+var MapnikLayerStats = require('./mapnik-layer-stats');
+var TorqueLayerStats = require('./torque-layer-stats');
+
+module.exports = function LayerStatsFactory(type) {
+    var layerStatsIterator = [];
+    var selectedType = type || 'ALL';
+
+    if (selectedType === 'ALL') {
+        layerStatsIterator.push(new EmptyLayerStats({ http: true, plain: true }));
+        layerStatsIterator.push(new MapnikLayerStats());
+        layerStatsIterator.push(new TorqueLayerStats());
+    } else if (selectedType === 'mapnik') {
+        layerStatsIterator.push(new EmptyLayerStats({ http: true, plain: true, torque: true }));
+        layerStatsIterator.push(new MapnikLayerStats());
+    } else if (selectedType === 'torque') {
+        layerStatsIterator.push(new EmptyLayerStats({ http: true, plain: true, mapnik: true }));
+        layerStatsIterator.push(new TorqueLayerStats());
+    }
+
+    return new LayerStats(layerStatsIterator);
+};

lib/cartodb/backends/layer-stats/layer-stats.js

@@ -0,0 +1,45 @@
+var queue = require('queue-async');
+
+function LayerStats(layerStatsIterator) {
+    this.layerStatsIterator = layerStatsIterator;
+}
+
+LayerStats.prototype.getStats = function (mapConfig, dbConnection, callback) {
+    var self = this;
+    var stats = [];
+
+    if (!mapConfig.getLayers().length) {
+        return callback(null, stats);
+    }
+    var metaQueue = queue(mapConfig.getLayers().length);
+    mapConfig.getLayers().forEach(function (layer, layerId) {
+        var layerType = mapConfig.layerType(layerId);
+
+        for (var i = 0; i < self.layerStatsIterator.length; i++) {
+            if (self.layerStatsIterator[i].is(layerType)) {
+                var getStats = self.layerStatsIterator[i].getStats.bind(self.layerStatsIterator[i]);
+                metaQueue.defer(getStats, layer, dbConnection);
+                break;
+            }
+        }
+    });
+
+    metaQueue.awaitAll(function (err, results) {
+        if (err) {
+            return callback(err);
+        }
+
+        if (!results) {
+            return callback(null, null);
+        }
+
+        mapConfig.getLayers().forEach(function (layer, layerIndex) {
+            stats[layerIndex] = results[layerIndex];
+        });
+
+        return callback(err, stats);
+    });
+
+};
+
+module.exports = LayerStats;

lib/cartodb/backends/layer-stats/mapnik-layer-stats.js

@@ -0,0 +1,28 @@
+var queryUtils = require('../../utils/query-utils');
+
+function MapnikLayerStats () {
+    this._types = {
+        mapnik: true,
+        cartodb: true
+    };
+}
+
+MapnikLayerStats.prototype.is = function (type) {
+    return this._types[type] ? this._types[type] : false;
+};
+
+MapnikLayerStats.prototype.getStats =
+function (layer, dbConnection, callback) {
+    var queryRowCountSql = queryUtils.getQueryRowCount(layer.options.sql);
+    // This query would gather stats for postgresql table if not exists
+    dbConnection.query(queryRowCountSql, function (err, res) {
+        if (err) {
+            return callback(null, {estimatedFeatureCount: -1});
+        } else {
+            // We decided that the relation is 1 row == 1 feature
+            return callback(null, {estimatedFeatureCount: res.rows[0].rows});
+        }
+    });
+};
+
+module.exports = MapnikLayerStats;

lib/cartodb/backends/layer-stats/torque-layer-stats.js

@@ -0,0 +1,16 @@
+function TorqueLayerStats() {
+    this._types = {
+        torque: true
+    };
+}
+
+TorqueLayerStats.prototype.is = function (type) {
+    return this._types[type] ? this._types[type] : false;
+};
+
+TorqueLayerStats.prototype.getStats =
+function (layer, dbConnection, callback) {
+    return callback(null, {});
+};
+
+module.exports = TorqueLayerStats;

lib/cartodb/backends/stats.js

@@ -0,0 +1,16 @@
+var layerStats = require('./layer-stats/factory');
+
+function StatsBackend() {
+}
+
+module.exports = StatsBackend;
+
+StatsBackend.prototype.getStats = function(mapConfig, dbConnection, callback) {
+    var enabledFeatures = global.environment.enabledFeatures;
+    var layerStatsEnabled = enabledFeatures ? enabledFeatures.layerStats: false;
+    if (layerStatsEnabled) {
+        layerStats().getStats(mapConfig, dbConnection, callback);
+    } else {
+        return callback(null, []);
+    }
+};

lib/cartodb/backends/template_maps.js

@@ -296,7 +296,7 @@ TemplateMaps.prototype.delTemplate = function(owner, tpl_id, callback) {
 // @param callback function(err)
 //
 TemplateMaps.prototype.updTemplate = function(owner, tpl_id, template, callback) {
-    
+
     var self = this;
 
     template = templateDefaults(template);
@@ -430,13 +430,17 @@ var _reNumber = /^([-+]?[\d\.]?\d+([eE][+-]?\d+)?)$/,
     _reCSSColorVal = /^#[0-9a-fA-F]{3,6}$/;
 
 function _replaceVars (str, params) {
-  //return _.template(str, params); // lazy way, possibly dangerous
-  // Construct regular expressions for each param
+    // Construct regular expressions for each param
     Object.keys(params).forEach(function(k) {
         str = str.replace(new RegExp("<%=\\s*" + k + "\\s*%>", "g"), params[k]);
     });
     return str;
 }
+
+function isObject(val) {
+    return ( _.isObject(val) && !_.isArray(val) && !_.isFunction(val));
+}
+
 TemplateMaps.prototype.instance = function(template, params) {
   var all_params = {};
   var phold = template.placeholders || {};
@@ -474,6 +478,13 @@ TemplateMaps.prototype.instance = function(template, params) {
 
   // NOTE: we're deep-cloning the layergroup here
   var layergroup = JSON.parse(JSON.stringify(template.layergroup));
+
+  if (layergroup.buffersize && isObject(layergroup.buffersize)) {
+      Object.keys(layergroup.buffersize).forEach(function(k) {
+        layergroup.buffersize[k] = parseInt(_replaceVars(layergroup.buffersize[k], all_params), 10);
+      });
+  }
+
   for (var i=0; i<layergroup.layers.length; ++i) {
     var lyropt = layergroup.layers[i].options;
 

lib/cartodb/backends/turbo-carto-postgres-datasource.js

@@ -10,15 +10,21 @@ function createTemplate(method) {
         'max({{=it._column}}) max_val,',
         'avg({{=it._column}}) avg_val,',
         method,
-        'FROM ({{=it._sql}}) _table_sql WHERE {{=it._column}} IS NOT NULL'
+        'FROM ({{=it._sql}}) _table_sql WHERE {{=it._column}} IS NOT NULL',
+        'AND',
+        '  {{=it._column}} != \'infinity\'::float',
+        'AND',
+        '  {{=it._column}} != \'-infinity\'::float',
+        'AND',
+        '  {{=it._column}} != \'NaN\'::float'
     ].join('\n'));
 }
 
 var methods = {
-    quantiles: 'CDB_QuantileBins(array_agg(distinct({{=it._column}}::numeric)), {{=it._buckets}}) as quantiles',
+    quantiles: 'CDB_QuantileBins(array_agg({{=it._column}}::numeric), {{=it._buckets}}) as quantiles',
     equal: 'CDB_EqualIntervalBins(array_agg({{=it._column}}::numeric), {{=it._buckets}}) as equal',
-    jenks: 'CDB_JenksBins(array_agg(distinct({{=it._column}}::numeric)), {{=it._buckets}}) as jenks',
-    headtails: 'CDB_HeadsTailsBins(array_agg(distinct({{=it._column}}::numeric)), {{=it._buckets}}) as headtails'
+    jenks: 'CDB_JenksBins(array_agg({{=it._column}}::numeric), {{=it._buckets}}) as jenks',
+    headtails: 'CDB_HeadsTailsBins(array_agg({{=it._column}}::numeric), {{=it._buckets}}) as headtails'
 };
 
 var methodTemplates = Object.keys(methods).reduce(function(methodTemplates, methodName) {

lib/cartodb/controllers/analyses.js

@@ -10,8 +10,10 @@ var BaseController = require('./base');
 var cors = require('../middleware/cors');
 var userMiddleware = require('../middleware/user');
 
-function AnalysesController(authApi, pgConnection) {
-    BaseController.call(this, authApi, pgConnection);
+
+function AnalysesController(prepareContext) {
+    BaseController.call(this);
+    this.prepareContext = prepareContext;
 }
 
 util.inherits(AnalysesController, BaseController);
@@ -19,7 +21,13 @@ util.inherits(AnalysesController, BaseController);
 module.exports = AnalysesController;
 
 AnalysesController.prototype.register = function(app) {
-    app.get(app.base_url_mapconfig + '/analyses/catalog', cors(), userMiddleware, this.catalog.bind(this));
+    app.get(
+        app.base_url_mapconfig + '/analyses/catalog',
+        cors(),
+        userMiddleware,
+        this.prepareContext,
+        this.catalog.bind(this)
+    );
 };
 
 AnalysesController.prototype.sendResponse = function(req, res, resource) {
@@ -27,17 +35,13 @@ AnalysesController.prototype.sendResponse = function(req, res, resource) {
     this.send(req, res, resource, 200);
 };
 
-AnalysesController.prototype.catalog = function(req, res) {
+AnalysesController.prototype.catalog = function (req, res, next) {
     var self = this;
-    var username = req.context.user;
+    var username = res.locals.user;
 
     step(
-        function reqParams() {
-            self.req2params(req, this);
-        },
-        function catalogQuery(err) {
-            assert.ifError(err);
-            var pg = new PSQL(dbParamsFromReqParams(req.params));
+        function catalogQuery() {
+            var pg = new PSQL(dbParamsFromReqParams(res.locals));
             getMetadata(username, pg, this);
         },
         function prepareResponse(err, results) {
@@ -80,7 +84,8 @@ AnalysesController.prototype.catalog = function(req, res) {
                     err = new Error('Unauthorized');
                     err.http_status = 401;
                 }
-                self.sendError(req, res, err);
+
+                next(req, res, err);
             } else {
                 self.sendResponse(req, res, { catalog: catalogWithTables });
             }

lib/cartodb/controllers/base.js

@@ -1,172 +1,9 @@
-var assert = require('assert');
-
-var _ = require('underscore');
-var step = require('step');
-var debug = require('debug')('windshaft:cartodb');
-
-var LZMA = require('lzma').LZMA;
-var lzmaWorker = new LZMA();
-
-// Whitelist query parameters and attach format
-var REQUEST_QUERY_PARAMS_WHITELIST = [
-    'config',
-    'map_key',
-    'api_key',
-    'auth_token',
-    'callback',
-    'zoom',
-    'lon',
-    'lat',
-    // widgets & filters
-    'filters', // json
-    'own_filter', // 0, 1
-    'bbox', // w,s,e,n
-    'bins', // number
-    'start', // number
-    'end', // number
-    'column_type', // string
-    // widgets search
-    'q'
-];
-
-function BaseController(authApi, pgConnection) {
-    this.authApi = authApi;
-    this.pgConnection = pgConnection;
+function BaseController() {
 }
 
 module.exports = BaseController;
 
-// jshint maxcomplexity:10
-/**
- * Whitelist input and get database name & default geometry type from
- * subdomain/user metadata held in CartoDB Redis
- * @param req - standard express request obj. Should have host & table
- * @param callback
- */
-BaseController.prototype.req2params = function(req, callback){
-    var self = this;
-
-    if ( req.query.lzma ) {
-
-        // Decode (from base64)
-        var lzma = new Buffer(req.query.lzma, 'base64')
-            .toString('binary')
-            .split('')
-            .map(function(c) {
-                return c.charCodeAt(0) - 128;
-            });
-
-
-        // Decompress
-        lzmaWorker.decompress(
-            lzma,
-            function(result) {
-                req.profiler.done('lzma');
-                try {
-                    delete req.query.lzma;
-                    _.extend(req.query, JSON.parse(result));
-                    self.req2params(req, callback);
-                } catch (err) {
-                    req.profiler.done('req2params');
-                    callback(new Error('Error parsing lzma as JSON: ' + err));
-                }
-            }
-        );
-        return;
-    }
-
-    var allowedQueryParams = REQUEST_QUERY_PARAMS_WHITELIST;
-    if (Array.isArray(req.context.allowedQueryParams)) {
-        allowedQueryParams = allowedQueryParams.concat(req.context.allowedQueryParams);
-    }
-    req.query = _.pick(req.query, allowedQueryParams);
-    req.params = _.extend({}, req.params); // shuffle things as request is a strange array/object
-
-    var user = req.context.user;
-
-    if ( req.params.token ) {
-        // Token might match the following patterns:
-        // - {user}@{tpl_id}@{token}:{cache_buster}
-        var tksplit = req.params.token.split(':');
-        req.params.token = tksplit[0];
-        if ( tksplit.length > 1 ) {
-            req.params.cache_buster= tksplit[1];
-        }
-        tksplit = req.params.token.split('@');
-        if ( tksplit.length > 1 ) {
-            req.params.signer = tksplit.shift();
-            if ( ! req.params.signer ) {
-                req.params.signer = user;
-            }
-            else if ( req.params.signer !== user ) {
-                var err = new Error(
-                        'Cannot use map signature of user "' + req.params.signer + '" on db of user "' + user + '"'
-                );
-                err.http_status = 403;
-                req.profiler.done('req2params');
-                callback(err);
-                return;
-            }
-            if ( tksplit.length > 1 ) {
-                /*var template_hash = */tksplit.shift(); // unused
-            }
-            req.params.token = tksplit.shift();
-        }
-    }
-
-    // bring all query values onto req.params object
-    _.extend(req.params, req.query);
-
-    req.profiler.done('req2params.setup');
-
-    step(
-        function getPrivacy(){
-            self.authApi.authorize(req, this);
-        },
-        function validateAuthorization(err, authorized) {
-            req.profiler.done('authorize');
-            assert.ifError(err);
-            if(!authorized) {
-                err = new Error("Sorry, you are unauthorized (permission denied)");
-                err.http_status = 403;
-                throw err;
-            }
-            return null;
-        },
-        function getDatabase(err){
-            assert.ifError(err);
-            self.pgConnection.setDBConn(user, req.params, this);
-        },
-        function finishSetup(err) {
-            if ( err ) {
-                req.profiler.done('req2params');
-                return callback(err, req);
-            }
-
-            // Add default database connection parameters
-            // if none given
-            _.defaults(req.params, {
-                dbuser: global.environment.postgres.user,
-                dbpassword: global.environment.postgres.password,
-                dbhost: global.environment.postgres.host,
-                dbport: global.environment.postgres.port
-            });
-
-            req.profiler.done('req2params');
-            callback(null, req);
-        }
-    );
-};
-// jshint maxcomplexity:6
-
-// jshint maxcomplexity:9
 BaseController.prototype.send = function(req, res, body, status, headers) {
-    if (req.params.dbhost) {
-        res.set('X-Served-By-DB-Host', req.params.dbhost);
-    }
-
-    res.set('X-Tiler-Profiler', req.profiler.toJSONString());
-
     if (headers) {
         res.set(headers);
     }
@@ -182,108 +19,4 @@ BaseController.prototype.send = function(req, res, body, status, headers) {
     } else {
         res.send(body);
     }
-
-    try {
-        // May throw due to dns, see
-        // See http://github.com/CartoDB/Windshaft/issues/166
-        req.profiler.sendStats();
-    } catch (err) {
-        debug("error sending profiling stats: " + err);
-    }
 };
-// jshint maxcomplexity:6
-
-BaseController.prototype.sendError = function(req, res, err, label) {
-    var allErrors = Array.isArray(err) ? err : [err];
-    label = label || 'UNKNOWN';
-    err = allErrors[0] || new Error(label);
-    allErrors[0] = err;
-
-    var statusCode = findStatusCode(err);
-
-    debug('[%s ERROR] -- %d: %s, %s', label, statusCode, err, err.stack);
-
-    // If a callback was requested, force status to 200
-    if (req.query && req.query.callback) {
-        statusCode = 200;
-    }
-
-    var errorResponseBody = {
-        errors: allErrors.map(errorMessage),
-        errors_with_context: allErrors.map(errorMessageWithContext)
-    };
-
-    this.send(req, res, errorResponseBody, statusCode);
-};
-
-function stripConnectionInfo(message) {
-    // Strip connection info, if any
-    return message
-        // See https://github.com/CartoDB/Windshaft/issues/173
-        .replace(/Connection string: '[^']*'\n\s/im, '')
-        // See https://travis-ci.org/CartoDB/Windshaft/jobs/20703062#L1644
-        .replace(/is the server.*encountered/im, 'encountered');
-}
-
-function errorMessage(err) {
-    // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
-    var message = (_.isString(err) ? err : err.message) || 'Unknown error';
-
-    return stripConnectionInfo(message);
-}
-
-function errorMessageWithContext(err) {
-    // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
-    var message = (_.isString(err) ? err : err.message) || 'Unknown error';
-
-    var error = {
-        type: err.type || 'unknown',
-        message: stripConnectionInfo(message),
-    };
-
-    for (var prop in err) {
-        // type & message are properties from Error's prototype and will be skipped
-        if (err.hasOwnProperty(prop)) {
-            error[prop] = err[prop];
-        }
-    }
-
-    return error;
-}
-module.exports.errorMessage = errorMessage;
-
-function findStatusCode(err) {
-    var statusCode;
-    if ( err.http_status ) {
-        statusCode = err.http_status;
-    } else {
-        statusCode = statusFromErrorMessage('' + err);
-    }
-    return statusCode;
-}
-module.exports.findStatusCode = findStatusCode;
-
-function statusFromErrorMessage(errMsg) {
-    // Find an appropriate statusCode based on message
-    // jshint maxcomplexity:7
-    var statusCode = 400;
-    if ( -1 !== errMsg.indexOf('permission denied') ) {
-        statusCode = 403;
-    }
-    else if ( -1 !== errMsg.indexOf('authentication failed') ) {
-        statusCode = 403;
-    }
-    else if (errMsg.match(/Postgis Plugin.*[\s|\n].*column.*does not exist/)) {
-        statusCode = 400;
-    }
-    else if ( -1 !== errMsg.indexOf('does not exist') ) {
-        if ( -1 !== errMsg.indexOf(' role ') ) {
-            statusCode = 403; // role 'xxx' does not exist
-        } else if ( errMsg.match(/function .* does not exist/) ) {
-            statusCode = 400; // invalid SQL (SQL function does not exist)
-        } else {
-            statusCode = 404;
-        }
-    }
-    return statusCode;
-}

lib/cartodb/controllers/layergroup.js

@@ -28,9 +28,9 @@ var QueryTables = require('cartodb-query-tables');
  * @param {AnalysisBackend} analysisBackend
  * @constructor
  */
-function LayergroupController(authApi, pgConnection, mapStore, tileBackend, previewBackend, attributesBackend,
+function LayergroupController(prepareContext, pgConnection, mapStore, tileBackend, previewBackend, attributesBackend,
                               surrogateKeysCache, userLimitsApi, layergroupAffectedTables, analysisBackend) {
-    BaseController.call(this, authApi, pgConnection);
+    BaseController.call(this);
 
     this.pgConnection = pgConnection;
     this.mapStore = mapStore;
@@ -43,77 +43,140 @@ function LayergroupController(authApi, pgConnection, mapStore, tileBackend, prev
 
     this.dataviewBackend = new DataviewBackend(analysisBackend);
     this.analysisStatusBackend = new AnalysisStatusBackend();
+
+    this.prepareContext = prepareContext;
 }
 
 util.inherits(LayergroupController, BaseController);
 
 module.exports = LayergroupController;
 
-
 LayergroupController.prototype.register = function(app) {
-    app.get(app.base_url_mapconfig +
-        '/:token/:z/:x/:y@:scale_factor?x.:format', cors(), userMiddleware,
-        this.tile.bind(this));
+    app.get(
+        app.base_url_mapconfig + '/:token/:z/:x/:y@:scale_factor?x.:format',
+        cors(),
+        userMiddleware,
+        this.prepareContext,
+        this.tile.bind(this)
+    );
 
-    app.get(app.base_url_mapconfig +
-        '/:token/:z/:x/:y.:format', cors(), userMiddleware,
-        this.tile.bind(this));
+    app.get(
+        app.base_url_mapconfig + '/:token/:z/:x/:y.:format',
+        cors(),
+        userMiddleware,
+        this.prepareContext,
+        this.tile.bind(this)
+    );
 
-    app.get(app.base_url_mapconfig +
-        '/:token/:layer/:z/:x/:y.(:format)', cors(), userMiddleware,
-        this.layer.bind(this));
+    app.get(
+        app.base_url_mapconfig + '/:token/:layer/:z/:x/:y.(:format)',
+        cors(),
+        userMiddleware,
+        validateLayerRouteMiddleware,
+        this.prepareContext,
+        this.layer.bind(this)
+    );
 
-    app.get(app.base_url_mapconfig +
-        '/:token/:layer/attributes/:fid', cors(), userMiddleware,
-        this.attributes.bind(this));
+    app.get(
+        app.base_url_mapconfig + '/:token/:layer/attributes/:fid',
+        cors(),
+        userMiddleware,
+        this.prepareContext,
+        this.attributes.bind(this)
+    );
 
-    app.get(app.base_url_mapconfig +
-        '/static/center/:token/:z/:lat/:lng/:width/:height.:format',
-        cors(), userMiddleware, allowQueryParams(['layer']),
-        this.center.bind(this));
+    app.get(
+        app.base_url_mapconfig + '/static/center/:token/:z/:lat/:lng/:width/:height.:format',
+        cors(),
+        userMiddleware,
+        allowQueryParams(['layer']),
+        this.prepareContext,
+        this.center.bind(this)
+    );
 
-    app.get(app.base_url_mapconfig +
-        '/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format',
-        cors(), userMiddleware, allowQueryParams(['layer']),
-        this.bbox.bind(this));
+    app.get(
+        app.base_url_mapconfig + '/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format',
+        cors(),
+        userMiddleware,
+        allowQueryParams(['layer']),
+        this.prepareContext,
+        this.bbox.bind(this)
+    );
 
     // Undocumented/non-supported API endpoint methods.
     // Use at your own peril.
-    app.get(app.base_url_mapconfig +
-            '/:token/dataview/:dataviewName', cors(), userMiddleware,
-        this.dataview.bind(this));
-    app.get(app.base_url_mapconfig +
-            '/:token/:layer/widget/:dataviewName', cors(), userMiddleware,
-        this.dataview.bind(this));
 
-    app.get(app.base_url_mapconfig +
-            '/:token/dataview/:dataviewName/search', cors(), userMiddleware,
-        this.dataviewSearch.bind(this));
-    app.get(app.base_url_mapconfig +
-            '/:token/:layer/widget/:dataviewName/search', cors(), userMiddleware,
-        this.dataviewSearch.bind(this));
+    var allowedDataviewQueryParams = [
+        'filters', // json
+        'own_filter', // 0, 1
+        'bbox', // w,s,e,n
+        'start', // number
+        'end', // number
+        'column_type', // string
+        'bins', // number
+        'aggregation', //string
+        'offset', // number
+        'q' // widgets search
+    ];
 
-    app.get(app.base_url_mapconfig +
-        '/:token/analysis/node/:nodeId', cors(), userMiddleware,
-        this.analysisNodeStatus.bind(this));
+    app.get(
+        app.base_url_mapconfig + '/:token/dataview/:dataviewName',
+        cors(),
+        userMiddleware,
+        allowQueryParams(allowedDataviewQueryParams),
+        this.prepareContext,
+        this.dataview.bind(this)
+    );
+
+    app.get(
+        app.base_url_mapconfig + '/:token/:layer/widget/:dataviewName',
+        cors(),
+        userMiddleware,
+        allowQueryParams(allowedDataviewQueryParams),
+        this.prepareContext,
+        this.dataview.bind(this)
+    );
+
+    app.get(
+        app.base_url_mapconfig + '/:token/dataview/:dataviewName/search',
+        cors(),
+        userMiddleware,
+        allowQueryParams(allowedDataviewQueryParams),
+        this.prepareContext,
+        this.dataviewSearch.bind(this)
+    );
+
+    app.get(
+        app.base_url_mapconfig + '/:token/:layer/widget/:dataviewName/search',
+        cors(),
+        userMiddleware,
+        allowQueryParams(allowedDataviewQueryParams),
+        this.prepareContext,
+        this.dataviewSearch.bind(this)
+    );
+
+    app.get(
+        app.base_url_mapconfig + '/:token/analysis/node/:nodeId',
+        cors(),
+        userMiddleware,
+        this.prepareContext,
+        this.analysisNodeStatus.bind(this)
+    );
 };
 
-LayergroupController.prototype.analysisNodeStatus = function(req, res) {
+LayergroupController.prototype.analysisNodeStatus = function(req, res, next) {
     var self = this;
 
     step(
-        function setupParams() {
-            self.req2params(req, this);
-        },
-        function retrieveNodeStatus(err) {
-            assert.ifError(err);
-            self.analysisStatusBackend.getNodeStatus(req.params, this);
+        function retrieveNodeStatus() {
+            self.analysisStatusBackend.getNodeStatus(res.locals, this);
         },
         function finish(err, nodeStatus, stats) {
             req.profiler.add(stats || {});
 
             if (err) {
-                self.sendError(req, res, err, 'GET NODE STATUS');
+                err.label = 'GET NODE STATUS';
+                next(err);
             } else {
                 self.sendResponse(req, res, nodeStatus, 200, {
                     'Cache-Control': 'public,max-age=5',
@@ -124,54 +187,50 @@ LayergroupController.prototype.analysisNodeStatus = function(req, res) {
     );
 };
 
-LayergroupController.prototype.dataview = function(req, res) {
+LayergroupController.prototype.dataview = function(req, res, next) {
     var self = this;
 
     step(
-        function setupParams() {
-            self.req2params(req, this);
-        },
-        function retrieveDataview(err) {
-            assert.ifError(err);
-
+        function retrieveDataview() {
             var mapConfigProvider = new MapStoreMapConfigProvider(
-                self.mapStore, req.context.user, self.userLimitsApi, req.params
+                self.mapStore, res.locals.user, self.userLimitsApi, res.locals
+            );
+            self.dataviewBackend.getDataview(
+                mapConfigProvider,
+                res.locals.user,
+                res.locals,
+                this
             );
-            self.dataviewBackend.getDataview(mapConfigProvider, req.context.user, req.params, this);
         },
         function finish(err, dataview, stats) {
             req.profiler.add(stats || {});
 
             if (err) {
-                self.sendError(req, res, err, 'GET DATAVIEW');
+                err.label = 'GET DATAVIEW';
+                next(err);
             } else {
                 self.sendResponse(req, res, dataview, 200);
             }
         }
     );
-
 };
 
-LayergroupController.prototype.dataviewSearch = function(req, res) {
+LayergroupController.prototype.dataviewSearch = function(req, res, next) {
     var self = this;
 
     step(
-        function setupParams() {
-            self.req2params(req, this);
-        },
-        function searchDataview(err) {
-            assert.ifError(err);
-
+        function searchDataview() {
             var mapConfigProvider = new MapStoreMapConfigProvider(
-                self.mapStore, req.context.user, self.userLimitsApi, req.params
+                self.mapStore, res.locals.user, self.userLimitsApi, res.locals
             );
-            self.dataviewBackend.search(mapConfigProvider, req.context.user, req.params, this);
+            self.dataviewBackend.search(mapConfigProvider, res.locals.user, req.params.dataviewName, res.locals, this);
         },
         function finish(err, searchResult, stats) {
             req.profiler.add(stats || {});
 
             if (err) {
-                self.sendError(req, res, err, 'GET DATAVIEW SEARCH');
+                err.label = 'GET DATAVIEW SEARCH';
+                next(err);
             } else {
                 self.sendResponse(req, res, searchResult, 200);
             }
@@ -180,28 +239,24 @@ LayergroupController.prototype.dataviewSearch = function(req, res) {
 
 };
 
-LayergroupController.prototype.attributes = function(req, res) {
+LayergroupController.prototype.attributes = function(req, res, next) {
     var self = this;
 
     req.profiler.start('windshaft.maplayer_attribute');
 
     step(
-        function setupParams() {
-            self.req2params(req, this);
-        },
-        function retrieveFeatureAttributes(err) {
-            assert.ifError(err);
-
+        function retrieveFeatureAttributes() {
             var mapConfigProvider = new MapStoreMapConfigProvider(
-                self.mapStore, req.context.user, self.userLimitsApi, req.params
+                self.mapStore, res.locals.user, self.userLimitsApi, res.locals
             );
-            self.attributesBackend.getFeatureAttributes(mapConfigProvider, req.params, false, this);
+            self.attributesBackend.getFeatureAttributes(mapConfigProvider, res.locals, false, this);
         },
         function finish(err, tile, stats) {
             req.profiler.add(stats || {});
 
             if (err) {
-                self.sendError(req, res, err, 'GET ATTRIBUTES');
+                err.label = 'GET ATTRIBUTES';
+                next(err);
             } else {
                 self.sendResponse(req, res, tile, 200);
             }
@@ -211,49 +266,48 @@ LayergroupController.prototype.attributes = function(req, res) {
 };
 
 // Gets a tile for a given token and set of tile ZXY coords. (OSM style)
-LayergroupController.prototype.tile = function(req, res) {
+LayergroupController.prototype.tile = function(req, res, next) {
     req.profiler.start('windshaft.map_tile');
-    this.tileOrLayer(req, res);
+    this.tileOrLayer(req, res, next);
 };
 
 // Gets a tile for a given token, layer set of tile ZXY coords. (OSM style)
 LayergroupController.prototype.layer = function(req, res, next) {
-    if (req.params.token === 'static') {
-        return next();
-    }
     req.profiler.start('windshaft.maplayer_tile');
-    this.tileOrLayer(req, res);
+    this.tileOrLayer(req, res, next);
 };
 
-LayergroupController.prototype.tileOrLayer = function (req, res) {
+LayergroupController.prototype.tileOrLayer = function (req, res, next) {
     var self = this;
 
     step(
-        function mapController$prepareParams() {
-            self.req2params(req, this);
-        },
-        function mapController$getTileOrGrid(err) {
-            assert.ifError(err);
+        function mapController$getTileOrGrid() {
             self.tileBackend.getTile(
-                new MapStoreMapConfigProvider(self.mapStore, req.context.user, self.userLimitsApi, req.params),
+                new MapStoreMapConfigProvider(self.mapStore, res.locals.user, self.userLimitsApi, res.locals),
                 req.params, this
             );
         },
         function mapController$finalize(err, tile, headers, stats) {
             req.profiler.add(stats);
-            self.finalizeGetTileOrGrid(err, req, res, tile, headers);
+            self.finalizeGetTileOrGrid(err, req, res, tile, headers, next);
         }
     );
 };
 
+function getStatusCode(tile, format){
+    return tile.length===0 && format==='mvt'? 204:200;
+}
+
 // This function is meant for being called as the very last
 // step by all endpoints serving tiles or grids
-LayergroupController.prototype.finalizeGetTileOrGrid = function(err, req, res, tile, headers) {
+LayergroupController.prototype.finalizeGetTileOrGrid = function(err, req, res, tile, headers, next) {
     var supportedFormats = {
         grid_json: true,
         json_torque: true,
         torque_json: true,
-        png: true
+        png: true,
+        png32: true,
+        mvt: true
     };
 
     var formatStat = 'invalid';
@@ -275,52 +329,50 @@ LayergroupController.prototype.finalizeGetTileOrGrid = function(err, req, res, t
         }
         err.message = errMsg;
 
-        this.sendError(req, res, err, 'TILE RENDER');
+        err.label = 'TILE RENDER';
+        next(err);
+
         global.statsClient.increment('windshaft.tiles.error');
         global.statsClient.increment('windshaft.tiles.' + formatStat + '.error');
     } else {
-        this.sendResponse(req, res, tile, 200, headers);
+        this.sendResponse(req, res, tile, getStatusCode(tile, formatStat), headers);
         global.statsClient.increment('windshaft.tiles.success');
         global.statsClient.increment('windshaft.tiles.' + formatStat + '.success');
     }
 };
 
-LayergroupController.prototype.bbox = function(req, res) {
+LayergroupController.prototype.bbox = function(req, res, next) {
     this.staticMap(req, res, +req.params.width, +req.params.height, {
         west: +req.params.west,
         north: +req.params.north,
         east: +req.params.east,
         south: +req.params.south
-    });
+    }, null, next);
 };
 
-LayergroupController.prototype.center = function(req, res) {
+LayergroupController.prototype.center = function(req, res, next) {
     this.staticMap(req, res, +req.params.width, +req.params.height, +req.params.z, {
         lng: +req.params.lng,
         lat: +req.params.lat
-    });
+    }, next);
 };
 
-LayergroupController.prototype.staticMap = function(req, res, width, height, zoom /* bounds */, center) {
+LayergroupController.prototype.staticMap = function(req, res, width, height, zoom /* bounds */, center, next) {
     var format = req.params.format === 'jpg' ? 'jpeg' : 'png';
-    req.params.layer = 'all';
-    req.params.format = 'png';
+    req.params.format = req.params.format || 'png';
+    res.locals.layer = res.locals.layer || 'all';
 
     var self = this;
 
     step(
-        function reqParams() {
-            self.req2params(req, this);
-        },
-        function getImage(err) {
-            assert.ifError(err);
+        function getImage() {
             if (center) {
                 self.previewBackend.getImage(
-                    new MapStoreMapConfigProvider(self.mapStore, req.context.user, self.userLimitsApi, req.params),
+                    new MapStoreMapConfigProvider(self.mapStore, res.locals.user, self.userLimitsApi, res.locals),
                     format, width, height, zoom, center, this);
             } else {
                 self.previewBackend.getImage(
-                    new MapStoreMapConfigProvider(self.mapStore, req.context.user, self.userLimitsApi, req.params),
+                    new MapStoreMapConfigProvider(self.mapStore, res.locals.user, self.userLimitsApi, res.locals),
                     format, width, height, zoom /* bounds */, this);
             }
         },
@@ -329,7 +381,8 @@ LayergroupController.prototype.staticMap = function(req, res, width, height, zoo
             req.profiler.add(stats || {});
 
             if (err) {
-                self.sendError(req, res, err, 'STATIC_MAP');
+                err.label = 'STATIC_MAP';
+                next(err);
             } else {
                 res.set('Content-Type', headers['Content-Type'] || 'image/' + format);
                 self.sendResponse(req, res, image, 200);
@@ -347,18 +400,18 @@ LayergroupController.prototype.sendResponse = function(req, res, body, status, h
 
     // Set Last-Modified header
     var lastUpdated;
-    if (req.params.cache_buster) {
+    if (res.locals.cache_buster) {
         // Assuming cache_buster is a timestamp
-        lastUpdated = new Date(parseInt(req.params.cache_buster));
+        lastUpdated = new Date(parseInt(res.locals.cache_buster));
     } else {
         lastUpdated = new Date();
     }
     res.set('Last-Modified', lastUpdated.toUTCString());
 
-    var dbName = req.params.dbname;
+    var dbName = res.locals.dbname;
     step(
         function getAffectedTables() {
-            self.getAffectedTables(req.context.user, dbName, req.params.token, this);
+            self.getAffectedTables(res.locals.user, dbName, res.locals.token, this);
         },
         function sendResponse(err, affectedTables) {
             req.profiler.done('affectedTables');
@@ -434,3 +487,12 @@ LayergroupController.prototype.getAffectedTables = function(user, dbName, layerg
         callback
     );
 };
+
+
+function validateLayerRouteMiddleware(req, res, next) {
+    if (req.params.token === 'static') {
+        return next('route');
+    }
+
+    next();
+}

lib/cartodb/controllers/map.js

@@ -30,12 +30,14 @@ var CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/cr
  * @param {UserLimitsApi} userLimitsApi
  * @param {LayergroupAffectedTables} layergroupAffectedTables
  * @param {MapConfigAdapter} mapConfigAdapter
+ * @param {StatsBackend} statsBackend
  * @constructor
  */
-function MapController(authApi, pgConnection, templateMaps, mapBackend, metadataBackend,
-                       surrogateKeysCache, userLimitsApi, layergroupAffectedTables, mapConfigAdapter) {
+function MapController(prepareContext, pgConnection, templateMaps, mapBackend, metadataBackend,
+                       surrogateKeysCache, userLimitsApi, layergroupAffectedTables, mapConfigAdapter,
+                       statsBackend) {
 
-    BaseController.call(this, authApi, pgConnection);
+    BaseController.call(this);
 
     this.pgConnection = pgConnection;
     this.templateMaps = templateMaps;
@@ -47,6 +49,9 @@ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadata
 
     this.mapConfigAdapter = mapConfigAdapter;
     this.resourceLocator = new ResourceLocator(global.environment);
+
+    this.statsBackend = statsBackend;
+    this.prepareContext = prepareContext;
 }
 
 util.inherits(MapController, BaseController);
@@ -55,38 +60,60 @@ module.exports = MapController;
 
 
 MapController.prototype.register = function(app) {
-    app.get(app.base_url_mapconfig, cors(), userMiddleware, this.createGet.bind(this));
-    app.post(app.base_url_mapconfig, cors(), userMiddleware, this.createPost.bind(this));
-    app.get(app.base_url_templated + '/:template_id/jsonp', cors(), userMiddleware, this.jsonp.bind(this));
-    app.post(app.base_url_templated + '/:template_id', cors(), userMiddleware, this.instantiate.bind(this));
+    app.get(
+        app.base_url_mapconfig,
+        cors(),
+        userMiddleware,
+        this.prepareContext,
+        this.createGet.bind(this)
+    );
+    app.post(
+        app.base_url_mapconfig,
+        cors(),
+        userMiddleware,
+        this.prepareContext,
+        this.createPost.bind(this)
+    );
+    app.get(
+        app.base_url_templated + '/:template_id/jsonp',
+        cors(),
+        userMiddleware,
+        this.prepareContext,
+        this.jsonp.bind(this)
+    );
+    app.post(
+        app.base_url_templated + '/:template_id',
+        cors(),
+        userMiddleware,
+        this.prepareContext,
+        this.instantiate.bind(this)
+    );
     app.options(app.base_url_mapconfig, cors('Content-Type'));
 };
 
-MapController.prototype.createGet = function(req, res){
+MapController.prototype.createGet = function(req, res, next){
     req.profiler.start('windshaft.createmap_get');
 
-    this.create(req, res, function createGet$prepareConfig(err, req) {
-        assert.ifError(err);
-        if ( ! req.params.config ) {
+    this.create(req, res, function createGet$prepareConfig(req, config) {
+        if ( ! config ) {
             throw new Error('layergroup GET needs a "config" parameter');
         }
-        return JSON.parse(req.params.config);
-    });
+        return JSON.parse(config);
+    }, next);
 };
 
-MapController.prototype.createPost = function(req, res) {
+MapController.prototype.createPost = function(req, res, next) {
     req.profiler.start('windshaft.createmap_post');
 
-    this.create(req, res, function createPost$prepareConfig(err, req) {
-        assert.ifError(err);
+    this.create(req, res, function createPost$prepareConfig(req) {
         if (!req.is('application/json')) {
             throw new Error('layergroup POST data must be of type application/json');
         }
         return req.body;
-    });
+    }, next);
 };
 
-MapController.prototype.instantiate = function(req, res) {
+MapController.prototype.instantiate = function(req, res, next) {
     req.profiler.start('windshaft-cartodb.instance_template_post');
 
     this.instantiateTemplate(req, res, function prepareTemplateParams(callback) {
@@ -94,10 +121,10 @@ MapController.prototype.instantiate = function(req, res) {
             return callback(new Error('Template POST data must be of type application/json'));
         }
         return callback(null, req.body);
-    });
+    }, next);
 };
 
-MapController.prototype.jsonp = function(req, res) {
+MapController.prototype.jsonp = function(req, res, next) {
     req.profiler.start('windshaft-cartodb.instance_template_get');
 
     this.instantiateTemplate(req, res, function prepareJsonTemplateParams(callback) {
@@ -116,10 +143,10 @@ MapController.prototype.jsonp = function(req, res) {
         }
 
         return callback(err, templateParams);
-    });
+    }, next);
 };
 
-MapController.prototype.create = function(req, res, prepareConfigFn) {
+MapController.prototype.create = function(req, res, prepareConfigFn, next) {
     var self = this;
 
     var mapConfig;
@@ -127,35 +154,36 @@ MapController.prototype.create = function(req, res, prepareConfigFn) {
     var context = {};
 
     step(
-        function setupParams(){
-            self.req2params(req, this);
+        function prepareConfig () {
+            const requestMapConfig = prepareConfigFn(req, res.locals.config);
+            return requestMapConfig;
         },
-        prepareConfigFn,
         function prepareAdapterMapConfig(err, requestMapConfig) {
             assert.ifError(err);
             context.analysisConfiguration = {
-                user: req.context.user,
+                user: res.locals.user,
                 db: {
-                    host: req.params.dbhost,
-                    port: req.params.dbport,
-                    dbname: req.params.dbname,
-                    user: req.params.dbuser,
-                    pass: req.params.dbpassword
+                    host: res.locals.dbhost,
+                    port: res.locals.dbport,
+                    dbname: res.locals.dbname,
+                    user: res.locals.dbuser,
+                    pass: res.locals.dbpassword
                 },
                 batch: {
-                    username: req.context.user,
-                    apiKey: req.params.api_key
+                    username: res.locals.user,
+                    apiKey: res.locals.api_key
                 }
             };
-            self.mapConfigAdapter.getMapConfig(req.context.user, requestMapConfig, req.params, context, this);
+            self.mapConfigAdapter.getMapConfig(res.locals.user, requestMapConfig, res.locals, context, this);
         },
         function createLayergroup(err, requestMapConfig) {
             assert.ifError(err);
             var datasource = context.datasource || Datasource.EmptyDatasource();
             mapConfig = new MapConfig(requestMapConfig, datasource);
             self.mapBackend.createLayergroup(
-                mapConfig, req.params,
-                new CreateLayergroupMapConfigProvider(mapConfig, req.context.user, self.userLimitsApi, req.params),
+                mapConfig, 
+                res.locals,
+                new CreateLayergroupMapConfigProvider(mapConfig, res.locals.user, self.userLimitsApi, res.locals),
                 this
             );
         },
@@ -183,11 +211,12 @@ MapController.prototype.create = function(req, res, prepareConfigFn) {
 
                     err = error;
                 }
-                self.sendError(req, res, err, 'ANONYMOUS LAYERGROUP');
+                err.label = 'ANONYMOUS LAYERGROUP';
+                next(err);
             } else {
                 var analysesResults = context.analysesResults || [];
-                self.addDataviewsAndWidgetsUrls(req.context.user, layergroup, mapConfig.obj());
-                self.addAnalysesMetadata(req.context.user, layergroup, analysesResults, true);
+                self.addDataviewsAndWidgetsUrls(res.locals.user, layergroup, mapConfig.obj());
+                self.addAnalysesMetadata(res.locals.user, layergroup, analysesResults, true);
                 addContextMetadata(layergroup, mapConfig.obj(), context);
                 res.set('X-Layergroup-Id', layergroup.layergroupid);
                 self.send(req, res, layergroup, 200);
@@ -207,18 +236,14 @@ function addContextMetadata(layergroup, mapConfig, context) {
     }
 }
 
-MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn) {
+MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn, next) {
     var self = this;
 
-    var cdbuser = req.context.user;
+    var cdbuser = res.locals.user;
 
     var mapConfigProvider;
     var mapConfig;
-
     step(
-        function setupParams(){
-            self.req2params(req, this);
-        },
         function getTemplateParams() {
             prepareParamsFn(this);
         },
@@ -233,8 +258,8 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
                 cdbuser,
                 req.params.template_id,
                 templateParams,
-                req.query.auth_token,
-                req.params
+                res.locals.auth_token,
+                res.locals
             );
             mapConfigProvider.getMapConfig(this);
         },
@@ -249,11 +274,14 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
         },
         function afterLayergroupCreate(err, layergroup) {
             assert.ifError(err);
-            self.afterLayergroupCreate(req, res, mapConfig, layergroup, mapConfigProvider.analysesResults, this);
+            self.afterLayergroupCreate(req, res, mapConfig, layergroup,
+                                       mapConfigProvider.analysesResults,
+                                       this);
         },
         function finishTemplateInstantiation(err, layergroup) {
             if (err) {
-                self.sendError(req, res, err, 'NAMED MAP LAYERGROUP');
+                err.label = 'NAMED MAP LAYERGROUP';
+                next(err);
             } else {
                 var templateHash = self.templateMaps.fingerPrint(mapConfigProvider.template).substring(0, 8);
                 layergroup.layergroupid = cdbuser + '@' + templateHash + '@' + layergroup.layergroupid;
@@ -272,10 +300,11 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
     );
 };
 
-MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, layergroup, analysesResults, callback) {
+MapController.prototype.afterLayergroupCreate =
+function(req, res, mapconfig, layergroup, analysesResults, callback) {
     var self = this;
 
-    var username = req.context.user;
+    var username = res.locals.user;
 
     var tasksleft = 2; // redis key and affectedTables
     var errors = [];
@@ -316,8 +345,9 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
         }
     });
 
-    var dbName = req.params.dbname;
+    var dbName = res.locals.dbname;
     var layergroupId = layergroup.layergroupid;
+    var dbConnection;
 
     step(
         function getPgConnection() {
@@ -325,7 +355,8 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
         },
         function getAffectedTablesAndLastUpdatedTime(err, connection) {
             assert.ifError(err);
-            QueryTables.getAffectedTablesFromQuery(connection, sql.join(';'), this);
+            dbConnection = connection;
+            QueryTables.getAffectedTablesFromQuery(dbConnection, sql.join(';'), this);
         },
         function handleAffectedTablesAndLastUpdatedTime(err, result) {
             req.profiler.done('queryTablesAndLastUpdated');
@@ -352,6 +383,21 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
 
             return null;
         },
+        function fetchLayersStats(err) {
+            assert.ifError(err);
+            var next = this;
+            self.statsBackend.getStats(mapconfig, dbConnection, function(err, layersStats) {
+                if (err) {
+                    return next(err);
+                }
+                if (layersStats.length > 0) {
+                    layergroup.metadata.layers.forEach(function (layer, index) {
+                        layer.meta.stats = layersStats[index];
+                    });
+                }
+                return next();
+            });
+        },
         function finish(err) {
             done(err);
         }

lib/cartodb/controllers/named_maps.js

@@ -10,9 +10,9 @@ var cors = require('../middleware/cors');
 var userMiddleware = require('../middleware/user');
 var allowQueryParams = require('../middleware/allow-query-params');
 
-function NamedMapsController(authApi, pgConnection, namedMapProviderCache, tileBackend, previewBackend,
+function NamedMapsController(prepareContext, namedMapProviderCache, tileBackend, previewBackend,
                              surrogateKeysCache, tablesExtentApi, metadataBackend) {
-    BaseController.call(this, authApi, pgConnection);
+    BaseController.call(this);
 
     this.namedMapProviderCache = namedMapProviderCache;
     this.tileBackend = tileBackend;
@@ -20,6 +20,7 @@ function NamedMapsController(authApi, pgConnection, namedMapProviderCache, tileB
     this.surrogateKeysCache = surrogateKeysCache;
     this.tablesExtentApi = tablesExtentApi;
     this.metadataBackend = metadataBackend;
+    this.prepareContext = prepareContext;
 }
 
 util.inherits(NamedMapsController, BaseController);
@@ -27,17 +28,26 @@ util.inherits(NamedMapsController, BaseController);
 module.exports = NamedMapsController;
 
 NamedMapsController.prototype.register = function(app) {
-    app.get(app.base_url_templated +
-        '/:template_id/:layer/:z/:x/:y.(:format)', cors(), userMiddleware,
-        this.tile.bind(this));
+    app.get(
+        app.base_url_templated + '/:template_id/:layer/:z/:x/:y.(:format)',
+        cors(),
+        userMiddleware,
+        this.prepareContext,
+        this.tile.bind(this)
+    );
 
-    app.get(app.base_url_mapconfig +
-        '/static/named/:template_id/:width/:height.:format', cors(), userMiddleware, allowQueryParams(['layer']),
-        this.staticMap.bind(this));
+    app.get(
+        app.base_url_mapconfig + '/static/named/:template_id/:width/:height.:format',
+        cors(),
+        userMiddleware,
+        allowQueryParams(['layer', 'zoom', 'lon', 'lat', 'bbox']),
+        this.prepareContext,
+        this.staticMap.bind(this)
+    );
 };
 
 NamedMapsController.prototype.sendResponse = function(req, res, resource, headers, namedMapProvider) {
-    this.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(req.context.user, namedMapProvider.getTemplateName()));
+    this.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(res.locals.user, namedMapProvider.getTemplateName()));
     res.set('Content-Type', headers['content-type'] || headers['Content-Type'] || 'image/png');
     res.set('Cache-Control', 'public,max-age=7200,must-revalidate');
 
@@ -74,24 +84,20 @@ NamedMapsController.prototype.sendResponse = function(req, res, resource, header
     );
 };
 
-NamedMapsController.prototype.tile = function(req, res) {
+NamedMapsController.prototype.tile = function(req, res, next) {
     var self = this;
 
-    var cdbUser = req.context.user;
+    var cdbUser = res.locals.user;
 
     var namedMapProvider;
     step(
-        function reqParams() {
-            self.req2params(req, this);
-        },
-        function getNamedMapProvider(err) {
-            assert.ifError(err);
+        function getNamedMapProvider() {
             self.namedMapProviderCache.get(
                 cdbUser,
                 req.params.template_id,
                 req.query.config,
                 req.query.auth_token,
-                req.params,
+                res.locals,
                 this
             );
         },
@@ -103,7 +109,8 @@ NamedMapsController.prototype.tile = function(req, res) {
         function handleImage(err, tile, headers, stats) {
             req.profiler.add(stats);
             if (err) {
-                self.sendError(req, res, err, 'NAMED_MAP_TILE');
+                err.label = 'NAMED_MAP_TILE';
+                next(err);
             } else {
                 self.sendResponse(req, res, tile, headers, namedMapProvider);
             }
@@ -111,28 +118,24 @@ NamedMapsController.prototype.tile = function(req, res) {
     );
 };
 
-NamedMapsController.prototype.staticMap = function(req, res) {
+NamedMapsController.prototype.staticMap = function(req, res, next) {
     var self = this;
 
-    var cdbUser = req.context.user;
+    var cdbUser = res.locals.user;
 
     var format = req.params.format === 'jpg' ? 'jpeg' : 'png';
-    req.params.format = 'png';
-    req.params.layer = 'all';
+    res.locals.format = req.params.format || 'png';
+    res.locals.layer = res.locals.layer || 'all';
 
     var namedMapProvider;
     step(
-        function reqParams() {
-            self.req2params(req, this);
-        },
-        function getNamedMapProvider(err) {
-            assert.ifError(err);
+        function getNamedMapProvider() {
             self.namedMapProviderCache.get(
                 cdbUser,
                 req.params.template_id,
                 req.query.config,
                 req.query.auth_token,
-                req.params,
+                res.locals,
                 this
             );
         },
@@ -140,12 +143,12 @@ NamedMapsController.prototype.staticMap = function(req, res) {
             assert.ifError(err);
 
             namedMapProvider = _namedMapProvider;
-
-            self.prepareLayerFilterFromPreviewLayers(cdbUser, req, namedMapProvider, this);
+ 
+            self.prepareLayerFilterFromPreviewLayers(cdbUser, req, res.locals, namedMapProvider, this);
         },
         function prepareImageOptions(err) {
             assert.ifError(err);
-            self.getStaticImageOptions(cdbUser, req.params, namedMapProvider, this);
+            self.getStaticImageOptions(cdbUser, res.locals, namedMapProvider, this);
         },
         function getImage(err, imageOpts) {
             assert.ifError(err);
@@ -179,7 +182,8 @@ NamedMapsController.prototype.staticMap = function(req, res) {
             req.profiler.add(stats || {});
 
             if (err) {
-                self.sendError(req, res, err, 'STATIC_VIZ_MAP');
+                err.label = 'STATIC_VIZ_MAP';
+                next(err);
             } else {
                 self.sendResponse(req, res, image, headers, namedMapProvider);
             }
@@ -187,7 +191,13 @@ NamedMapsController.prototype.staticMap = function(req, res) {
     );
 };
 
-NamedMapsController.prototype.prepareLayerFilterFromPreviewLayers = function (user, req, namedMapProvider, callback) {
+NamedMapsController.prototype.prepareLayerFilterFromPreviewLayers = function (
+    user, 
+    req, 
+    params, 
+    namedMapProvider, 
+    callback
+) {
     var self = this;
     namedMapProvider.getTemplate(function (err, template) {
         if (err) {
@@ -212,7 +222,7 @@ NamedMapsController.prototype.prepareLayerFilterFromPreviewLayers = function (us
         }
 
         // overwrites 'all' default filter
-        req.params.layer = layerVisibilityFilter.join(',');
+        params.layer = layerVisibilityFilter.join(',');
 
         // recreates the provider
         self.namedMapProviderCache.get(
@@ -220,7 +230,7 @@ NamedMapsController.prototype.prepareLayerFilterFromPreviewLayers = function (us
             req.params.template_id,
             req.query.config,
             req.query.auth_token,
-            req.params,
+            params,
             callback
         );
     });

lib/cartodb/controllers/named_maps_admin.js

@@ -15,8 +15,8 @@ var userMiddleware = require('../middleware/user');
  * @param {TemplateMaps} templateMaps
  * @constructor
  */
-function NamedMapsAdminController(authApi, pgConnection, templateMaps) {
-    BaseController.call(this, authApi, pgConnection);
+function NamedMapsAdminController(authApi, templateMaps) {
+    BaseController.call(this);
 
     this.authApi = authApi;
     this.templateMaps = templateMaps;
@@ -26,19 +26,52 @@ util.inherits(NamedMapsAdminController, BaseController);
 
 module.exports = NamedMapsAdminController;
 
-NamedMapsAdminController.prototype.register = function(app) {
-    app.post(app.base_url_templated, cors(), userMiddleware, this.create.bind(this));
-    app.put(app.base_url_templated + '/:template_id', cors(), userMiddleware, this.update.bind(this));
-    app.get(app.base_url_templated + '/:template_id', cors(), userMiddleware, this.retrieve.bind(this));
-    app.delete(app.base_url_templated + '/:template_id', cors(), userMiddleware, this.destroy.bind(this));
-    app.get(app.base_url_templated, cors(), userMiddleware, this.list.bind(this));
-    app.options(app.base_url_templated + '/:template_id', cors('Content-Type'));
+NamedMapsAdminController.prototype.register = function (app) {
+    app.post(
+        app.base_url_templated + '/',
+        cors(),
+        userMiddleware,
+        this.create.bind(this)
+    );
+
+    app.put(
+        app.base_url_templated + '/:template_id',
+        cors(),
+        userMiddleware,
+        this.update.bind(this)
+    );
+
+    app.get(
+        app.base_url_templated + '/:template_id',
+        cors(),
+        userMiddleware,
+        this.retrieve.bind(this)
+    );
+
+    app.delete(
+        app.base_url_templated + '/:template_id',
+        cors(),
+        userMiddleware,
+        this.destroy.bind(this)
+    );
+
+    app.get(
+        app.base_url_templated + '/',
+        cors(),
+        userMiddleware,
+        this.list.bind(this)
+    );
+
+    app.options(
+        app.base_url_templated + '/:template_id',
+        cors('Content-Type')
+    );
 };
 
-NamedMapsAdminController.prototype.create = function(req, res) {
+NamedMapsAdminController.prototype.create = function(req, res, next) {
     var self = this;
 
-    var cdbuser = req.context.user;
+    var cdbuser = res.locals.user;
 
     step(
         function checkPerms(){
@@ -55,14 +88,14 @@ NamedMapsAdminController.prototype.create = function(req, res) {
             assert.ifError(err);
             return { template_id: tpl_id };
         },
-        finishFn(self, req, res, 'POST TEMPLATE')
+        finishFn(self, req, res, 'POST TEMPLATE', null, next)
     );
 };
 
-NamedMapsAdminController.prototype.update = function(req, res) {
+NamedMapsAdminController.prototype.update = function(req, res, next) {
     var self = this;
 
-    var cdbuser = req.context.user;
+    var cdbuser = res.locals.user;
     var template;
     var tpl_id;
 
@@ -84,16 +117,16 @@ NamedMapsAdminController.prototype.update = function(req, res) {
 
             return { template_id: tpl_id };
         },
-        finishFn(self, req, res, 'PUT TEMPLATE')
+        finishFn(self, req, res, 'PUT TEMPLATE', null, next)
     );
 };
 
-NamedMapsAdminController.prototype.retrieve = function(req, res) {
+NamedMapsAdminController.prototype.retrieve = function(req, res, next) {
     var self = this;
 
     req.profiler.start('windshaft-cartodb.get_template');
 
-    var cdbuser = req.context.user;
+    var cdbuser = res.locals.user;
     var tpl_id;
     step(
         function checkPerms(){
@@ -118,16 +151,16 @@ NamedMapsAdminController.prototype.retrieve = function(req, res) {
             delete tpl_val.auth_id;
             return { template: tpl_val };
         },
-        finishFn(self, req, res, 'GET TEMPLATE')
+        finishFn(self, req, res, 'GET TEMPLATE', null, next)
     );
 };
 
-NamedMapsAdminController.prototype.destroy = function(req, res) {
+NamedMapsAdminController.prototype.destroy = function(req, res, next) {
     var self = this;
 
     req.profiler.start('windshaft-cartodb.delete_template');
 
-    var cdbuser = req.context.user;
+    var cdbuser = res.locals.user;
     var tpl_id;
     step(
         function checkPerms(){
@@ -144,15 +177,15 @@ NamedMapsAdminController.prototype.destroy = function(req, res) {
             assert.ifError(err);
             return '';
         },
-        finishFn(self, req, res, 'DELETE TEMPLATE', 204)
+        finishFn(self, req, res, 'DELETE TEMPLATE', 204, next)
     );
 };
 
-NamedMapsAdminController.prototype.list = function(req, res) {
+NamedMapsAdminController.prototype.list = function(req, res, next) {
     var self = this;
     req.profiler.start('windshaft-cartodb.get_template_list');
 
-    var cdbuser = req.context.user;
+    var cdbuser = res.locals.user;
 
     step(
         function checkPerms(){
@@ -168,14 +201,15 @@ NamedMapsAdminController.prototype.list = function(req, res) {
             assert.ifError(err);
             return { template_ids: tpl_ids };
         },
-        finishFn(self, req, res, 'GET TEMPLATE LIST')
+        finishFn(self, req, res, 'GET TEMPLATE LIST', null, next)
     );
 };
 
-function finishFn(controller, req, res, description, status) {
+function finishFn(controller, req, res, description, status, next) {
     return function finish(err, response){
         if (err) {
-            controller.sendError(req, res, err, description);
+            err.label = description;
+            next(err);
         } else {
             controller.send(req, res, response, status || 200);
         }

lib/cartodb/middleware/allow-query-params.js

@@ -3,7 +3,7 @@ module.exports = function allowQueryParams(params) {
         throw new Error('allowQueryParams must receive an Array of params');
     }
     return function allowQueryParamsMiddleware(req, res, next) {
-        req.context.allowedQueryParams = params;
+        res.locals.allowedQueryParams = params;
         next();
     };
 };

lib/cartodb/middleware/context/authorize.js

@@ -0,0 +1,20 @@
+module.exports = function authorizeMiddleware (authApi) {
+    return function (req, res, next) {
+        req.profiler.done('req2params.setup');
+
+        authApi.authorize(req, res, (err, authorized) => {
+            req.profiler.done('authorize');
+            if (err) {
+                return next(err);
+            }
+
+            if(!authorized) {
+                err = new Error("Sorry, you are unauthorized (permission denied)");
+                err.http_status = 403;
+                return next(err);
+            }
+
+            return next();
+        });
+    };
+};

lib/cartodb/middleware/context/clean-up-query-params.js

@@ -0,0 +1,32 @@
+const _ = require('underscore');
+
+// Whitelist query parameters and attach format
+const REQUEST_QUERY_PARAMS_WHITELIST = [
+    'config',
+    'map_key',
+    'api_key',
+    'auth_token',
+    'callback',
+    'zoom',
+    'lon',
+    'lat',
+    // analysis
+    'filters' // json
+];
+
+module.exports = function cleanUpQueryParamsMiddleware () {
+    return function cleanUpQueryParams (req, res, next) {
+        var allowedQueryParams = REQUEST_QUERY_PARAMS_WHITELIST;
+
+        if (Array.isArray(res.locals.allowedQueryParams)) {
+            allowedQueryParams = allowedQueryParams.concat(res.locals.allowedQueryParams);
+        }
+
+        req.query = _.pick(req.query, allowedQueryParams);
+
+        // bring all query values onto res.locals object
+        _.extend(res.locals, req.query);
+        
+        next();
+    };
+};

lib/cartodb/middleware/context/db-conn-setup.js

@@ -0,0 +1,31 @@
+const _ = require('underscore');
+
+module.exports = function dbConnSetupMiddleware(pgConnection) {
+    return function dbConnSetup(req, res, next) {
+        const user = res.locals.user;
+        pgConnection.setDBConn(user, res.locals, (err) => {
+            if (err) {
+                if (err.message && -1 !== err.message.indexOf('name not found')) {
+                    err.http_status = 404;
+                }
+                req.profiler.done('req2params');
+                return next(err);
+            }
+
+            // Add default database connection parameters
+            // if none given
+            _.defaults(res.locals, {
+                dbuser: global.environment.postgres.user,
+                dbpassword: global.environment.postgres.password,
+                dbhost: global.environment.postgres.host,
+                dbport: global.environment.postgres.port
+            });
+            
+            res.set('X-Served-By-DB-Host', res.locals.dbhost);
+
+            req.profiler.done('req2params');
+            
+            next(null);
+        });
+    };
+};

lib/cartodb/middleware/context/index.js

@@ -0,0 +1,15 @@
+const locals = require('./locals');
+const cleanUpQueryParams = require('./clean-up-query-params');
+const layergroupToken = require('./layergroup-token');
+const authorize = require('./authorize');
+const dbConnSetup = require('./db-conn-setup');
+
+module.exports = function prepareContextMiddleware(authApi, pgConnection) {
+    return [
+        locals,
+        cleanUpQueryParams(),
+        layergroupToken,
+        authorize(authApi),
+        dbConnSetup(pgConnection)
+    ];
+};

lib/cartodb/middleware/context/layergroup-token.js

@@ -0,0 +1,32 @@
+var LayergroupToken = require('../../models/layergroup-token');
+
+module.exports = function layergroupTokenMiddleware(req, res, next) {
+    if (!res.locals.token) {
+        return next();
+    }
+
+    var user = res.locals.user;
+
+    var layergroupToken = LayergroupToken.parse(res.locals.token);
+    res.locals.token = layergroupToken.token;
+    res.locals.cache_buster = layergroupToken.cacheBuster;
+
+    if (layergroupToken.signer) {
+        res.locals.signer = layergroupToken.signer;
+        if (!res.locals.signer) {
+            res.locals.signer = user;
+        } else if (res.locals.signer !== user) {
+            var err = new Error(`Cannot use map signature of user "${res.locals.signer}" on db of user "${user}"`);
+            err.type = 'auth';
+            err.http_status = 403;
+            if (req.query && req.query.callback) {
+                err.http_status = 200;
+            }
+            
+            req.profiler.done('req2params');
+            return next(err);
+        }
+    }
+
+    return next();
+};

lib/cartodb/middleware/context/locals.js

@@ -0,0 +1,8 @@
+const _ = require('underscore');
+
+module.exports = function localsMiddleware(req, res, next) {
+    _.extend(res.locals, req.params);
+    
+    next();
+};
+

lib/cartodb/middleware/error-middleware.js

@@ -0,0 +1,162 @@
+const _ = require('underscore');
+const debug = require('debug')('windshaft:cartodb:error-middleware');
+
+module.exports = function errorMiddleware (/* options */) {
+    return function error (err, req, res, next) {
+        // jshint unused:false
+        // jshint maxcomplexity:9
+        var allErrors = Array.isArray(err) ? err : [err];
+
+        allErrors = populateTimeoutErrors(allErrors);
+
+        const label = err.label || 'UNKNOWN';
+        err = allErrors[0] || new Error(label);
+        allErrors[0] = err;
+
+        var statusCode = findStatusCode(err);
+
+        if (err.message === 'Tile does not exist' && res.locals.format === 'mvt') {
+            statusCode = 204;
+        }
+
+        debug('[%s ERROR] -- %d: %s, %s', label, statusCode, err, err.stack);
+
+        // If a callback was requested, force status to 200
+        if (req.query && req.query.callback) {
+            statusCode = 200;
+        }
+
+        var errorResponseBody = {
+            errors: allErrors.map(errorMessage),
+            errors_with_context: allErrors.map(errorMessageWithContext)
+        };
+
+        res.status(statusCode);
+
+        if (req.query && req.query.callback) {
+            res.jsonp(errorResponseBody);
+        } else {
+            res.json(errorResponseBody);
+        }
+    };
+};
+
+function isRenderTimeoutError (err) {
+    return err.message === 'Render timed out';
+}
+
+function isDatasourceTimeoutError (err) {
+    return err.message && err.message.match(/canceling statement due to statement timeout/i);
+}
+
+function isTimeoutError (err) {
+    return isRenderTimeoutError(err) || isDatasourceTimeoutError(err);
+}
+
+function populateTimeoutErrors (errors) {
+    return errors.map(function (error) {
+        if (isRenderTimeoutError(error)) {
+            error.subtype = 'render';
+        }
+
+        if (isDatasourceTimeoutError(error)) {
+            error.subtype = 'datasource';
+        }
+
+        if (isTimeoutError(error)) {
+            error.message = 'You are over platform\'s limits. Please contact us to know more details';
+            error.type = 'limit';
+            error.http_status = 429;
+        }
+
+        return error;
+    });
+}
+
+function findStatusCode(err) {
+    var statusCode;
+    if ( err.http_status ) {
+        statusCode = err.http_status;
+    } else {
+        statusCode = statusFromErrorMessage('' + err);
+    }
+    return statusCode;
+}
+
+module.exports.findStatusCode = findStatusCode;
+
+function statusFromErrorMessage(errMsg) {
+    // Find an appropriate statusCode based on message
+    // jshint maxcomplexity:7
+    var statusCode = 400;
+    if ( -1 !== errMsg.indexOf('permission denied') ) {
+        statusCode = 403;
+    }
+    else if ( -1 !== errMsg.indexOf('authentication failed') ) {
+        statusCode = 403;
+    }
+    else if (errMsg.match(/Postgis Plugin.*[\s|\n].*column.*does not exist/)) {
+        statusCode = 400;
+    }
+    else if ( -1 !== errMsg.indexOf('does not exist') ) {
+        if ( -1 !== errMsg.indexOf(' role ') ) {
+            statusCode = 403; // role 'xxx' does not exist
+        } else if ( errMsg.match(/function .* does not exist/) ) {
+            statusCode = 400; // invalid SQL (SQL function does not exist)
+        } else {
+            statusCode = 404;
+        }
+    }
+
+
+    return statusCode;
+}
+
+function errorMessage(err) {
+    // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
+    var message = (_.isString(err) ? err : err.message) || 'Unknown error';
+
+    return stripConnectionInfo(message);
+}
+
+module.exports.errorMessage = errorMessage;
+
+function stripConnectionInfo(message) {
+    // Strip connection info, if any
+    return message
+        // See https://github.com/CartoDB/Windshaft/issues/173
+        .replace(/Connection string: '[^']*'\n\s/im, '')
+        // See https://travis-ci.org/CartoDB/Windshaft/jobs/20703062#L1644
+        .replace(/is the server.*encountered/im, 'encountered');
+}
+
+var ERROR_INFO_TO_EXPOSE = {
+    message: true,
+    layer: true,
+    type: true,
+    analysis: true,
+    subtype: true
+};
+
+function shouldBeExposed (prop) {
+    return !!ERROR_INFO_TO_EXPOSE[prop];
+}
+
+function errorMessageWithContext(err) {
+    // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
+    var message = (_.isString(err) ? err : err.message) || 'Unknown error';
+
+    var error = {
+        type: err.type || 'unknown',
+        message: stripConnectionInfo(message),
+    };
+
+    for (var prop in err) {
+        // type & message are properties from Error's prototype and will be skipped
+        if (err.hasOwnProperty(prop) && shouldBeExposed(prop)) {
+            error[prop] = err[prop];
+        }
+    }
+
+    return error;
+}

lib/cartodb/middleware/lzma.js

@@ -0,0 +1,30 @@
+'use strict';
+
+const LZMA = require('lzma').LZMA;
+
+const lzmaWorker = new LZMA();
+
+module.exports = function lzmaMiddleware(req, res, next) {
+    if (!req.query.hasOwnProperty('lzma')) {
+        return next();
+    }
+
+    // Decode (from base64)
+    var lzma = new Buffer(req.query.lzma, 'base64')
+        .toString('binary')
+        .split('')
+        .map(function(c) {
+            return c.charCodeAt(0) - 128;
+        });
+
+    // Decompress
+    lzmaWorker.decompress(lzma, function(result) {
+        try {
+            delete req.query.lzma;
+            Object.assign(req.query, JSON.parse(result));
+            next();
+        } catch (err) {
+            next(new Error('Error parsing lzma as JSON: ' + err));
+        }
+    });
+};

lib/cartodb/middleware/stats.js

@@ -0,0 +1,27 @@
+const Profiler = require('../stats/profiler_proxy');
+const debug = require('debug')('windshaft:cartodb:stats');
+const onHeaders = require('on-headers');
+
+module.exports = function statsMiddleware(options) {
+    const { enabled = true, statsClient } = options;
+
+    return function stats(req, res, next) {
+        req.profiler = new Profiler({
+            statsd_client: statsClient,
+            profile: enabled
+        });
+
+        onHeaders(res, () => res.set('X-Tiler-Profiler', req.profiler.toJSONString()));
+
+        res.on('finish', () => {
+            try {
+                // May throw due to dns, see: http://github.com/CartoDB/Windshaft/issues/166
+                req.profiler.sendStats();
+            } catch (err) {
+                debug("error sending profiling stats: " + err);
+            }
+        });
+
+        next();
+    };
+};

lib/cartodb/middleware/user.js

@@ -2,6 +2,13 @@ var CdbRequest = require('../models/cdb_request');
 var cdbRequest = new CdbRequest();
 
 module.exports = function userMiddleware(req, res, next) {
-    req.context.user = cdbRequest.userByReq(req);
+    res.locals.user = cdbRequest.userByReq(req);
+
+    // avoid a req.params.user equals to undefined
+    // overwrites res.locals.user
+    if(!req.params.user) {
+        delete req.params.user;
+    }
+
     next();
 };

lib/cartodb/models/dataview/aggregation.js

@@ -1,71 +1,178 @@
-var _ = require('underscore');
-var BaseWidget = require('./base');
-var debug = require('debug')('windshaft:widget:aggregation');
+const BaseDataview = require('./base');
+const debug = require('debug')('windshaft:dataview:aggregation');
 
-var dot = require('dot');
-dot.templateSettings.strip = false;
+const filteredQueryTpl = ctx => `
+    filtered_source AS (
+        SELECT *
+        FROM (${ctx.query}) _cdb_filtered_source
+        ${ctx.aggregationColumn && ctx.isFloatColumn ? `
+        WHERE
+            ${ctx.aggregationColumn} != 'infinity'::float
+        AND
+            ${ctx.aggregationColumn} != '-infinity'::float
+        AND
+            ${ctx.aggregationColumn} != 'NaN'::float` :
+        ''
+        }
+    )
+`;
 
-var summaryQueryTpl = dot.template([
-    'summary AS (',
-    '  SELECT',
-    '  count(1) AS count,',
-    '  sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count',
-    '  FROM ({{=it._query}}) _cdb_aggregation_nulls',
-    ')'
-].join('\n'));
+const summaryQueryTpl = ctx => `
+    summary AS (
+        SELECT
+            count(1) AS count,
+            sum(CASE WHEN ${ctx.column} IS NULL THEN 1 ELSE 0 END) AS nulls_count
+            ${ctx.isFloatColumn ? `,
+            sum(
+                CASE
+                    WHEN ${ctx.aggregationColumn} = 'infinity'::float OR ${ctx.aggregationColumn} = '-infinity'::float
+                    THEN 1
+                    ELSE 0
+                    END
+            ) AS infinities_count,
+            sum(CASE WHEN ${ctx.aggregationColumn} = 'NaN'::float THEN 1 ELSE 0 END) AS nans_count` :
+            ''
+            }
+        FROM (${ctx.query}) _cdb_aggregation_nulls
+    )
+`;
 
-var rankedCategoriesQueryTpl = dot.template([
-    'categories AS(',
-    '  SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,',
-    '    row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank',
-    '  FROM ({{=it._query}}) _cdb_aggregation_all',
-    '  {{?it._aggregationColumn!==null}}WHERE {{=it._aggregationColumn}} IS NOT NULL{{?}}',
-    '  GROUP BY {{=it._column}}',
-    '  ORDER BY 2 DESC',
-    ')'
-].join('\n'));
+const rankedCategoriesQueryTpl = ctx => `
+    categories AS(
+        SELECT
+            ${ctx.column} AS category,
+            ${ctx.aggregationFn} AS value,
+            row_number() OVER (ORDER BY ${ctx.aggregationFn} desc) as rank
+        FROM filtered_source
+        ${ctx.aggregationColumn !== null ? `WHERE ${ctx.aggregationColumn} IS NOT NULL` : ''}
+        GROUP BY ${ctx.column}
+        ORDER BY 2 DESC
+    )
+`;
 
-var categoriesSummaryMinMaxQueryTpl = dot.template([
-    'categories_summary_min_max AS(',
-    '  SELECT max(value) max_val, min(value) min_val',
-    '  FROM categories',
-    ')'
-].join('\n'));
+const categoriesSummaryMinMaxQueryTpl = () => `
+    categories_summary_min_max AS(
+        SELECT
+            max(value) max_val,
+            min(value) min_val
+        FROM categories
+    )
+`;
 
-var categoriesSummaryCountQueryTpl = dot.template([
-    'categories_summary_count AS(',
-    '  SELECT count(1) AS categories_count',
-    '  FROM (',
-    '    SELECT {{=it._column}} AS category',
-    '    FROM ({{=it._query}}) _cdb_categories',
-    '    GROUP BY {{=it._column}}',
-    '  ) _cdb_categories_count',
-    ')'
-].join('\n'));
+const categoriesSummaryCountQueryTpl = ctx => `
+    categories_summary_count AS(
+        SELECT count(1) AS categories_count
+        FROM (
+            SELECT ${ctx.column} AS category
+            FROM (${ctx.query}) _cdb_categories
+            GROUP BY ${ctx.column}
+        ) _cdb_categories_count
+    )
+`;
 
-var rankedAggregationQueryTpl = dot.template([
-    'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count',
-    '  FROM categories, summary, categories_summary_min_max, categories_summary_count',
-    '  WHERE rank < {{=it._limit}}',
-    'UNION ALL',
-    'SELECT \'Other\' category, {{=it._aggregationFn}}(value) as value, true as agg, nulls_count, min_val, max_val,',
-    '       count, categories_count',
-    '  FROM categories, summary, categories_summary_min_max, categories_summary_count',
-    '  WHERE rank >= {{=it._limit}}',
-    'GROUP BY nulls_count, min_val, max_val, count, categories_count'
-].join('\n'));
+const specialNumericValuesColumns = () => `, nans_count, infinities_count`;
 
-var aggregationQueryTpl = dot.template([
-    'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,',
-    '  nulls_count, min_val, max_val, count, categories_count',
-    'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count',
-    'GROUP BY category, nulls_count, min_val, max_val, count, categories_count',
-    'ORDER BY value DESC'
-].join('\n'));
+const rankedAggregationQueryTpl = ctx => `
+    SELECT
+        CAST(category AS text),
+        value,
+        false as agg,
+        nulls_count,
+        min_val,
+        max_val,
+        count,
+        categories_count
+        ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' }
+        FROM categories, summary, categories_summary_min_max, categories_summary_count
+        WHERE rank < ${ctx.limit}
+    UNION ALL
+    SELECT
+        'Other' category,
+        ${ctx.aggregation !== 'count' ? ctx.aggregation : 'sum'}(value) as value,
+        true as agg,
+        nulls_count,
+        min_val,
+        max_val,
+        count,
+        categories_count
+        ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' }
+        FROM categories, summary, categories_summary_min_max, categories_summary_count
+        WHERE rank >= ${ctx.limit}
+    GROUP BY
+        nulls_count,
+        min_val,
+        max_val,
+        count,
+        categories_count
+        ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' }
+`;
 
-var CATEGORIES_LIMIT = 6;
+const aggregationQueryTpl = ctx => `
+    SELECT
+        CAST(${ctx.column} AS text) AS category,
+        ${ctx.aggregationFn} AS value,
+        false as agg,
+        nulls_count,
+        min_val,
+        max_val,
+        count,
+        categories_count
+        ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' }
+    FROM (${ctx.query}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count
+    GROUP BY
+        category,
+        nulls_count,
+        min_val,
+        max_val,
+        count,
+        categories_count
+        ${ctx.isFloatColumn ? `${specialNumericValuesColumns(ctx)}` : '' }
+    ORDER BY value DESC
+`;
 
-var VALID_OPERATIONS = {
+const aggregationFnQueryTpl = ctx => `${ctx.aggregation}(${ctx.aggregationColumn})`;
+
+const aggregationDataviewQueryTpl = ctx => `
+    WITH
+    ${filteredQueryTpl(ctx)},
+    ${summaryQueryTpl(ctx)},
+    ${rankedCategoriesQueryTpl(ctx)},
+    ${categoriesSummaryMinMaxQueryTpl(ctx)},
+    ${categoriesSummaryCountQueryTpl(ctx)}
+    ${!!ctx.override.ownFilter ? `${aggregationQueryTpl(ctx)}` : `${rankedAggregationQueryTpl(ctx)}`}
+`;
+
+const filterCategoriesQueryTpl = ctx => `
+    SELECT
+        ${ctx.column} AS category,
+        ${ctx.value} AS value
+    FROM (${ctx.query}) _cdb_aggregation_search
+    WHERE CAST(${ctx.column} as text) ILIKE ${ctx.userQuery}
+    GROUP BY ${ctx.column}
+`;
+
+const searchQueryTpl = ctx => `
+    WITH
+    search_unfiltered AS (
+        ${ctx.searchUnfiltered}
+    ),
+    search_filtered AS (
+        ${ctx.searchFiltered}
+    ),
+    search_union AS (
+        SELECT * FROM search_unfiltered
+        UNION ALL
+        SELECT * FROM search_filtered
+    )
+    SELECT category, sum(value) AS value
+    FROM search_union
+    GROUP BY category
+    ORDER BY value desc
+`;
+
+const CATEGORIES_LIMIT = 6;
+
+const VALID_OPERATIONS = {
     count: [],
     sum: ['aggregationColumn'],
     avg: ['aggregationColumn'],
@@ -73,7 +180,7 @@ var VALID_OPERATIONS = {
     max: ['aggregationColumn']
 };
 
-var TYPE = 'aggregation';
+const TYPE = 'aggregation';
 
 /**
  {
@@ -84,210 +191,150 @@ var TYPE = 'aggregation';
      }
  }
  */
-function Aggregation(query, options) {
-    if (!_.isString(options.column)) {
-        throw new Error('Aggregation expects `column` in widget options');
+module.exports = class Aggregation extends BaseDataview {
+    constructor (query, options = {}, queries = {}) {
+        super();
+
+        this._checkOptions(options);
+
+        this.query = query;
+        this.queries = queries;
+        this.column = options.column;
+        this.aggregation = options.aggregation;
+        this.aggregationColumn = options.aggregationColumn;
+        this._isFloatColumn = null;
     }
 
-    if (!_.isString(options.aggregation)) {
-        throw new Error('Aggregation expects `aggregation` operation in widget options');
-    }
-
-    if (!VALID_OPERATIONS[options.aggregation]) {
-        throw new Error("Aggregation does not support '" + options.aggregation + "' operation");
-    }
-
-    var requiredOptions = VALID_OPERATIONS[options.aggregation];
-    var missingOptions = _.difference(requiredOptions, Object.keys(options));
-    if (missingOptions.length > 0) {
-        throw new Error(
-            "Aggregation '" + options.aggregation + "' is missing some options: " + missingOptions.join(',')
-        );
-    }
-
-    BaseWidget.apply(this);
-
-    this.query = query;
-    this.column = options.column;
-    this.aggregation = options.aggregation;
-    this.aggregationColumn = options.aggregationColumn;
-}
-
-Aggregation.prototype = new BaseWidget();
-Aggregation.prototype.constructor = Aggregation;
-
-module.exports = Aggregation;
-
-Aggregation.prototype.sql = function(psql, override, callback) {
-    if (!callback) {
-        callback = override;
-        override = {};
-    }
-
-    var _query = this.query;
-
-    var aggregationSql;
-
-    if (!!override.ownFilter) {
-        aggregationSql = [
-            this.getCategoriesCTESql(_query, this.column, this.aggregation, this.aggregationColumn),
-            aggregationQueryTpl({
-                _query: _query,
-                _column: this.column,
-                _aggregation: this.getAggregationSql(),
-                _limit: CATEGORIES_LIMIT
-            })
-        ].join('\n');
-    } else {
-        aggregationSql = [
-            this.getCategoriesCTESql(_query, this.column, this.aggregation, this.aggregationColumn),
-            rankedAggregationQueryTpl({
-                _query: _query,
-                _column: this.column,
-                _aggregationFn: this.aggregation !== 'count' ? this.aggregation : 'sum',
-                _limit: CATEGORIES_LIMIT
-            })
-        ].join('\n');
-    }
-
-    debug(aggregationSql);
-
-    return callback(null, aggregationSql);
-};
-
-Aggregation.prototype.getCategoriesCTESql = function(query, column, aggregation, aggregationColumn) {
-    return [
-                "WITH",
-                [
-                    summaryQueryTpl({
-                        _query: query,
-                        _column: column
-                    }),
-                    rankedCategoriesQueryTpl({
-                        _query: query,
-                        _column: column,
-                        _aggregation: this.getAggregationSql(),
-                        _aggregationColumn: aggregation !== 'count' ? aggregationColumn : null
-                    }),
-                    categoriesSummaryMinMaxQueryTpl({
-                        _query: query,
-                        _column: column
-                    }),
-                    categoriesSummaryCountQueryTpl({
-                        _query: query,
-                        _column: column
-                    })
-                ].join(',\n')
-            ].join('\n');
-};
-
-var aggregationFnQueryTpl = dot.template('{{=it._aggregationFn}}({{=it._aggregationColumn}})');
-Aggregation.prototype.getAggregationSql = function() {
-    return aggregationFnQueryTpl({
-        _aggregationFn: this.aggregation,
-        _aggregationColumn: this.aggregationColumn || 1
-    });
-};
-
-Aggregation.prototype.format = function(result) {
-    var categories = [];
-    var count = 0;
-    var nulls = 0;
-    var minValue = 0;
-    var maxValue = 0;
-    var categoriesCount = 0;
-
-
-    if (result.rows.length) {
-        var firstRow = result.rows[0];
-        count = firstRow.count;
-        nulls = firstRow.nulls_count;
-        minValue = firstRow.min_val;
-        maxValue = firstRow.max_val;
-        categoriesCount = firstRow.categories_count;
-
-        result.rows.forEach(function(row) {
-            categories.push(_.omit(row, 'count', 'nulls_count', 'min_val', 'max_val', 'categories_count'));
-        });
-    }
-
-    return {
-        aggregation: this.aggregation,
-        count: count,
-        nulls: nulls,
-        min: minValue,
-        max: maxValue,
-        categoriesCount: categoriesCount,
-        categories: categories
-    };
-};
-
-var filterCategoriesQueryTpl = dot.template([
-    'SELECT {{=it._column}} AS category, {{=it._value}} AS value',
-    'FROM ({{=it._query}}) _cdb_aggregation_search',
-    'WHERE CAST({{=it._column}} as text) ILIKE {{=it._userQuery}}',
-    'GROUP BY {{=it._column}}'
-].join('\n'));
-
-var searchQueryTpl = dot.template([
-    'WITH',
-    'search_unfiltered AS (',
-    '  {{=it._searchUnfiltered}}',
-    '),',
-    'search_filtered AS (',
-    '  {{=it._searchFiltered}}',
-    '),',
-    'search_union AS (',
-    '  SELECT * FROM search_unfiltered',
-    '  UNION ALL',
-    '  SELECT * FROM search_filtered',
-    ')',
-    'SELECT category, sum(value) AS value',
-    'FROM search_union',
-    'GROUP BY category',
-    'ORDER BY value desc'
-].join('\n'));
-
-
-Aggregation.prototype.search = function(psql, userQuery, callback) {
-    var self = this;
-
-    var _userQuery = psql.escapeLiteral('%' + userQuery + '%');
-
-    // TODO unfiltered will be wrong as filters are already applied at this point
-    var query = searchQueryTpl({
-        _searchUnfiltered: filterCategoriesQueryTpl({
-            _query: this.query,
-            _column: this.column,
-            _value: '0',
-            _userQuery: _userQuery
-        }),
-        _searchFiltered: filterCategoriesQueryTpl({
-            _query: this.query,
-            _column: this.column,
-            _value: 'count(1)',
-            _userQuery: _userQuery
-        })
-    });
-
-    psql.query(query, function(err, result) {
-        if (err) {
-            return callback(err, result);
+    _checkOptions (options) {
+        if (typeof options.column !== 'string') {
+            throw new Error(`Aggregation expects 'column' in dataview options`);
         }
 
-        return callback(null, {type: self.getType(), categories: result.rows });
-    }, true); // use read-only transaction
-};
+        if (typeof options.aggregation !== 'string') {
+            throw new Error(`Aggregation expects 'aggregation' operation in dataview options`);
+        }
 
-Aggregation.prototype.getType = function() {
-    return TYPE;
-};
+        if (!VALID_OPERATIONS[options.aggregation]) {
+            throw new Error(`Aggregation does not support '${options.aggregation}' operation`);
+        }
 
-Aggregation.prototype.toString = function() {
-    return JSON.stringify({
-        _type: TYPE,
-        _query: this.query,
-        _column: this.column,
-        _aggregation: this.aggregation
-    });
+        const requiredOptions = VALID_OPERATIONS[options.aggregation];
+        const missingOptions = requiredOptions.filter(requiredOption => !options.hasOwnProperty(requiredOption));
+
+        if (missingOptions.length > 0) {
+            throw new Error(
+                `Aggregation '${options.aggregation}' is missing some options: ${missingOptions.join(',')}`
+            );
+        }
+    }
+
+    sql (psql, override, callback) {
+        if (!callback) {
+            callback = override;
+            override = {};
+        }
+
+        if (this._shouldCheckColumnType()) {
+            this._isFloatColumn = false;
+            this.getColumnType(psql, this.aggregationColumn, this.queries.no_filters, (err, type) => {
+                if (!err && !!type) {
+                    this._isFloatColumn = type.float;
+                }
+                this.sql(psql, override, callback);
+            });
+            return null;
+        }
+
+        const aggregationSql = aggregationDataviewQueryTpl({
+            override: override,
+            query: this.query,
+            column: this.column,
+            aggregation: this.aggregation,
+            aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null,
+            aggregationFn: aggregationFnQueryTpl({
+                aggregation: this.aggregation,
+                aggregationColumn: this.aggregationColumn || 1
+            }),
+            isFloatColumn: this._isFloatColumn,
+            limit: CATEGORIES_LIMIT
+        });
+
+        debug(aggregationSql);
+
+        return callback(null, aggregationSql);
+    }
+
+    _shouldCheckColumnType () {
+        return this.aggregationColumn && this._isFloatColumn === null;
+    }
+
+    format (result) {
+        const {
+            count = 0,
+            nulls_count = 0,
+            nans_count = 0,
+            infinities_count = 0,
+            min_val = 0,
+            max_val = 0,
+            categories_count = 0
+         } = result.rows[0] || {};
+
+        return {
+            aggregation: this.aggregation,
+            count: count,
+            nulls: nulls_count,
+            nans: nans_count,
+            infinities: infinities_count,
+            min: min_val,
+            max: max_val,
+            categoriesCount: categories_count,
+            categories: result.rows.map(({ category, value, agg }) => ({ category, value, agg }))
+        };
+    }
+
+    search (psql, userQuery, callback) {
+        const escapedUserQuery = psql.escapeLiteral(`%${userQuery}%`);
+        const value = this.aggregation !== 'count' && this.aggregationColumn ?
+            `${this.aggregation}(${this.aggregationColumn})` :
+            'count(1)';
+
+        // TODO unfiltered will be wrong as filters are already applied at this point
+        const query = searchQueryTpl({
+            searchUnfiltered: filterCategoriesQueryTpl({
+                query: this.query,
+                column: this.column,
+                value: '0',
+                userQuery: escapedUserQuery
+            }),
+            searchFiltered: filterCategoriesQueryTpl({
+                query: this.query,
+                column: this.column,
+                value: value,
+                userQuery: escapedUserQuery
+            })
+        });
+
+        debug(query);
+
+        psql.query(query, (err, result) => {
+            if (err) {
+                return callback(err, result);
+            }
+
+            return callback(null, {type: this.getType(), categories: result.rows });
+        }, true); // use read-only transaction
+    }
+
+    getType () {
+        return TYPE;
+    }
+
+    toString () {
+        return JSON.stringify({
+            _type: TYPE,
+            _query: this.query,
+            _column: this.column,
+            _aggregation: this.aggregation
+        });
+    }
 };

lib/cartodb/models/dataview/base.js

@@ -1,26 +1,59 @@
-function BaseDataview() {}
+const FLOAT_OIDS = {
+    700: true,
+    701: true,
+    1700: true
+};
 
-module.exports = BaseDataview;
+const DATE_OIDS = {
+    1082: true,
+    1114: true,
+    1184: true
+};
 
-BaseDataview.prototype.getResult = function(psql, override, callback) {
-    var self = this;
-    this.sql(psql, override, function(err, query) {
-        psql.query(query, function(err, result) {
+const columnTypeQueryTpl = ctx => `SELECT pg_typeof(${ctx.column})::oid FROM (${ctx.query}) _cdb_column_type limit 1`;
 
+function getPGTypeName (pgType) {
+    return {
+        float: FLOAT_OIDS.hasOwnProperty(pgType),
+        date: DATE_OIDS.hasOwnProperty(pgType)
+    };
+}
+
+module.exports = class BaseDataview {
+    getResult (psql, override, callback) {
+        this.sql(psql, override, (err, query) => {
             if (err) {
-                return callback(err, result);
+                return callback(err);
             }
 
-            result = self.format(result, override);
-            result.type = self.getType();
+            psql.query(query, (err, result) => {
+                if (err) {
+                    return callback(err, result);
+                }
 
-            return callback(null, result);
+                result = this.format(result, override);
+                result.type = this.getType();
 
-        }, true); // use read-only transaction
-    });
+                return callback(null, result);
 
-};
-
-BaseDataview.prototype.search = function(psql, userQuery, callback) {
-    return callback(null, this.format({ rows: [] }));
+            }, true); // use read-only transaction
+        });
+    }
+
+    search (psql, userQuery, callback) {
+        return callback(null, this.format({ rows: [] }));
+    }
+
+    getColumnType (psql, column, query, callback) {
+        const readOnlyTransaction = true;
+        const columnTypeQuery = columnTypeQueryTpl({ column, query });
+
+        psql.query(columnTypeQuery, (err, result) => {
+            if (err) {
+                return callback(err);
+            }
+            const pgType = result.rows[0].pg_typeof;
+            callback(null, getPGTypeName(pgType));
+        }, readOnlyTransaction);
+    }
 };

lib/cartodb/models/dataview/factory.js

@@ -1,18 +1,20 @@
-var dataviews = require('./');
+const dataviews = require('./');
 
-var DataviewFactory = {
-    dataviews: Object.keys(dataviews).reduce(function(allDataviews, dataviewClassName) {
-        allDataviews[dataviewClassName.toLowerCase()] = dataviews[dataviewClassName];
-        return allDataviews;
-    }, {}),
+module.exports = class DataviewFactory {
+    static get dataviews() {
+        return Object.keys(dataviews).reduce((allDataviews, dataviewClassName) => {
+            allDataviews[dataviewClassName.toLowerCase()] = dataviews[dataviewClassName];
+            return allDataviews;
+        }, {});
+    }
+
+    static getDataview (query, dataviewDefinition) {
+        const { type, options, sql } = dataviewDefinition;
 
-    getDataview: function(query, dataviewDefinition) {
-        var type = dataviewDefinition.type;
         if (!this.dataviews[type]) {
             throw new Error('Invalid dataview type: "' + type + '"');
         }
-        return new this.dataviews[type](query, dataviewDefinition.options, dataviewDefinition.sql);
+
+        return new this.dataviews[type](query, options, sql);
     }
 };
-
-module.exports = DataviewFactory;

lib/cartodb/models/dataview/formula.js

@@ -1,18 +1,36 @@
-var _ = require('underscore');
-var BaseWidget = require('./base');
-var debug = require('debug')('windshaft:widget:formula');
+const BaseDataview = require('./base');
+const debug = require('debug')('windshaft:dataview:formula');
 
-var dot = require('dot');
-dot.templateSettings.strip = false;
+const countInfinitiesQueryTpl = ctx => `
+    SELECT count(1) FROM (${ctx.query}) __cdb_formula_infinities
+    WHERE ${ctx.column} = 'infinity'::float OR ${ctx.column} = '-infinity'::float
+`;
 
-var formulaQueryTpl = dot.template([
-    'SELECT',
-    '{{=it._operation}}({{=it._column}}) AS result,',
-    '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
-    'FROM ({{=it._query}}) _cdb_formula'
-].join('\n'));
+const countNansQueryTpl = ctx => `
+    SELECT count(1) FROM (${ctx.query}) __cdb_formula_nans
+    WHERE ${ctx.column} = 'NaN'::float
+`;
 
-var VALID_OPERATIONS = {
+const filterOutSpecialNumericValuesTpl = ctx => `
+    WHERE
+        ${ctx.column} != 'infinity'::float
+    AND
+        ${ctx.column} != '-infinity'::float
+    AND
+        ${ctx.column} != 'NaN'::float
+`;
+
+const formulaQueryTpl = ctx => `
+    SELECT
+        ${ctx.operation}(${ctx.column}) AS result,
+        (SELECT count(1) FROM (${ctx.query}) _cdb_formula_nulls WHERE ${ctx.column} IS NULL) AS nulls_count
+        ${ctx.isFloatColumn ? `,(${countInfinitiesQueryTpl(ctx)}) AS infinities_count` : ''}
+        ${ctx.isFloatColumn ? `,(${countNansQueryTpl(ctx)}) AS nans_count` : ''}
+    FROM (${ctx.query}) __cdb_formula
+    ${ctx.isFloatColumn && ctx.operation !== 'count' ? `${filterOutSpecialNumericValuesTpl(ctx)}` : ''}
+`;
+
+const VALID_OPERATIONS = {
     count: true,
     avg: true,
     sum: true,
@@ -20,7 +38,7 @@ var VALID_OPERATIONS = {
     max: true
 };
 
-var TYPE = 'formula';
+const TYPE = 'formula';
 
 /**
  {
@@ -31,74 +49,90 @@ var TYPE = 'formula';
      }
  }
  */
-function Formula(query, options) {
-    if (!_.isString(options.operation)) {
-        throw new Error('Formula expects `operation` in widget options');
+module.exports = class Formula extends BaseDataview {
+    constructor (query, options = {}, queries = {}) {
+        super();
+
+        this._checkOptions(options);
+
+        this.query = query;
+        this.queries = queries;
+        this.column = options.column || '1';
+        this.operation = options.operation;
+        this._isFloatColumn = null;
     }
 
-    if (!VALID_OPERATIONS[options.operation]) {
-        throw new Error("Formula does not support '" + options.operation + "' operation");
+    _checkOptions (options) {
+        if (typeof options.operation !== 'string') {
+            throw new Error(`Formula expects 'operation' in dataview options`);
+        }
+
+        if (!VALID_OPERATIONS[options.operation]) {
+            throw new Error(`Formula does not support '${options.operation}' operation`);
+        }
+
+        if (options.operation !== 'count' && typeof options.column !== 'string') {
+            throw new Error(`Formula expects 'column' in dataview options`);
+        }
     }
 
-    if (options.operation !== 'count' && !_.isString(options.column)) {
-        throw new Error('Formula expects `column` in widget options');
+
+    sql (psql, override, callback) {
+        if (!callback) {
+            callback = override;
+            override = {};
+        }
+
+        if (this._isFloatColumn === null) {
+            this._isFloatColumn = false;
+            this.getColumnType(psql, this.column, this.queries.no_filters, (err, type) => {
+                if (!err && !!type) {
+                    this._isFloatColumn = type.float;
+                }
+                this.sql(psql, override, callback);
+            });
+            return null;
+        }
+
+        const formulaSql = formulaQueryTpl({
+            isFloatColumn: this._isFloatColumn,
+            query: this.query,
+            operation: this.operation,
+            column: this.column
+        });
+
+        debug(formulaSql);
+
+        return callback(null, formulaSql);
     }
 
-    BaseWidget.apply(this);
+    format (res) {
+        const {
+            result = 0,
+            nulls_count = 0,
+            nans_count,
+            infinities_count
+        } = res.rows[0] || {};
 
-    this.query = query;
-    this.column = options.column || '1';
-    this.operation = options.operation;
-}
-
-Formula.prototype = new BaseWidget();
-Formula.prototype.constructor = Formula;
-
-module.exports = Formula;
-
-Formula.prototype.sql = function(psql, override, callback) {
-    if (!callback) {
-        callback = override;
-        override = {};
+        return {
+            operation: this.operation,
+            result,
+            nulls: nulls_count,
+            nans: nans_count,
+            infinities: infinities_count
+        };
     }
 
-    var _query = this.query;
-    var formulaSql = formulaQueryTpl({
-        _query: _query,
-        _operation: this.operation,
-        _column: this.column
-    });
+    getType () {
+        return TYPE;
+    }
 
-    debug(formulaSql);
-
-    return callback(null, formulaSql);
-};
-
-Formula.prototype.format = function(result) {
-    var formattedResult = {
-        operation: this.operation,
-        result: 0,
-        nulls: 0
-    };
-
-    if (result.rows.length) {
-        formattedResult.operation = this.operation;
-        formattedResult.result = result.rows[0].result;
-        formattedResult.nulls = result.rows[0].nulls_count;
-    }
-
-    return formattedResult;
-};
-
-Formula.prototype.getType = function() {
-    return TYPE;
-};
-
-Formula.prototype.toString = function() {
-    return JSON.stringify({
-        _type: TYPE,
-        _query: this.query,
-        _column: this.column,
-        _operation: this.operation
-    });
+    toString () {
+        return JSON.stringify({
+            _type: TYPE,
+            _query: this.query,
+            _column: this.column,
+            _operation: this.operation
+        });
+    }
 };

lib/cartodb/models/dataview/histogram.js

@@ -1,308 +1,72 @@
-var _ = require('underscore');
-var BaseWidget = require('./base');
-var debug = require('debug')('windshaft:dataview:histogram');
+const debug = require('debug')('windshaft:dataview:histogram');
+const NumericHistogram = require('./histograms/numeric-histogram');
+const DateHistogram = require('./histograms/date-histogram');
 
-var dot = require('dot');
-dot.templateSettings.strip = false;
+const DATE_HISTOGRAM = 'DateHistogram';
+const NUMERIC_HISTOGRAM = 'NumericHistogram';
 
-var columnTypeQueryTpl = dot.template(
-    'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1'
-);
-var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})");
+module.exports = class Histogram {
+    constructor (query, options, queries) {
+        this.query = query;
+        this.options = options || {};
+        this.queries = queries;
 
-var BIN_MIN_NUMBER = 6;
-var BIN_MAX_NUMBER = 48;
-
-var basicsQueryTpl = dot.template([
-    'basics AS (',
-    '  SELECT',
-    '    max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,',
-    '    avg({{=it._column}}) AS avg_val, count(1) AS total_rows',
-    '  FROM ({{=it._query}}) _cdb_basics',
-    ')'
-].join(' \n'));
-
-var overrideBasicsQueryTpl = dot.template([
-    'basics AS (',
-    '  SELECT',
-    '    max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,',
-    '    avg({{=it._column}}) AS avg_val, count(1) AS total_rows',
-    '  FROM ({{=it._query}}) _cdb_basics',
-    ')'
-].join('\n'));
-
-var iqrQueryTpl = dot.template([
-    'iqrange AS (',
-    '  SELECT max(quartile_max) - min(quartile_max) AS iqr',
-    '  FROM (',
-    '    SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (',
-    '      SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}',
-    '    ) AS quartile',
-    '    FROM ({{=it._query}}) _cdb_rank) _cdb_quartiles',
-    '    WHERE quartile = 1 or quartile = 3',
-    '    GROUP BY quartile',
-    '  ) _cdb_iqr',
-    ')'
-].join('\n'));
-
-var binsQueryTpl = dot.template([
-    'bins AS (',
-    '  SELECT CASE WHEN total_rows = 0 OR iqr = 0',
-    '      THEN 1',
-    '      ELSE GREATEST(',
-    '        LEAST({{=it._minBins}}, CAST(total_rows AS INT)),',
-    '        LEAST(',
-    '          CAST(((max_val - min_val) / (2 * iqr * power(total_rows, 1/3))) AS INT),',
-    '          {{=it._maxBins}}',
-    '        )',
-    '      )',
-    '    END AS bins_number',
-    '  FROM basics, iqrange, ({{=it._query}}) _cdb_bins',
-    '  LIMIT 1',
-    ')'
-].join('\n'));
-
-var overrideBinsQueryTpl = dot.template([
-    'bins AS (',
-    '  SELECT {{=it._bins}} AS bins_number',
-    ')'
-].join('\n'));
-
-var nullsQueryTpl = dot.template([
-    'nulls AS (',
-    '  SELECT',
-    '  count(*) AS nulls_count',
-    '  FROM ({{=it._query}}) _cdb_histogram_nulls',
-    '  WHERE {{=it._column}} IS NULL',
-    ')'
-].join('\n'));
-
-var histogramQueryTpl = dot.template([
-    'SELECT',
-    '    (max_val - min_val) / cast(bins_number as float) AS bin_width,',
-    '    bins_number,',
-    '    nulls_count,',
-    '    avg_val,',
-    '    CASE WHEN min_val = max_val',
-    '      THEN 0',
-    '      ELSE GREATEST(1, LEAST(WIDTH_BUCKET({{=it._column}}, min_val, max_val, bins_number), bins_number)) - 1',
-    '    END AS bin,',
-    '    min({{=it._column}})::numeric AS min,',
-    '    max({{=it._column}})::numeric AS max,',
-    '    avg({{=it._column}})::numeric AS avg,',
-    '    count(*) AS freq',
-    'FROM ({{=it._query}}) _cdb_histogram, basics, nulls, bins',
-    'WHERE {{=it._column}} IS NOT NULL',
-    'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val',
-    'ORDER BY bin'
-].join('\n'));
-
-
-var TYPE = 'histogram';
-
-/**
- {
-     type: 'histogram',
-     options: {
-         column: 'name',
-         bins: 10 // OPTIONAL
-     }
- }
- */
-function Histogram(query, options, queries) {
-    if (!_.isString(options.column)) {
-        throw new Error('Histogram expects `column` in widget options');
+        this.histogramImplementation = this._getHistogramImplementation();
     }
 
-    this.query = query;
-    this.queries = queries;
-    this.column = options.column;
-    this.bins = options.bins;
+    _getHistogramImplementation (override) {
+        let implementation = null;
 
-    this._columnType = null;
-}
-
-Histogram.prototype = new BaseWidget();
-Histogram.prototype.constructor = Histogram;
-
-module.exports = Histogram;
-
-var DATE_OIDS = {
-    1082: true,
-    1114: true,
-    1184: true
-};
-
-Histogram.prototype.sql = function(psql, override, callback) {
-    if (!callback) {
-        callback = override;
-        override = {};
-    }
-
-    var self = this;
-
-    var _column = this.column;
-
-    var columnTypeQuery = columnTypeQueryTpl({
-        column: _column, query: this.queries.no_filters
-    });
-
-    if (this._columnType === null) {
-        psql.query(columnTypeQuery, function(err, result) {
-            // assume numeric, will fail later
-            self._columnType = 'numeric';
-            if (!err && !!result.rows[0]) {
-                var pgType = result.rows[0].pg_typeof;
-                if (DATE_OIDS.hasOwnProperty(pgType)) {
-                    self._columnType = 'date';
-                }
-            }
-            self.sql(psql, override, callback);
-        }, true); // use read-only transaction
-        return null;
-    }
-
-    if (this._columnType === 'date') {
-        _column = columnCastTpl({column: _column});
-    }
-
-    var _query = this.query;
-
-    var basicsQuery, binsQuery;
-
-    if (override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins')) {
-        debug('overriding with %j', override);
-        basicsQuery = overrideBasicsQueryTpl({
-            _query: _query,
-            _column: _column,
-            _start: getBinStart(override),
-            _end: getBinEnd(override)
-        });
-
-        binsQuery = [
-            overrideBinsQueryTpl({
-                _bins: override.bins
-            })
-        ].join(',\n');
-    } else {
-        basicsQuery = basicsQueryTpl({
-            _query: _query,
-            _column: _column
-        });
-
-        if (override && _.has(override, 'bins')) {
-            binsQuery = [
-                overrideBinsQueryTpl({
-                    _bins: override.bins
-                })
-            ].join(',\n');
-        } else {
-            binsQuery = [
-                iqrQueryTpl({
-                    _query: _query,
-                    _column: _column
-                }),
-                binsQueryTpl({
-                    _query: _query,
-                    _minBins: BIN_MIN_NUMBER,
-                    _maxBins: BIN_MAX_NUMBER
-                })
-            ].join(',\n');
+        switch (this._getHistogramSubtype(override)) {
+            case DATE_HISTOGRAM:
+                debug('Delegating to DateHistogram with options: %j and overriding: %j', this.options, override);
+                implementation = new DateHistogram(this.query, this.options, this.queries);
+                break;
+            case NUMERIC_HISTOGRAM:
+                debug('Delegating to NumericHistogram with options: %j and overriding: %j', this.options, override);
+                implementation = new NumericHistogram(this.query, this.options, this.queries);
+                break;
+            default:
+                throw new Error('Unsupported Histogram type');
         }
+
+        return implementation;
     }
 
+    _getHistogramSubtype (override) {
+        if(this._isDateHistogram(override)) {
+            return DATE_HISTOGRAM;
+        }
 
-    var histogramSql = [
-        "WITH",
-        [
-            basicsQuery,
-            binsQuery,
-            nullsQueryTpl({
-                _query: _query,
-                _column: _column
-            })
-        ].join(',\n'),
-        histogramQueryTpl({
-            _query: _query,
-            _column: _column
-        })
-    ].join('\n');
+        return NUMERIC_HISTOGRAM;
+    }
 
-    debug(histogramSql);
+    _isDateHistogram (override = {}) {
+        return (this.options.hasOwnProperty('aggregation') || override.hasOwnProperty('aggregation'));
+    }
 
-    return callback(null, histogramSql);
-};
-
-Histogram.prototype.format = function(result, override) {
-    override = override || {};
-    var buckets = [];
-
-    var binsCount = getBinsCount(override);
-    var width = getWidth(override);
-    var binsStart = getBinStart(override);
-    var nulls = 0;
-    var avg;
-
-    if (result.rows.length) {
-        var firstRow = result.rows[0];
-        binsCount = firstRow.bins_number;
-        width = firstRow.bin_width || width;
-        avg = firstRow.avg_val;
-        nulls = firstRow.nulls_count;
-        binsStart = override.hasOwnProperty('start') ? getBinStart(override) : firstRow.min;
-
-        buckets = result.rows.map(function(row) {
-            return _.omit(row, 'bins_number', 'bin_width', 'nulls_count', 'avg_val');
-        });
-    }
-
-    return {
-        bin_width: width,
-        bins_count: binsCount,
-        bins_start: binsStart,
-        nulls: nulls,
-        avg: avg,
-        bins: buckets
-    };
-};
-
-function getBinStart(override) {
-    if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) {
-        return Math.min(override.start, override.end);
-    }
-    return override.start || 0;
-}
-
-function getBinEnd(override) {
-    if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) {
-        return Math.max(override.start, override.end);
-    }
-    return override.end || 0;
-}
-
-function getBinsCount(override) {
-    return override.bins || 0;
-}
-
-function getWidth(override) {
-    var width = 0;
-    var binsCount = override.bins;
-
-    if (binsCount && Number.isFinite(override.start) && Number.isFinite(override.end)) {
-        width = (override.end - override.start) / binsCount;
-    }
-
-    return width;
-}
-
-Histogram.prototype.getType = function() {
-    return TYPE;
-};
-
-Histogram.prototype.toString = function() {
-    return JSON.stringify({
-        _type: TYPE,
-        _column: this.column,
-        _query: this.query
-    });
+    getResult (psql, override, callback) {
+        this.histogramImplementation = this._getHistogramImplementation(override);
+        this.histogramImplementation.getResult(psql, override, callback);
+    }
+
+    // In order to keep previous behaviour with overviews,
+    // we have to expose the following methods to bypass
+    // the concrete overview implementation
+
+    sql (psql, override, callback) {
+        this.histogramImplementation.sql(psql, override, callback);
+    }
+
+    format (result, override) {
+        return this.histogramImplementation.format(result, override);
+    }
+
+    getType () {
+        return this.histogramImplementation.getType();
+    }
+
+    toString () {
+        return this.histogramImplementation.toString();
+    }
 };

lib/cartodb/models/dataview/histograms/base-histogram.js

@@ -0,0 +1,85 @@
+const BaseDataview = require('../base');
+
+const TYPE = 'histogram';
+
+module.exports = class BaseHistogram extends BaseDataview {
+    constructor (query, options, queries) {
+        super();
+
+        if (typeof options.column !== 'string') {
+            throw new Error('Histogram expects `column` in widget options');
+        }
+
+        this.query = query;
+        this.queries = queries;
+        this.column = options.column;
+        this.bins = options.bins;
+
+        this._columnType = null;
+    }
+
+    sql (psql, override, callback) {
+        if (!callback) {
+            callback = override;
+            override = {};
+        }
+
+        if (this._columnType === null) {
+            this.getColumnType(psql, this.column, this.queries.no_filters, (err, type) => {
+                // assume numeric, will fail later
+                this._columnType = 'numeric';
+                if (!err && !!type) {
+                    this._columnType = Object.keys(type).find(function (key) {
+                        return type[key];
+                    });
+                }
+                this.sql(psql, override, callback);
+            }, true); // use read-only transaction
+            return null;
+        }
+
+        return this._buildQuery(psql, override, callback);
+    }
+
+    format (result, override) {
+        const histogram = this._getSummary(result, override);
+        histogram.bins = this._getBuckets(result);
+        return histogram;
+    }
+
+    getType () {
+        return TYPE;
+    }
+
+    toString () {
+        return JSON.stringify({
+            _type: TYPE,
+            _column: this.column,
+            _query: this.query
+        });
+    }
+
+    _hasOverridenRange (override) {
+        return override && override.hasOwnProperty('start') && override.hasOwnProperty('end');
+    }
+
+    _getBinStart (override = {}) {
+        if (this._hasOverridenRange(override)) {
+            return Math.min(override.start, override.end);
+        }
+
+        return override.start || 0;
+    }
+
+    _getBinEnd (override = {}) {
+        if (this._hasOverridenRange(override)) {
+            return Math.max(override.start, override.end);
+        }
+
+        return override.end || 0;
+    }
+
+    _getBinsCount (override = {}) {
+        return override.bins || 0;
+    }
+};

lib/cartodb/models/dataview/histograms/date-histogram.js

@@ -0,0 +1,302 @@
+const BaseHistogram = require('./base-histogram');
+const debug = require('debug')('windshaft:dataview:date-histogram');
+
+const dateIntervalQueryTpl = ctx => `
+    WITH
+    __cdb_dates AS (
+        SELECT
+            MAX(${ctx.column}::timestamp) AS __cdb_end,
+            MIN(${ctx.column}::timestamp) AS __cdb_start
+        FROM (${ctx.query}) __cdb_source
+    ),
+    __cdb_interval_in_days AS (
+        SELECT
+            DATE_PART('day', __cdb_end - __cdb_start) AS __cdb_days
+        FROM __cdb_dates
+    ),
+    __cdb_interval_in_hours AS (
+        SELECT
+            __cdb_days * 24 + DATE_PART('hour', __cdb_end - __cdb_start) AS __cdb_hours
+        FROM __cdb_interval_in_days, __cdb_dates
+    ),
+    __cdb_interval_in_minutes AS (
+        SELECT
+            __cdb_hours * 60 + DATE_PART('minute', __cdb_end - __cdb_start) AS __cdb_minutes
+        FROM __cdb_interval_in_hours, __cdb_dates
+    ),
+    __cdb_interval_in_seconds AS (
+        SELECT
+            __cdb_minutes * 60 + DATE_PART('second', __cdb_end - __cdb_start) AS __cdb_seconds
+        FROM __cdb_interval_in_minutes, __cdb_dates
+    )
+    SELECT
+        ROUND(__cdb_days / 365) AS year,
+        ROUND(__cdb_days / 90) AS quarter,
+        ROUND(__cdb_days / 30) AS month,
+        ROUND(__cdb_days / 7) AS week,
+        __cdb_days AS day,
+        __cdb_hours AS hour,
+        __cdb_minutes AS minute,
+        __cdb_seconds AS second
+    FROM __cdb_interval_in_days, __cdb_interval_in_hours, __cdb_interval_in_minutes, __cdb_interval_in_seconds
+`;
+
+const nullsQueryTpl = ctx => `
+    __cdb_nulls AS (
+        SELECT
+            count(*) AS __cdb_nulls_count
+        FROM (${ctx.query}) __cdb_histogram_nulls
+        WHERE ${ctx.column} IS NULL
+    )
+`;
+
+const dateBasicsQueryTpl = ctx => `
+    __cdb_basics AS (
+        SELECT
+            max(date_part('epoch', ${ctx.column})) AS __cdb_max_val,
+            min(date_part('epoch', ${ctx.column})) AS __cdb_min_val,
+            avg(date_part('epoch', ${ctx.column})) AS __cdb_avg_val,
+            min(
+                date_trunc(
+                    '${ctx.aggregation}', ${ctx.column}::timestamp AT TIME ZONE '${ctx.offset}'
+                )
+            ) AS __cdb_start_date,
+            max(${ctx.column}::timestamp AT TIME ZONE '${ctx.offset}') AS __cdb_end_date,
+            count(1) AS __cdb_total_rows
+        FROM (${ctx.query}) __cdb_basics_query
+    )
+`;
+
+const dateOverrideBasicsQueryTpl = ctx => `
+    __cdb_basics AS (
+        SELECT
+            max(${ctx.end})::float AS __cdb_max_val,
+            min(${ctx.start})::float AS __cdb_min_val,
+            avg(date_part('epoch', ${ctx.column})) AS __cdb_avg_val,
+            min(
+                date_trunc(
+                    '${ctx.aggregation}',
+                    TO_TIMESTAMP(${ctx.start})::timestamp AT TIME ZONE '${ctx.offset}'
+                )
+            ) AS __cdb_start_date,
+            max(
+                TO_TIMESTAMP(${ctx.end})::timestamp AT TIME ZONE '${ctx.offset}'
+            ) AS __cdb_end_date,
+            count(1) AS __cdb_total_rows
+        FROM (${ctx.query}) __cdb_basics_query
+    )
+`;
+
+const dateBinsQueryTpl = ctx => `
+    __cdb_bins AS (
+        SELECT
+            __cdb_bins_array,
+            ARRAY_LENGTH(__cdb_bins_array, 1) AS __cdb_bins_number
+        FROM (
+            SELECT
+                ARRAY(
+                    SELECT GENERATE_SERIES(
+                        __cdb_start_date::timestamptz,
+                        __cdb_end_date::timestamptz,
+                        ${ctx.aggregation === 'quarter' ? `'3 month'::interval` : `'1 ${ctx.aggregation}'::interval`}
+                    )
+                ) AS __cdb_bins_array
+            FROM __cdb_basics
+        ) __cdb_bins_array_query
+    )
+`;
+
+const dateHistogramQueryTpl = ctx => `
+    SELECT
+        (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width,
+        __cdb_bins_number AS bins_number,
+        __cdb_nulls_count AS nulls_count,
+        CASE WHEN __cdb_min_val = __cdb_max_val
+        THEN 0
+        ELSE GREATEST(
+            1,
+            LEAST(
+                WIDTH_BUCKET(
+                    ${ctx.column}::timestamp AT TIME ZONE '${ctx.offset}',
+                    __cdb_bins_array
+                ),
+                __cdb_bins_number
+            )
+        ) - 1
+        END AS bin,
+        min(
+            date_part(
+                'epoch',
+                date_trunc(
+                    '${ctx.aggregation}', ${ctx.column}::timestamp AT TIME ZONE '${ctx.offset}'
+                ) AT TIME ZONE '${ctx.offset}'
+            )
+        )::numeric AS timestamp,
+        date_part('epoch', __cdb_start_date)::numeric AS timestamp_start,
+        min(date_part('epoch', ${ctx.column}))::numeric AS min,
+        max(date_part('epoch', ${ctx.column}))::numeric AS max,
+        avg(date_part('epoch', ${ctx.column}))::numeric AS avg,
+        count(*) AS freq
+    FROM (${ctx.query}) __cdb_histogram, __cdb_basics, __cdb_bins, __cdb_nulls
+    WHERE date_part('epoch', ${ctx.column}) IS NOT NULL
+    GROUP BY bin, bins_number, bin_width, nulls_count, timestamp_start
+    ORDER BY bin
+`;
+
+const MAX_INTERVAL_VALUE = 366;
+
+const DATE_AGGREGATIONS = {
+    'auto': true,
+    'minute': true,
+    'hour': true,
+    'day': true,
+    'week': true,
+    'month': true,
+    'quarter': true,
+    'year': true
+};
+
+/**
+    date_histogram: {
+        type: 'histogram',
+        options: {
+            column: 'date', // column data type: date
+            aggregation: 'day' // MANDATORY
+            offset: -7200 // OPTIONAL (UTC offset in seconds)
+        }
+    }
+*/
+module.exports = class DateHistogram extends BaseHistogram {
+    constructor (query, options, queries) {
+        super(query, options, queries);
+
+        this.aggregation = options.aggregation;
+        this.offset = options.offset;
+    }
+
+    _buildQueryTpl (ctx) {
+        return `
+            WITH
+            ${this._hasOverridenRange(ctx.override) ? dateOverrideBasicsQueryTpl(ctx) : dateBasicsQueryTpl(ctx)},
+            ${dateBinsQueryTpl(ctx)},
+            ${nullsQueryTpl(ctx)}
+            ${dateHistogramQueryTpl(ctx)}
+        `;
+    }
+
+    _buildQuery (psql, override, callback) {
+        if (!this._isValidAggregation(override)) {
+            return callback(new Error('Invalid aggregation value. Valid ones: ' +
+                Object.keys(DATE_AGGREGATIONS).join(', ')
+            ));
+        }
+
+        if (this._getAggregation(override) === 'auto') {
+            this._getAutomaticAggregation(psql, function (err, aggregation) {
+                if (err || aggregation === 'none') {
+                    this.aggregation = 'day';
+                } else {
+                    this.aggregation = aggregation;
+                }
+                override.aggregation = this.aggregation;
+                this._buildQuery(psql, override, callback);
+            }.bind(this));
+            return null;
+        }
+
+        const histogramSql = this._buildQueryTpl({
+            override: override,
+            query: this.query,
+            column: this.column,
+            aggregation: this._getAggregation(override),
+            start: this._getBinStart(override),
+            end: this._getBinEnd(override),
+            offset: this._parseOffset(override)
+        });
+
+        debug(histogramSql);
+
+        return callback(null, histogramSql);
+    }
+
+    _isValidAggregation (override) {
+        return DATE_AGGREGATIONS.hasOwnProperty(this._getAggregation(override));
+    }
+
+    _getAutomaticAggregation (psql, callback) {
+        const dateIntervalQuery = dateIntervalQueryTpl({
+            query: this.query,
+            column: this.column
+        });
+
+        psql.query(dateIntervalQuery, function (err, result) {
+            if (err) {
+                return callback(err);
+            }
+
+            const aggegations = result.rows[0];
+            const aggregation = Object.keys(aggegations)
+                .map(key => ({ name: key, value: aggegations[key] }))
+                .reduce((closer, current) => {
+                    if (current.value > MAX_INTERVAL_VALUE) {
+                        return closer;
+                    }
+
+                    const closerDiff = MAX_INTERVAL_VALUE - closer.value;
+                    const currentDiff = MAX_INTERVAL_VALUE - current.value;
+
+                    if (Number.isFinite(current.value) && closerDiff > currentDiff) {
+                        return current;
+                    }
+
+                    return closer;
+                }, { name: 'none', value: -1 });
+
+            callback(null, aggregation.name);
+        });
+    }
+
+    _getSummary (result, override) {
+        const firstRow = result.rows[0] || {};
+
+        return {
+            aggregation: this._getAggregation(override),
+            offset: this._getOffset(override),
+            timestamp_start: firstRow.timestamp_start,
+
+            bin_width: firstRow.bin_width,
+            bins_count: firstRow.bins_number,
+            bins_start: firstRow.timestamp,
+            nulls: firstRow.nulls_count,
+            infinities: firstRow.infinities_count,
+            nans: firstRow.nans_count,
+            avg: firstRow.avg_val
+        };
+    }
+
+    _getBuckets (result) {
+        return result.rows.map(({ bin, min, max, avg, freq, timestamp }) => ({ bin, min, max, avg, freq, timestamp }));
+    }
+
+    _getAggregation (override = {}) {
+        return override.aggregation ? override.aggregation : this.aggregation;
+    }
+
+    _getOffset (override = {}) {
+        return Number.isFinite(override.offset) ? override.offset : (this.offset || 0);
+    }
+
+    _parseOffset (override) {
+        if (this._shouldIgnoreOffset(override)) {
+            return '0';
+        }
+
+        const offsetInHours = Math.ceil(this._getOffset(override) / 3600);
+
+        return '' + offsetInHours;
+    }
+
+    _shouldIgnoreOffset (override) {
+        return (this._getAggregation(override) === 'hour' || this._getAggregation(override) === 'minute');
+    }
+};

lib/cartodb/models/dataview/histograms/numeric-histogram.js

@@ -0,0 +1,234 @@
+const BaseHistogram = require('./base-histogram');
+const debug = require('debug')('windshaft:dataview:numeric-histogram');
+
+const columnCastTpl = ctx => `date_part('epoch', ${ctx.column})`;
+
+const filterOutSpecialNumericValues = ctx => `
+        ${ctx.column} != 'infinity'::float
+    AND
+        ${ctx.column} != '-infinity'::float
+    AND
+        ${ctx.column} != 'NaN'::float
+`;
+
+const filteredQueryTpl = ctx => `
+    __cdb_filtered_source AS (
+        SELECT *
+        FROM (${ctx.query}) __cdb_filtered_source_query
+        WHERE ${ctx.column} IS NOT NULL
+        ${ctx.isFloatColumn ? `AND ${filterOutSpecialNumericValues(ctx)}` : ''}
+    )
+`;
+
+const basicsQueryTpl = ctx => `
+    __cdb_basics AS (
+        SELECT
+            max(${ctx.column}) AS __cdb_max_val, min(${ctx.column}) AS __cdb_min_val,
+            avg(${ctx.column}) AS __cdb_avg_val, count(1) AS __cdb_total_rows
+        FROM __cdb_filtered_source
+    )
+`;
+
+const overrideBasicsQueryTpl = ctx => `
+    __cdb_basics AS (
+        SELECT
+            max(${ctx.end}) AS __cdb_max_val, min(${ctx.start}) AS __cdb_min_val,
+            avg(${ctx.column}) AS __cdb_avg_val, count(1) AS __cdb_total_rows
+        FROM __cdb_filtered_source
+    )
+`;
+
+const iqrQueryTpl = ctx => `
+    __cdb_iqrange AS (
+        SELECT max(quartile_max) - min(quartile_max) AS __cdb_iqr
+        FROM (
+            SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (
+                SELECT ${ctx.column} AS _cdb_iqr_column, ntile(4) over (order by ${ctx.column}
+            ) AS quartile
+            FROM __cdb_filtered_source) _cdb_quartiles
+            WHERE quartile = 1 or quartile = 3
+            GROUP BY quartile
+        ) __cdb_iqr
+    )
+`;
+
+const binsQueryTpl = ctx => `
+    __cdb_bins AS (
+        SELECT
+            CASE WHEN __cdb_total_rows = 0 OR __cdb_iqr = 0
+            THEN 1
+            ELSE GREATEST(
+                LEAST(${ctx.minBins}, CAST(__cdb_total_rows AS INT)),
+                LEAST(
+                    CAST(((__cdb_max_val - __cdb_min_val) / (2 * __cdb_iqr * power(__cdb_total_rows, 1/3))) AS INT),
+                    ${ctx.maxBins}
+                )
+            )
+            END AS __cdb_bins_number
+        FROM __cdb_basics, __cdb_iqrange, __cdb_filtered_source
+        LIMIT 1
+    )
+`;
+
+const overrideBinsQueryTpl = ctx => `
+    __cdb_bins AS (
+        SELECT ${ctx.override.bins} AS __cdb_bins_number
+    )
+`;
+
+const nullsQueryTpl = ctx => `
+    __cdb_nulls AS (
+        SELECT
+            count(*) AS __cdb_nulls_count
+        FROM (${ctx.query}) __cdb_histogram_nulls
+        WHERE ${ctx.column} IS NULL
+    )
+`;
+
+const infinitiesQueryTpl = ctx => `
+    __cdb_infinities AS (
+        SELECT
+            count(*) AS __cdb_infinities_count
+        FROM (${ctx.query}) __cdb_infinities_query
+        WHERE
+            ${ctx.column} = 'infinity'::float
+        OR
+            ${ctx.column} = '-infinity'::float
+    )
+`;
+
+const nansQueryTpl = ctx => `
+    __cdb_nans AS (
+        SELECT
+            count(*) AS __cdb_nans_count
+        FROM (${ctx.query}) __cdb_nans_query
+        WHERE ${ctx.column} = 'NaN'::float
+    )
+`;
+
+const specialNumericValuesColumnDefinitionTpl = () => `
+    __cdb_infinities_count AS infinities_count,
+    __cdb_nans_count AS nans_count
+`;
+
+const specialNumericValuesCTETpl = () => `
+    __cdb_infinities, __cdb_nans
+`;
+
+const specialNumericValuesColumnTpl = () => `
+    infinities_count, nans_count
+`;
+
+const histogramQueryTpl = ctx => `
+    SELECT
+        (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width,
+        __cdb_bins_number AS bins_number,
+        __cdb_nulls_count AS nulls_count,
+        ${ctx.isFloatColumn ? `${specialNumericValuesColumnDefinitionTpl()},` : ''}
+        __cdb_avg_val AS avg_val,
+        CASE WHEN __cdb_min_val = __cdb_max_val
+            THEN 0
+            ELSE GREATEST(
+                1,
+                LEAST(
+                    WIDTH_BUCKET(${ctx.column}, __cdb_min_val, __cdb_max_val, __cdb_bins_number),
+                    __cdb_bins_number
+                )
+            ) - 1
+        END AS bin,
+        min(${ctx.column})::numeric AS min,
+        max(${ctx.column})::numeric AS max,
+        avg(${ctx.column})::numeric AS avg,
+        count(*) AS freq
+    FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins
+        ${ctx.isFloatColumn ? `, ${specialNumericValuesCTETpl()}` : ''}
+    GROUP BY bin, bins_number, bin_width, nulls_count, avg_val
+        ${ctx.isFloatColumn ? `, ${specialNumericValuesColumnTpl()}` : ''}
+    ORDER BY bin
+`;
+
+const BIN_MIN_NUMBER = 6;
+const BIN_MAX_NUMBER = 48;
+
+/**
+Numeric histogram:
+{
+    type: 'histogram',
+    options: {
+        column: 'name', // column data type: numeric
+        bins: 10 // OPTIONAL
+    }
+}
+*/
+module.exports = class NumericHistogram extends BaseHistogram {
+    constructor (query, options, queries) {
+        super(query, options, queries);
+    }
+
+    _buildQuery (psql, override, callback) {
+        const histogramSql = this._buildQueryTpl({
+            override: override,
+            column: this._columnType === 'date' ? columnCastTpl({ column: this.column }) : this.column,
+            isFloatColumn: this._columnType === 'float',
+            query: this.query,
+            start: this._getBinStart(override),
+            end: this._getBinEnd(override),
+            minBins: BIN_MIN_NUMBER,
+            maxBins: BIN_MAX_NUMBER,
+        });
+
+        debug(histogramSql);
+
+        return callback(null, histogramSql);
+    }
+
+    _buildQueryTpl (ctx) {
+        return `
+            WITH
+            ${filteredQueryTpl(ctx)},
+            ${this._hasOverridenRange(ctx.override) ? overrideBasicsQueryTpl(ctx) : basicsQueryTpl(ctx)},
+            ${this._hasOverridenBins(ctx.override) ?
+                overrideBinsQueryTpl(ctx) :
+                `${iqrQueryTpl(ctx)}, ${binsQueryTpl(ctx)}`
+            },
+            ${nullsQueryTpl(ctx)}
+            ${ctx.isFloatColumn ? `,${infinitiesQueryTpl(ctx)}, ${nansQueryTpl(ctx)}` : ''}
+            ${histogramQueryTpl(ctx)}
+        `;
+    }
+
+    _hasOverridenBins (override) {
+        return override && override.hasOwnProperty('bins');
+    }
+
+    _getSummary (result, override) {
+        const firstRow = result.rows[0] || {};
+
+        return {
+            bin_width: firstRow.bin_width,
+            bins_count: firstRow.bins_number,
+            bins_start: this._populateBinStart(firstRow, override),
+            nulls: firstRow.nulls_count,
+            infinities: firstRow.infinities_count,
+            nans: firstRow.nans_count,
+            avg: firstRow.avg_val,
+        };
+    }
+
+    _getBuckets (result) {
+        return result.rows.map(({ bin, min, max, avg, freq }) => ({ bin, min, max, avg, freq }));
+    }
+
+    _populateBinStart (firstRow, override = {}) {
+        let binStart;
+
+        if (override.hasOwnProperty('start')) {
+            binStart = this._getBinStart(override);
+        } else {
+            binStart = firstRow.min;
+        }
+
+        return binStart;
+    }
+
+};

lib/cartodb/models/dataview/index.js

@@ -1,6 +1,5 @@
 module.exports = {
     Aggregation: require('./aggregation'),
     Formula: require('./formula'),
-    Histogram: require('./histogram'),
-    List: require('./list')
+    Histogram: require('./histogram')
 };

lib/cartodb/models/dataview/list.js

@@ -1,66 +0,0 @@
-var dot = require('dot');
-dot.templateSettings.strip = false;
-
-var BaseWidget = require('./base');
-
-var TYPE = 'list';
-
-var listSqlTpl = dot.template('select {{=it._columns}} from ({{=it._query}}) as _cdb_list');
-
-/**
-{
-    type: 'list',
-    options: {
-        columns: ['name', 'description']
-    }
-}
-*/
-
-function List(query, options) {
-    options = options || {};
-
-    if (!Array.isArray(options.columns)) {
-        throw new Error('List expects `columns` array in widget options');
-    }
-
-    BaseWidget.apply(this);
-
-    this.query = query;
-    this.columns = options.columns;
-}
-
-List.prototype = new BaseWidget();
-List.prototype.constructor = List;
-
-module.exports = List;
-
-List.prototype.sql = function(psql, override, callback) {
-    if (!callback) {
-        callback = override;
-    }
-
-    var listSql = listSqlTpl({
-        _query: this.query,
-        _columns: this.columns.join(', ')
-    });
-
-    return callback(null, listSql);
-};
-
-List.prototype.format = function(result) {
-    return {
-        rows: result.rows
-    };
-};
-
-List.prototype.getType = function() {
-    return TYPE;
-};
-
-List.prototype.toString = function() {
-    return JSON.stringify({
-        _type: TYPE,
-        _query: this.query,
-        _columns: this.columns.join(', ')
-    });
-};

lib/cartodb/models/dataview/overviews/aggregation.js

@@ -1,14 +1,36 @@
 var BaseOverviewsDataview = require('./base');
 var BaseDataview = require('../aggregation');
+var debug = require('debug')('windshaft:widget:aggregation:overview');
 
 var dot = require('dot');
 dot.templateSettings.strip = false;
 
+var filteredQueryTpl = dot.template([
+    'filtered_source AS (',
+    '  SELECT *',
+    '  FROM ({{=it._query}}) _cdb_filtered_source',
+    '  {{?it._aggregationColumn  && it._isFloatColumn}}WHERE',
+    '    {{=it._aggregationColumn}} != \'infinity\'::float',
+    '  AND',
+    '    {{=it._aggregationColumn}} != \'-infinity\'::float',
+    '  AND',
+    '    {{=it._aggregationColumn}} != \'NaN\'::float{{?}}',
+    ')'
+].join(' \n'));
+
 var summaryQueryTpl = dot.template([
     'summary AS (',
     '  SELECT',
     '  sum(_feature_count) AS count,',
     '  sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count',
+    '  {{?it._isFloatColumn}},sum(',
+    '    CASE',
+    '      WHEN {{=it._aggregationColumn}} = \'infinity\'::float OR {{=it._aggregationColumn}} = \'-infinity\'::float',
+    '      THEN 1',
+    '      ELSE 0',
+    '    END',
+    '  ) AS infinities_count,',
+    '  sum(CASE WHEN {{=it._aggregationColumn}} = \'NaN\'::float THEN 1 ELSE 0 END) AS nans_count{{?}}',
     '  FROM ({{=it._query}}) _cdb_aggregation_nulls',
     ')'
 ].join('\n'));
@@ -17,7 +39,7 @@ var rankedCategoriesQueryTpl = dot.template([
     'categories AS(',
     '  SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,',
     '    row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank',
-    '  FROM ({{=it._query}}) _cdb_aggregation_all',
+    '  FROM filtered_source',
     '  {{?it._aggregationColumn!==null}}WHERE {{=it._aggregationColumn}} IS NOT NULL{{?}}',
     '  GROUP BY {{=it._column}}',
     '  ORDER BY 2 DESC',
@@ -36,40 +58,46 @@ var categoriesSummaryCountQueryTpl = dot.template([
     '  SELECT count(1) AS categories_count',
     '  FROM (',
     '    SELECT {{=it._column}} AS category',
-    '    FROM ({{=it._query}}) _cdb_categories',
+    '    FROM filtered_source',
     '    GROUP BY {{=it._column}}',
     '  ) _cdb_categories_count',
     ')'
 ].join('\n'));
 
 var rankedAggregationQueryTpl = dot.template([
-    'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count',
+    'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val,',
+    '    count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
     '  FROM categories, summary, categories_summary_min_max, categories_summary_count',
     '  WHERE rank < {{=it._limit}}',
     'UNION ALL',
-    'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val, count, categories_count',
+    'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val,',
+    '    count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
     '  FROM categories, summary, categories_summary_min_max, categories_summary_count',
     '  WHERE rank >= {{=it._limit}}',
-    'GROUP BY nulls_count, min_val, max_val, count, categories_count'
+    'GROUP BY nulls_count, min_val, max_val, count,',
+    '  categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}'
 ].join('\n'));
 
 var aggregationQueryTpl = dot.template([
     'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,',
-    '  nulls_count, min_val, max_val, count, categories_count',
-    'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary_min_max, categories_summary_count',
-    'GROUP BY category, nulls_count, min_val, max_val, count, categories_count',
+    '  nulls_count, min_val, max_val, count, categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
+    'FROM filtered_source, summary, categories_summary_min_max, categories_summary_count',
+    'GROUP BY category, nulls_count, min_val, max_val, count,',
+    '  categories_count{{?it._isFloatColumn}}, nans_count, infinities_count{{?}}',
     'ORDER BY value DESC'
 ].join('\n'));
 
 var CATEGORIES_LIMIT = 6;
 
- function Aggregation(query, options, queryRewriter, queryRewriteData, params) {
-    BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params);
+ function Aggregation(query, options, queryRewriter, queryRewriteData, params, queries) {
+    BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries);
 
     this.query = query;
+    this.queries = queries;
     this.column = options.column;
     this.aggregation = options.aggregation;
     this.aggregationColumn = options.aggregationColumn;
+    this._isFloatColumn = null;
 }
 
 Aggregation.prototype = Object.create(BaseOverviewsDataview.prototype);
@@ -78,27 +106,49 @@ Aggregation.prototype.constructor = Aggregation;
 module.exports = Aggregation;
 
 Aggregation.prototype.sql = function(psql, override, callback) {
+    var self = this;
+
     if (!callback) {
         callback = override;
         override = {};
     }
 
     var _query = this.rewrittenQuery(this.query);
+    var _aggregationColumn = this.aggregation !== 'count' ? this.aggregationColumn : null;
+
+    if (this.aggregationColumn && this._isFloatColumn === null) {
+        this._isFloatColumn = false;
+        this.getColumnType(psql, this.aggregationColumn, this.queries.no_filters, function (err, type) {
+            if (!err && !!type) {
+                self._isFloatColumn = type.float;
+            }
+            self.sql(psql, override, callback);
+        });
+        return null;
+    }
 
     var aggregationSql;
     if (!!override.ownFilter) {
         aggregationSql = [
             "WITH",
             [
-                summaryQueryTpl({
+                filteredQueryTpl({
+                    _isFloatColumn: this._isFloatColumn,
                     _query: _query,
-                    _column: this.column
+                    _column: this.column,
+                    _aggregationColumn: _aggregationColumn
+                }),
+                summaryQueryTpl({
+                    _isFloatColumn: this._isFloatColumn,
+                    _query: _query,
+                    _column: this.column,
+                    _aggregationColumn: _aggregationColumn
                 }),
                 rankedCategoriesQueryTpl({
                     _query: _query,
                     _column: this.column,
                     _aggregation: this.getAggregationSql(),
-                    _aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null
+                    _aggregationColumn: _aggregationColumn
                 }),
                 categoriesSummaryMinMaxQueryTpl({
                     _query: _query,
@@ -110,6 +160,7 @@ Aggregation.prototype.sql = function(psql, override, callback) {
                 })
             ].join(',\n'),
             aggregationQueryTpl({
+                _isFloatColumn: this._isFloatColumn,
                 _query: _query,
                 _column: this.column,
                 _aggregation: this.getAggregationSql(),
@@ -120,15 +171,23 @@ Aggregation.prototype.sql = function(psql, override, callback) {
         aggregationSql = [
             "WITH",
             [
-                summaryQueryTpl({
+                filteredQueryTpl({
+                    _isFloatColumn: this._isFloatColumn,
                     _query: _query,
-                    _column: this.column
+                    _column: this.column,
+                    _aggregationColumn: _aggregationColumn
+                }),
+                summaryQueryTpl({
+                    _isFloatColumn: this._isFloatColumn,
+                    _query: _query,
+                    _column: this.column,
+                    _aggregationColumn: _aggregationColumn
                 }),
                 rankedCategoriesQueryTpl({
                     _query: _query,
                     _column: this.column,
                     _aggregation: this.getAggregationSql(),
-                    _aggregationColumn: this.aggregation !== 'count' ? this.aggregationColumn : null
+                    _aggregationColumn: _aggregationColumn
                 }),
                 categoriesSummaryMinMaxQueryTpl({
                     _query: _query,
@@ -140,6 +199,7 @@ Aggregation.prototype.sql = function(psql, override, callback) {
                 })
             ].join(',\n'),
             rankedAggregationQueryTpl({
+                _isFloatColumn: this._isFloatColumn,
                 _query: _query,
                 _column: this.column,
                 _limit: CATEGORIES_LIMIT
@@ -147,6 +207,8 @@ Aggregation.prototype.sql = function(psql, override, callback) {
         ].join('\n');
     }
 
+    debug(aggregationSql);
+
     return callback(null, aggregationSql);
 };
 

lib/cartodb/models/dataview/overviews/base.js

@@ -1,14 +1,15 @@
 var _ = require('underscore');
 var BaseDataview = require('../base');
 
-function BaseOverviewsDataview(query, queryOptions, BaseDataview, queryRewriter, queryRewriteData, options) {
+function BaseOverviewsDataview(query, queryOptions, BaseDataview, queryRewriter, queryRewriteData, options, queries) {
   this.BaseDataview = BaseDataview;
   this.query = query;
   this.queryOptions = queryOptions;
   this.queryRewriter = queryRewriter;
   this.queryRewriteData = queryRewriteData;
   this.options = options;
-  this.baseDataview = new this.BaseDataview(this.query, this.queryOptions);
+  this.queries = queries;
+  this.baseDataview = new this.BaseDataview(this.query, this.queryOptions, this.queries);
 }
 
 module.exports = BaseOverviewsDataview;

lib/cartodb/models/dataview/overviews/formula.js

@@ -1,34 +1,61 @@
 var BaseOverviewsDataview = require('./base');
 var BaseDataview = require('../formula');
+var debug = require('debug')('windshaft:widget:formula:overview');
 
 var dot = require('dot');
 dot.templateSettings.strip = false;
 
 var formulaQueryTpls = {
-  'count': dot.template([
-      'SELECT',
-      'sum(_feature_count) AS result,',
-      '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
-      'FROM ({{=it._query}}) _cdb_formula'
-  ].join('\n')),
-  'sum': dot.template([
-      'SELECT',
-      'sum({{=it._column}}*_feature_count) AS result,',
-      '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
-      'FROM ({{=it._query}}) _cdb_formula'
-  ].join('\n')),
-  'avg': dot.template([
-      'SELECT',
-      'sum({{=it._column}}*_feature_count)/sum(_feature_count) AS result,',
-      '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
-      'FROM ({{=it._query}}) _cdb_formula'
-  ].join('\n')),
+    'count': dot.template([
+        'SELECT',
+        'sum(_feature_count) AS result,',
+        '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
+        '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities',
+        '  WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count,',
+        '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans',
+        '  WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}',
+        'FROM ({{=it._query}}) _cdb_formula'
+    ].join('\n')),
+    'sum': dot.template([
+        'SELECT',
+        'sum({{=it._column}}*_feature_count) AS result,',
+        '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
+        '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities',
+        '  WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count',
+        ',(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans',
+        '  WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}',
+        'FROM ({{=it._query}}) _cdb_formula',
+        '{{?it._isFloatColumn}}WHERE',
+        '  {{=it._column}} != \'infinity\'::float',
+        'AND',
+        '  {{=it._column}} != \'-infinity\'::float',
+        'AND',
+        '  {{=it._column}} != \'NaN\'::float{{?}}'
+    ].join('\n')),
+    'avg': dot.template([
+        'SELECT',
+        'sum({{=it._column}}*_feature_count)/sum(_feature_count) AS result,',
+        '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
+        '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities',
+        '  WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count',
+        ',(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans',
+        '  WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}',
+        'FROM ({{=it._query}}) _cdb_formula',
+        '{{?it._isFloatColumn}}WHERE',
+        '  {{=it._column}} != \'infinity\'::float',
+        'AND',
+        '  {{=it._column}} != \'-infinity\'::float',
+        'AND',
+        '  {{=it._column}} != \'NaN\'::float{{?}}'
+    ].join('\n')),
 };
 
-function Formula(query, options, queryRewriter, queryRewriteData, params) {
-    BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params);
+function Formula(query, options, queryRewriter, queryRewriteData, params, queries) {
+    BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries);
     this.column = options.column || '1';
     this.operation = options.operation;
+    this._isFloatColumn = null;
+    this.queries = queries;
 }
 
 Formula.prototype = Object.create(BaseOverviewsDataview.prototype);
@@ -36,21 +63,38 @@ Formula.prototype.constructor = Formula;
 
 module.exports = Formula;
 
-Formula.prototype.sql = function(psql, override, callback) {
+Formula.prototype.sql = function (psql, override, callback) {
+    var self = this;
     var formulaQueryTpl = formulaQueryTpls[this.operation];
 
-    if ( formulaQueryTpl ) {
+    if (formulaQueryTpl) {
         // supported formula for use with overviews
+        if (this._isFloatColumn === null) {
+            this._isFloatColumn = false;
+            this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) {
+                if (!err && !!type) {
+                    self._isFloatColumn = type.float;
+                }
+                self.sql(psql, override, callback);
+            });
+            return null;
+        }
+
         var formulaSql = formulaQueryTpl({
-            _query:     this.rewrittenQuery(this.query),
+            _isFloatColumn: this._isFloatColumn,
+            _query: this.rewrittenQuery(this.query),
             _operation: this.operation,
-            _column:    this.column
+            _column: this.column
         });
+
         callback = callback || override;
 
+        debug(formulaSql);
+
         return callback(null, formulaSql);
     }
 
+
     // default behaviour
     return this.defaultSql(psql, override, callback);
 };

lib/cartodb/models/dataview/overviews/histogram.js

@@ -1,23 +1,35 @@
 var _ = require('underscore');
 var BaseOverviewsDataview = require('./base');
 var BaseDataview = require('../histogram');
+var debug = require('debug')('windshaft:dataview:histogram:overview');
 
 var dot = require('dot');
 dot.templateSettings.strip = false;
 
-var columnTypeQueryTpl = dot.template(
-    'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1'
-);
-
 var BIN_MIN_NUMBER = 6;
 var BIN_MAX_NUMBER = 48;
 
+var filteredQueryTpl = dot.template([
+    'filtered_source AS (',
+    '  SELECT *',
+    '  FROM ({{=it._query}}) _cdb_filtered_source',
+    '  WHERE',
+    '    {{=it._column}} IS NOT NULL',
+    '  {{?it._isFloatColumn}}AND',
+    '    {{=it._column}} != \'infinity\'::float',
+    '  AND',
+    '    {{=it._column}} != \'-infinity\'::float',
+    '  AND',
+    '    {{=it._column}} != \'NaN\'::float{{?}}',
+    ')'
+].join(' \n'));
+
 var basicsQueryTpl = dot.template([
     'basics AS (',
     '  SELECT',
     '    max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,',
     '    sum({{=it._column}}*_feature_count)/sum(_feature_count) AS avg_val, sum(_feature_count) AS total_rows',
-    '  FROM ({{=it._query}}) _cdb_basics',
+    '  FROM filtered_source',
     ')'
 ].join(' \n'));
 
@@ -26,7 +38,7 @@ var overrideBasicsQueryTpl = dot.template([
     '  SELECT',
     '    max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,',
     '    sum({{=it._column}}*_feature_count)/sum(_feature_count) AS avg_val, sum(_feature_count) AS total_rows',
-    '  FROM ({{=it._query}}) _cdb_basics',
+    '  FROM filtered_source',
     ')'
 ].join('\n'));
 
@@ -37,7 +49,7 @@ var iqrQueryTpl = dot.template([
     '    SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (',
     '      SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}',
     '    ) AS quartile',
-    '    FROM ({{=it._query}}) _cdb_rank) _cdb_quartiles',
+    '    FROM filtered_source) _cdb_quartiles',
     '    WHERE quartile = 1 or quartile = 3',
     '    GROUP BY quartile',
     '  ) _cdb_iqr',
@@ -56,7 +68,7 @@ var binsQueryTpl = dot.template([
     '        )',
     '      )',
     '    END AS bins_number',
-    '  FROM basics, iqrange, ({{=it._query}}) _cdb_bins',
+    '  FROM basics, iqrange, filtered_source',
     '  LIMIT 1',
     ')'
 ].join('\n'));
@@ -76,11 +88,34 @@ var nullsQueryTpl = dot.template([
     ')'
 ].join('\n'));
 
+var infinitiesQueryTpl = dot.template([
+    'infinities AS (',
+    '  SELECT',
+    '    count(*) AS infinities_count',
+    '  FROM ({{=it._query}}) _cdb_histogram_infinities',
+    '  WHERE',
+    '    {{=it._column}} = \'infinity\'::float',
+    '  OR',
+    '    {{=it._column}} = \'-infinity\'::float',
+    ')'
+].join('\n'));
+
+var nansQueryTpl = dot.template([
+    'nans AS (',
+    '  SELECT',
+    '    count(*) AS nans_count',
+    '  FROM ({{=it._query}}) _cdb_histogram_infinities',
+    '  WHERE {{=it._column}} = \'NaN\'::float',
+    ')'
+].join('\n'));
+
 var histogramQueryTpl = dot.template([
     'SELECT',
     '    (max_val - min_val) / cast(bins_number as float) AS bin_width,',
     '    bins_number,',
     '    nulls_count,',
+    '    {{?it._isFloatColumn}}infinities_count,',
+    '    nans_count,{{?}}',
     '    avg_val,',
     '    CASE WHEN min_val = max_val',
     '      THEN 0',
@@ -90,14 +125,14 @@ var histogramQueryTpl = dot.template([
     '    max({{=it._column}})::numeric AS max,',
     '    sum({{=it._column}}*_feature_count)/sum(_feature_count)::numeric AS avg,',
     '    sum(_feature_count) AS freq',
-    'FROM ({{=it._query}}) _cdb_histogram, basics, nulls, bins',
-    'WHERE {{=it._column}} IS NOT NULL',
+    'FROM filtered_source, basics, nulls, bins{{?it._isFloatColumn}},infinities, nans{{?}}',
     'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val',
+    '   {{?it._isFloatColumn}}, infinities_count, nans_count{{?}}',
     'ORDER BY bin'
 ].join('\n'));
 
 function Histogram(query, options, queryRewriter, queryRewriteData, params, queries) {
-    BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params);
+    BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries);
 
     this.query = query;
     this.queries = queries;
@@ -112,36 +147,23 @@ Histogram.prototype.constructor = Histogram;
 
 module.exports = Histogram;
 
-
-var DATE_OIDS = {
-    1082: true,
-    1114: true,
-    1184: true
-};
-
 Histogram.prototype.sql = function(psql, override, callback) {
+    var self = this;
+
     if (!callback) {
         callback = override;
         override = {};
     }
 
-    var self = this;
-
-    var _column = this.column;
-
-    var columnTypeQuery = columnTypeQueryTpl({
-        column: _column, query: this.rewrittenQuery(this.queries.no_filters)
-    });
 
     if (this._columnType === null) {
-        psql.query(columnTypeQuery, function(err, result) {
+        this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) {
             // assume numeric, will fail later
             self._columnType = 'numeric';
-            if (!err && !!result.rows[0]) {
-                var pgType = result.rows[0].pg_typeof;
-                if (DATE_OIDS.hasOwnProperty(pgType)) {
-                    self._columnType = 'date';
-                }
+            if (!err && !!type) {
+                self._columnType = Object.keys(type).find(function (key) {
+                    return type[key];
+                });
             }
             self.sql(psql, override, callback);
         }, true); // use read-only transaction
@@ -154,11 +176,24 @@ Histogram.prototype.sql = function(psql, override, callback) {
         return this.defaultSql(psql, override, callback);
     }
 
+    var histogramSql = this._buildQuery(override);
+
+    return callback(null, histogramSql);
+};
+
+Histogram.prototype._buildQuery = function (override) {
+    var filteredQuery, basicsQuery, binsQuery;
+    var _column = this.column;
     var _query = this.rewrittenQuery(this.query);
 
-    var basicsQuery, binsQuery;
+    filteredQuery = filteredQueryTpl({
+        _isFloatColumn: this._columnType === 'float',
+        _query: _query,
+        _column: _column
+    });
 
-    if (override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins')) {
+    if (this._shouldOverride(override)) {
+        debug('overriding with %j', override);
         basicsQuery = overrideBasicsQueryTpl({
             _query: _query,
             _column: _column,
@@ -177,7 +212,7 @@ Histogram.prototype.sql = function(psql, override, callback) {
             _column: _column
         });
 
-        if (override && _.has(override, 'bins')) {
+        if (this._shouldOverrideBins(override)) {
             binsQuery = [
                 overrideBinsQueryTpl({
                     _bins: override.bins
@@ -198,22 +233,50 @@ Histogram.prototype.sql = function(psql, override, callback) {
         }
     }
 
+    var cteSql = [
+        filteredQuery,
+        basicsQuery,
+        binsQuery,
+        nullsQueryTpl({
+            _query: _query,
+            _column: _column
+        })
+    ];
 
-    var histogramSql = [
-        "WITH",
-        [
-            basicsQuery,
-            binsQuery,
-            nullsQueryTpl({
+    if (this._columnType === 'float') {
+        cteSql.push(
+            infinitiesQueryTpl({
+                _query: _query,
+                _column: _column
+            }),
+            nansQueryTpl({
                 _query: _query,
                 _column: _column
             })
-        ].join(',\n'),
+        );
+    }
+
+    var histogramSql = [
+        "WITH",
+        cteSql.join(',\n'),
         histogramQueryTpl({
+            _isFloatColumn: this._columnType === 'float',
             _query: _query,
             _column: _column
         })
     ].join('\n');
 
-    return callback(null, histogramSql);
+    debug(histogramSql);
+
+    return histogramSql;
 };
+
+Histogram.prototype._shouldOverride = function (override) {
+    return override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins');
+};
+
+Histogram.prototype._shouldOverrideBins = function (override) {
+    return override && _.has(override, 'bins');
+};
+
+

lib/cartodb/models/dataview/overviews/index.js

@@ -1,6 +1,5 @@
 module.exports = {
   Aggregation: require('./aggregation'),
   Formula: require('./formula'),
-  Histogram: require('./histogram'),
-  List: require('./list')
+  Histogram: require('./histogram')
 };

lib/cartodb/models/dataview/overviews/list.js

@@ -1,11 +0,0 @@
-var BaseOverviewsDataview = require('./base');
-var BaseDataview = require('../list');
-
-function List(query, options, queryRewriter, queryRewriteData, params) {
-    BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params);
-}
-
-List.prototype = Object.create(BaseOverviewsDataview.prototype);
-List.prototype.constructor = List;
-
-module.exports = List;

lib/cartodb/models/filter/bbox.js

@@ -8,7 +8,7 @@ var filterQueryTpl = dot.template([
 ].join('\n'));
 
 var bboxFilterTpl = dot.template(
-    '{{=it._column}} && ST_Transform(ST_MakeEnvelope({{=it._bbox}}, 4326), {{=it._srid}})'
+    'ST_Intersects({{=it._column}}, ST_Transform(ST_MakeEnvelope({{=it._bbox}}, 4326), {{=it._srid}}))'
 );
 
 var LATITUDE_MAX_VALUE = 85.0511287798066;
@@ -66,7 +66,8 @@ function getBoundingBoxes(west, south, east, north) {
         bboxes.push([west, south, east, north]);
     } else {
         bboxes.push([west, south, 180, north]);
-        bboxes.push([-180, south, east % 180, north]);
+        // here we assume west,east have been adjusted => west >= -180 => east > 180
+        bboxes.push([-180, south, east - 360, north]);
     }
 
     return bboxes;

lib/cartodb/models/mapconfig/adapter/mapconfig-buffer-size-adapter.js

@@ -0,0 +1,25 @@
+function MapConfigBufferSizeAdapter() {
+    this.formats = ['png', 'png32', 'mvt', 'grid.json'];
+}
+
+module.exports = MapConfigBufferSizeAdapter;
+
+MapConfigBufferSizeAdapter.prototype.getMapConfig = function (user, requestMapConfig, params, context, callback) {
+    if (!context.templateParams || !context.templateParams.buffersize) {
+        return callback(null, requestMapConfig);
+    }
+
+    this.formats.forEach(function (format) {
+        if (Number.isFinite(context.templateParams.buffersize[format])) {
+            if (requestMapConfig.buffersize === undefined) {
+                requestMapConfig.buffersize = {};
+            }
+
+            requestMapConfig.buffersize[format] = context.templateParams.buffersize[format];
+        }
+    });
+
+    setImmediate(function () {
+        callback(null, requestMapConfig);
+    });
+};

lib/cartodb/models/mapconfig/adapter/mapconfig-named-layers-adapter.js

@@ -43,7 +43,6 @@ MapConfigNamedLayersAdapter.prototype.getMapConfig = function (user, requestMapC
 
                     if (nestedNamedLayers.length > 0) {
                         var nestedNamedMapsError = new Error('Nested named layers are not allowed');
-                        // nestedNamedMapsError.http_status = 400;
                         return done(nestedNamedMapsError);
                     }
 

lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js

@@ -4,13 +4,6 @@ var dot = require('dot');
 dot.templateSettings.strip = false;
 var queue = require('queue-async');
 var PSQL = require('cartodb-psql');
-/**
- * cartodb-psql creates `global.Promise` as an empty constructor.
- * However, `turbo-carto` relies on a polyfil that fails to create the polyfil
- * as it finds `global.Promise` but it doesn't find `Promise.resolve`.
- */
-global.Promise = global.Promise || function() {};
-global.Promise.resolve = global.Promise.resolve || function() {};
 var turboCarto = require('turbo-carto');
 
 var SubstitutionTokens = require('../../../utils/substitution-tokens');

lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js

@@ -26,7 +26,7 @@ CreateLayergroupMapConfigProvider.prototype.getMapConfig = function(callback) {
     var context = {};
     step(
         function prepareContextLimits() {
-            self.userLimitsApi.getRenderLimits(self.user, this);
+            self.userLimitsApi.getRenderLimits(self.user, self.params.api_key, this);
         },
         function handleRenderLimits(err, renderLimits) {
             assert.ifError(err);

lib/cartodb/models/mapconfig/provider/map-store-provider.js

@@ -27,7 +27,7 @@ MapStoreMapConfigProvider.prototype.getMapConfig = function(callback) {
     var context = {};
     step(
         function prepareContextLimits() {
-            self.userLimitsApi.getRenderLimits(self.user, this);
+            self.userLimitsApi.getRenderLimits(self.user, self.params.api_key, this);
         },
         function handleRenderLimits(err, renderLimits) {
             assert.ifError(err);

lib/cartodb/models/mapconfig/provider/named-map-provider.js

@@ -90,6 +90,7 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) {
         },
         function instantiateTemplate(err, templateParams) {
             assert.ifError(err);
+            context.templateParams = templateParams;
             return self.templateMaps.instance(self.template, templateParams);
         },
         function prepareAdapterMapConfig(err, requestMapConfig) {
@@ -113,7 +114,7 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) {
         function prepareContextLimits(err, _mapConfig) {
             assert.ifError(err);
             mapConfig = _mapConfig;
-            self.userLimitsApi.getRenderLimits(self.owner, this);
+            self.userLimitsApi.getRenderLimits(self.owner, self.params.api_key, this);
         },
         function cacheAndReturnMapConfig(err, renderLimits) {
             self.err = err;

lib/cartodb/server.js

@@ -12,7 +12,8 @@ var VarnishHttpCacheBackend = require('./cache/backend/varnish_http');
 var FastlyCacheBackend = require('./cache/backend/fastly');
 
 var StatsClient = require('./stats/client');
-var Profiler = require('./stats/profiler_proxy');
+const stats = require('./middleware/stats');
+
 var RendererStatsReporter = require('./stats/reporter/renderer');
 
 var windshaft = require('windshaft');
@@ -35,12 +36,20 @@ var timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, {encodin
 
 var SqlWrapMapConfigAdapter = require('./models/mapconfig/adapter/sql-wrap-mapconfig-adapter');
 var MapConfigNamedLayersAdapter = require('./models/mapconfig/adapter/mapconfig-named-layers-adapter');
+var MapConfigBufferSizeAdapter = require('./models/mapconfig/adapter/mapconfig-buffer-size-adapter');
 var AnalysisMapConfigAdapter = require('./models/mapconfig/adapter/analysis-mapconfig-adapter');
 var MapConfigOverviewsAdapter = require('./models/mapconfig/adapter/mapconfig-overviews-adapter');
 var TurboCartoAdapter = require('./models/mapconfig/adapter/turbo-carto-adapter');
 var DataviewsWidgetsAdapter = require('./models/mapconfig/adapter/dataviews-widgets-adapter');
 var MapConfigAdapter = require('./models/mapconfig/adapter');
 
+var StatsBackend = require('./backends/stats');
+
+const lzmaMiddleware = require('./middleware/lzma');
+const errorMiddleware = require('./middleware/error-middleware');
+
+const prepareContextMiddleware = require('./middleware/context');
+
 module.exports = function(serverOptions) {
     // Make stats client globally accessible
     global.statsClient = StatsClient.getInstance(serverOptions.statsd);
@@ -115,8 +124,27 @@ module.exports = function(serverOptions) {
     var onTileErrorStrategy;
     if (global.environment.enabledFeatures.onTileErrorStrategy !== false) {
         onTileErrorStrategy = function onTileErrorStrategy$TimeoutTile(err, tile, headers, stats, format, callback) {
-            if (err && err.message === 'Render timed out' && format === 'png') {
-                return callback(null, timeoutErrorTile, { 'Content-Type': 'image/png' }, {});
+
+            function isRenderTimeoutError (err) {
+                return err.message === 'Render timed out';
+            }
+
+            function isDatasourceTimeoutError (err) {
+                return err.message && err.message.match(/canceling statement due to statement timeout/i);
+            }
+
+            function isTimeoutError (err) {
+                return isRenderTimeoutError(err) || isDatasourceTimeoutError(err);
+            }
+
+            function isRasterFormat (format) {
+                return format === 'png' || format === 'jpg';
+            }
+
+            if (isTimeoutError(err) && isRasterFormat(format)) {
+                return callback(null, timeoutErrorTile, {
+                    'Content-Type': 'image/png',
+                }, {});
             } else {
                 return callback(err, tile, headers, stats);
             }
@@ -130,7 +158,8 @@ module.exports = function(serverOptions) {
             grainstore: serverOptions.grainstore,
             mapnik: serverOptions.renderer.mapnik
         },
-        http: serverOptions.renderer.http
+        http: serverOptions.renderer.http,
+        mvt: serverOptions.renderer.mvt
     });
 
     // initialize render cache
@@ -150,11 +179,14 @@ module.exports = function(serverOptions) {
 
     var analysisBackend = new AnalysisBackend(metadataBackend, serverOptions.analysis);
 
+    var statsBackend = new StatsBackend();
+
     var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache();
     app.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
 
     var mapConfigAdapter = new MapConfigAdapter(
         new MapConfigNamedLayersAdapter(templateMaps, pgConnection),
+        new MapConfigBufferSizeAdapter(),
         new SqlWrapMapConfigAdapter(),
         new DataviewsWidgetsAdapter(),
         new AnalysisMapConfigAdapter(analysisBackend),
@@ -181,12 +213,16 @@ module.exports = function(serverOptions) {
 
     var versions = getAndValidateVersions(serverOptions);
 
+    const prepareContext = typeof serverOptions.req2params === 'function' ?
+        serverOptions.req2params :
+        prepareContextMiddleware(authApi, pgConnection);
+
     /*******************************************************************************************************************
      * Routing
      ******************************************************************************************************************/
 
     new controller.Layergroup(
-        authApi,
+        prepareContext,
         pgConnection,
         mapStore,
         tileBackend,
@@ -199,7 +235,7 @@ module.exports = function(serverOptions) {
     ).register(app);
 
     new controller.Map(
-        authApi,
+        prepareContext,
         pgConnection,
         templateMaps,
         mapBackend,
@@ -207,12 +243,12 @@ module.exports = function(serverOptions) {
         surrogateKeysCache,
         userLimitsApi,
         layergroupAffectedTablesCache,
-        mapConfigAdapter
+        mapConfigAdapter,
+        statsBackend
     ).register(app);
 
     new controller.NamedMaps(
-        authApi,
-        pgConnection,
+        prepareContext,
         namedMapProviderCache,
         tileBackend,
         previewBackend,
@@ -221,9 +257,9 @@ module.exports = function(serverOptions) {
         metadataBackend
     ).register(app);
 
-    new controller.NamedMapsAdmin(authApi, pgConnection, templateMaps).register(app);
+    new controller.NamedMapsAdmin(authApi, templateMaps).register(app);
 
-    new controller.Analyses(authApi, pgConnection).register(app);
+    new controller.Analyses(prepareContext).register(app);
 
     new controller.ServerInfo(versions).register(app);
 
@@ -231,6 +267,8 @@ module.exports = function(serverOptions) {
      * END Routing
      ******************************************************************************************************************/
 
+    app.use(errorMiddleware());
+
     return app;
 };
 
@@ -303,15 +341,28 @@ function bootstrap(opts) {
     app.enable('jsonp callback');
     app.disable('x-powered-by');
     app.disable('etag');
+
+    // Fix: https://github.com/CartoDB/Windshaft-cartodb/issues/705
+    // See: http://expressjs.com/en/4x/api.html#app.set
+    app.set('json replacer', function (key, value) {
+        if (value !== value) {
+            return 'NaN';
+        }
+
+        if (value === Infinity) {
+            return 'Infinity';
+        }
+
+        if (value === -Infinity) {
+            return '-Infinity';
+        }
+
+        return value;
+    });
+
     app.use(bodyParser.json());
 
     app.use(function bootstrap$prepareRequestResponse(req, res, next) {
-        req.context = req.context || {};
-        req.profiler = new Profiler({
-            statsd_client: global.statsClient,
-            profile: opts.useProfiler
-        });
-
         if (global.environment && global.environment.api_hostname) {
             res.set('X-Served-By-Host', global.environment.api_hostname);
         }
@@ -319,6 +370,13 @@ function bootstrap(opts) {
         next();
     });
 
+    app.use(stats({
+        enabled: opts.useProfiler,
+        statsClient: global.statsClient
+    }));
+
+    app.use(lzmaMiddleware);
+
     // temporary measure until we upgrade to newer version expressjs so we can check err.status
     app.use(function(err, req, res, next) {
         if (err) {

lib/cartodb/server_options.js

@@ -81,6 +81,7 @@ module.exports = {
         statsInterval: rendererConfig.statsInterval
     },
     renderer: {
+        mvt: rendererConfig.mvt,
         mapnik: _.defaults(rendererConfig.mapnik, {
             geojson: {
                 dbPoolParams: {

lib/cartodb/utils/query-utils.js

@@ -0,0 +1,26 @@
+function prepareQuery(sql) {
+  var affectedTableRegexCache = {
+      bbox: /!bbox!/g,
+      scale_denominator: /!scale_denominator!/g,
+      pixel_width: /!pixel_width!/g,
+      pixel_height: /!pixel_height!/g
+  };
+
+  return sql
+      .replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)')
+      .replace(affectedTableRegexCache.scale_denominator, '0')
+      .replace(affectedTableRegexCache.pixel_width, '1')
+      .replace(affectedTableRegexCache.pixel_height, '1');
+}
+
+module.exports.extractTableNames = function extractTableNames(query) {
+    return [
+        'SELECT * FROM CDB_QueryTablesText($windshaft$',
+        prepareQuery(query),
+        '$windshaft$) as tablenames'
+    ].join('');
+};
+
+module.exports.getQueryRowCount = function getQueryRowEstimation(query) {
+    return 'select CDB_EstimateRowCount(\'' + query + '\') as rows';
+};

package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "name": "windshaft-cartodb",
-  "version": "3.6.7",
+  "version": "4.0.1",
   "description": "A map tile server for CartoDB",
   "keywords": [
     "cartodb"
@@ -16,47 +16,59 @@
   "contributors": [
     "Simon Tokumine <simon@vizzuality.com>",
     "Javi Santana <jsantana@vizzuality.com>",
-    "Sandro Santilli <strk@vizzuality.com>"
+    "Sandro Santilli <strk@vizzuality.com>",
+    "Carlos Matallín <matallo@carto.com>",
+    "Daniel Garcia Aubert <dgaubert@carto.com>",
+    "Mario de Frutos <mario.defrutos@carto.com>"
   ],
   "dependencies": {
-    "body-parser": "~1.14.0",
-    "camshaft": "0.54.4",
-    "cartodb-psql": "0.10.1",
-    "cartodb-query-tables": "0.2.0",
-    "cartodb-redis": "0.13.2",
-    "debug": "~2.2.0",
+    "body-parser": "^1.18.2",
+    "camshaft": "0.59.4",
+    "cartodb-psql": "0.10.2",
+    "cartodb-query-tables": "0.3.0",
+    "cartodb-redis": "0.14.0",
+    "debug": "^3.1.0",
     "dot": "~1.0.2",
-    "express": "~4.13.3",
+    "express": "~4.16.0",
     "fastly-purge": "~1.0.1",
     "log4js": "cartodb/log4js-node#cdb",
     "lru-cache": "2.6.5",
     "lzma": "~2.3.2",
     "node-statsd": "~0.0.7",
+    "on-headers": "^1.0.1",
     "queue-async": "~1.0.7",
     "redis-mpool": "0.4.1",
-    "request": "~2.79.0",
+    "request": "^2.83.0",
+    "semver": "~5.3.0",
     "step": "~0.0.6",
     "step-profiler": "~0.3.0",
-    "turbo-carto": "0.19.0",
+    "turbo-carto": "0.20.1",
     "underscore": "~1.6.0",
-    "windshaft": "3.1.2",
+    "windshaft": "~4.0.0",
     "yargs": "~5.0.0"
   },
   "devDependencies": {
     "istanbul": "~0.4.3",
-    "jshint": "~2.6.0",
-    "mocha": "~1.21.4",
+    "jshint": "~2.9.4",
+    "mocha": "~3.4.1",
+    "moment": "~2.18.1",
     "nock": "~2.11.0",
     "redis": "~0.12.1",
-    "semver": "~1.1.4",
     "strftime": "~0.8.2"
   },
   "scripts": {
+    "lint": "jshint lib test",
     "preinstall": "make pre-install",
-    "test": "make test-all"
+    "test": "make test-all",
+    "update-internal-deps": "rm -rf node_modules && rm -f yarn.lock && yarn",
+    "docker-install": "sudo apt install docker.io && sudo usermod -aG docker $(whoami)",
+    "docker-pull": "docker pull cartoimages/windshaft-testing",
+    "docker-test": "docker run -v `pwd`:/srv cartoimages/windshaft-testing bash docker-test.sh",
+    "docker-bash": "docker run  -it -v `pwd`:/srv cartoimages/windshaft-testing bash",
+    "docker-publish": "docker push cartoimages/windshaft-carto-testing"
   },
   "engines": {
     "node": ">=6.9",
-    "yarn": "^0.21.3"
+    "yarn": ">=0.27.5 <1.0.0"
   }
 }

run_tests.sh

@@ -137,7 +137,7 @@ if test x"$OPT_COVERAGE" = xyes; then
   ./node_modules/.bin/istanbul cover node_modules/.bin/_mocha -- -u tdd -t 5000 ${TESTS}
 else
   echo "Running tests"
-  mocha -u tdd -t 5000 ${TESTS}
+  ./node_modules/.bin/_mocha -c -u tdd -t 5000 ${TESTS}
 fi
 ret=$?
 

test/acceptance/analysis/analysis-layers-geojson.js

@@ -1,81 +0,0 @@
-require('../../support/test_helper');
-
-var assert = require('../../support/assert');
-var TestClient = require('../../support/test-client');
-
-describe('analysis-layers-dataviews-geojson', function() {
-
-    function createMapConfig(layers, dataviews, analysis) {
-        return {
-            version: '1.5.0',
-            layers: layers,
-            dataviews: dataviews || {},
-            analyses: analysis || []
-        };
-    }
-
-    var CARTOCSS = [
-        "#points {",
-        "  marker-fill-opacity: 1.0;",
-        "  marker-line-color: #FFF;",
-        "  marker-line-width: 0.5;",
-        "  marker-line-opacity: 1.0;",
-        "  marker-placement: point;",
-        "  marker-type: ellipse;",
-        "  marker-width: 8;",
-        "  marker-fill: red;",
-        "  marker-allow-overlap: true;",
-        "}"
-    ].join('\n');
-
-    var mapConfig = createMapConfig(
-        [
-            {
-                "type": "cartodb",
-                "options": {
-                    "source": {
-                        "id": "2570e105-7b37-40d2-bdf4-1af889598745"
-                    },
-                    "cartocss": CARTOCSS,
-                    "cartocss_version": "2.3.0"
-                }
-            }
-        ],
-        {
-            pop_max_histogram: {
-                source: {
-                    id: '2570e105-7b37-40d2-bdf4-1af889598745'
-                },
-                type: 'histogram',
-                options: {
-                    column: 'pop_max'
-                }
-            }
-        },
-        [
-            {
-                "id": "2570e105-7b37-40d2-bdf4-1af889598745",
-                "type": "source",
-                "params": {
-                    "query": "select * from populated_places_simple_reduced"
-                }
-            }
-        ]
-    );
-
-    it('should get pop_max column from dataview', function(done) {
-        var testClient = new TestClient(mapConfig, 1234);
-
-        testClient.getTile(0, 0, 0, {format: 'geojson', layers: 0}, function(err, res, geojson) {
-            assert.ok(!err, err);
-
-            assert.ok(Array.isArray(geojson.features));
-            assert.ok(geojson.features.length > 0);
-            var feature = geojson.features[0];
-            assert.ok(feature.properties.hasOwnProperty('pop_max'), 'Missing pop_max property');
-
-            testClient.drain(done);
-        });
-    });
-
-});

test/acceptance/analysis/analysis-layers-use-cases.js

@@ -5,34 +5,34 @@ var TestClient = require('../../support/test-client');
 var dot = require('dot');
 var debug = require('debug')('windshaft:cartodb:test');
 
-describe('analysis-layers use cases', function() {
+describe('analysis-layers use cases', function () {
 
 
-    var multitypeStyleTemplate = dot.template([
-        "#points['mapnik::geometry_type'=1] {",
-        "  marker-fill-opacity: {{=it._opacity}};",
-        "  marker-line-color: #FFF;",
-        "  marker-line-width: 0.5;",
-        "  marker-line-opacity: {{=it._opacity}};",
-        "  marker-placement: point;",
-        "  marker-type: ellipse;",
-        "  marker-width: 8;",
-        "  marker-fill: {{=it._color}};",
-        "  marker-allow-overlap: true;",
-        "}",
-        "#lines['mapnik::geometry_type'=2] {",
-        "  line-color: {{=it._color}};",
-        "  line-width: 2;",
-        "  line-opacity: {{=it._opacity}};",
-        "}",
-        "#polygons['mapnik::geometry_type'=3] {",
-        "  polygon-fill: {{=it._color}};",
-        "  polygon-opacity: {{=it._opacity}};",
-        "  line-color: #FFF;",
-        "  line-width: 0.5;",
-        "  line-opacity: {{=it._opacity}};",
-        "}"
-    ].join('\n'));
+    var multitypeStyleTemplate = dot.template(
+        `#points['mapnik::geometry_type'=1] {
+          marker-fill-opacity: {{=it._opacity}};
+          marker-line-color: #FFF;
+          marker-line-width: 0.5;
+          marker-line-opacity: {{=it._opacity}};
+          marker-placement: point;
+          marker-type: ellipse;
+          marker-width: 8;
+          marker-fill: {{=it._color}};
+          marker-allow-overlap: true;
+        }
+        #lines['mapnik::geometry_type'=2] {
+          line-color: {{=it._color}};
+          line-width: 2;
+          line-opacity: {{=it._opacity}};
+        }
+        #polygons['mapnik::geometry_type'=3] {
+          polygon-fill: {{=it._color}};
+          polygon-opacity: {{=it._opacity}};
+          line-color: #FFF;
+          line-width: 0.5;
+          line-opacity: {{=it._opacity}};
+        }`
+    );
 
 
     function cartocss(color, opacity) {
@@ -47,18 +47,53 @@ describe('analysis-layers use cases', function() {
             version: '1.5.0',
             layers: layers,
             dataviews: dataviews || {},
-            analysis: analysis || []
+            analyses: analysis || []
         };
     }
 
-    function analysisDef(analysis) {
-        return JSON.stringify(analysis);
-    }
-
     var DEFAULT_MULTITYPE_STYLE = cartocss();
 
     var TILE_ANALYSIS_TABLES = { z: 14, x: 8023, y: 6177 };
 
+    var pointInPolygonDef = {
+        id: 'a1',
+        type: 'point-in-polygon',
+        params: {
+            points_source: {
+                type: 'source',
+                params: {
+                    query: 'select * from analysis_rent_listings'
+                }
+            },
+            polygons_source: {
+                type: 'buffer',
+                params: {
+                    source: {
+                        type: 'source',
+                        params: {
+                            query: 'select * from analysis_banks'
+                        }
+                    },
+                    radius: 250
+                }
+            }
+        }
+    };
+
+    var bufferDef = {
+        id: 'b1',
+        type: 'buffer',
+        params: {
+            source: {
+                type: 'source',
+                params: {
+                    query: 'select * from analysis_banks'
+                }
+            },
+            radius: 250
+        }
+    };
+
     var useCases = [
         {
             desc: '1 mapnik layer',
@@ -68,7 +103,7 @@ describe('analysis-layers use cases', function() {
                     {
                         type: 'cartodb',
                         options: {
-                            sql: "select * from analysis_rent_listings",
+                            sql: 'select * from analysis_rent_listings',
                             cartocss: DEFAULT_MULTITYPE_STYLE,
                             cartocss_version: '2.3.0'
                         }
@@ -83,7 +118,7 @@ describe('analysis-layers use cases', function() {
                 {
                     type: 'cartodb',
                     options: {
-                        sql: "select * from analysis_banks",
+                        sql: 'select * from analysis_banks',
                         cartocss: cartocss('#2167AB'),
                         cartocss_version: '2.3.0'
                     }
@@ -91,7 +126,7 @@ describe('analysis-layers use cases', function() {
                 {
                     type: 'cartodb',
                     options: {
-                        sql: "select * from analysis_rent_listings",
+                        sql: 'select * from analysis_rent_listings',
                         cartocss: DEFAULT_MULTITYPE_STYLE,
                         cartocss_version: '2.3.0'
                     }
@@ -105,30 +140,27 @@ describe('analysis-layers use cases', function() {
                 {
                     type: 'cartodb',
                     options: {
-                        sql: "select * from analysis_rent_listings",
+                        sql: 'select * from analysis_rent_listings',
                         cartocss: DEFAULT_MULTITYPE_STYLE,
                         cartocss_version: '2.3.0'
                     }
                 },
                 {
-                    type: 'analysis',
+                    type: 'cartodb',
                     options: {
-                        def: analysisDef({
-                            "type": "buffer",
-                            "params": {
-                                "source": {
-                                    "type": "source",
-                                    "params": {
-                                        "query": "select * from analysis_banks"
-                                    }
-                                },
-                                "radius": 250
-                            }
-                        }),
-                        cartocss: cartocss('black', 0.5)
+                        source: {
+                            id: 'b1'
+                        },
+                        cartocss: DEFAULT_MULTITYPE_STYLE,
+                        cartocss_version: '2.3.0'
                     }
                 }
-            ])
+            ],
+                {},
+                [
+                    bufferDef
+                ]
+            )
         },
 
         {
@@ -137,531 +169,115 @@ describe('analysis-layers use cases', function() {
                 {
                     type: 'cartodb',
                     options: {
-                        sql: "select * from analysis_rent_listings",
+                        sql: 'select * from analysis_rent_listings',
                         cartocss: DEFAULT_MULTITYPE_STYLE,
                         cartocss_version: '2.3.0'
                     }
                 },
                 {
-                    type: 'analysis',
+                    type: 'cartodb',
                     options: {
-                        def: analysisDef({
-                            "type": "point-in-polygon",
-                            "params": {
-                                "pointsSource": {
-                                    "type": "source",
-                                    "params": {
-                                        "query": "select * from analysis_rent_listings"
-                                    }
-                                },
-                                "polygonsSource": {
-                                    "type": "buffer",
-                                    "params": {
-                                        "source": {
-                                            "type": "source",
-                                            "params": {
-                                                "query": "select * from analysis_banks"
-                                            }
-                                        },
-                                        "radius": 250
-                                    }
-                                }
-                            }
-                        }),
-                        cartocss: cartocss('green', 1.0)
+                        source: {
+                            id: 'a1'
+                        },
+                        cartocss: DEFAULT_MULTITYPE_STYLE,
+                        cartocss_version: '2.3.0'
                     }
                 }
-            ])
+            ],
+                {},
+                [
+                    pointInPolygonDef
+                ]
+            )
         },
 
         {
             desc: 'point-in-polygon from buffer atm-machines and rent listings + rent listings',
-            mapConfig: mapConfig([
-                {
-                    type: 'analysis',
-                    options: {
-                        def: analysisDef({
-                            "type": "point-in-polygon",
-                            "params": {
-                                "pointsSource": {
-                                    "type": "source",
-                                    "params": {
-                                        "query": "select * from analysis_rent_listings"
-                                    }
-                                },
-                                "polygonsSource": {
-                                    "type": "buffer",
-                                    "params": {
-                                        "source": {
-                                            "type": "source",
-                                            "params": {
-                                                "query": "select * from analysis_banks"
-                                            }
-                                        },
-                                        "radius": 250
-                                    }
-                                }
-                            }
-                        }),
-                        cartocss: cartocss('green', 1.0)
-                    }
-                },
-                {
-                    type: 'cartodb',
-                    options: {
-                        sql: "select * from analysis_rent_listings",
-                        cartocss: DEFAULT_MULTITYPE_STYLE,
-                        cartocss_version: '2.3.0'
-                    }
-                }
-            ])
-        },
-
-        {
-            desc: 'buffer + point-in-polygon from buffer atm-machines and rent listings + rent listings',
-            mapConfig: mapConfig([
-                {
-                    type: 'cartodb',
-                    options: {
-                        sql: "select * from analysis_rent_listings",
-                        cartocss: DEFAULT_MULTITYPE_STYLE,
-                        cartocss_version: '2.3.0'
-                    }
-                },
-                {
-                    type: 'analysis',
-                    options: {
-                        def: analysisDef({
-                            "type": "buffer",
-                            "params": {
-                                "source": {
-                                    "type": "source",
-                                    "params": {
-                                        "query": "select * from analysis_banks"
-                                    }
-                                },
-                                "radius": 300
-                            }
-                        }),
-                        cartocss: cartocss('magenta', 0.5)
-                    }
-                },
-                {
-                    type: 'analysis',
-                    options: {
-                        def: analysisDef({
-                            "type": "point-in-polygon",
-                            "params": {
-                                "pointsSource": {
-                                    "type": "source",
-                                    "params": {
-                                        "query": "select * from analysis_rent_listings"
-                                    }
-                                },
-                                "polygonsSource": {
-                                    "type": "buffer",
-                                    "params": {
-                                        "source": {
-                                            "type": "source",
-                                            "params": {
-                                                "query": "select * from analysis_banks"
-                                            }
-                                        },
-                                        "radius": 300
-                                    }
-                                }
-                            }
-                        }),
-                        cartocss: cartocss('green', 1.0)
-                    }
-                }
-            ])
-        },
-
-        {
-            skip: true,
-            desc: 'buffer + point-in-polygon from buffer atm-machines and rent listings + rent listings',
-            mapConfig: mapConfig([
-                {
-                    type: 'cartodb',
-                    options: {
-                        "source": { id: "a" },
-                        "cartocss": DEFAULT_MULTITYPE_STYLE,
-                        "cartocss_version": "2.3.0"
-                    }
-                },
-                {
-                    type: 'cartodb',
-                    options: {
-                        "source": { id: "b1" },
-                        "cartocss": cartocss('green', 1.0),
-                        "cartocss_version": "2.3.0"
-                    }
-                },
-                {
-                    type: 'cartodb',
-                    options: {
-                        "source": { id: "b2" },
-                        "cartocss": cartocss('magenta', 0.5),
-                        "cartocss_version": "2.3.0"
-                    }
-                }
-            ],
-            [
-                {
-                    id: "b2",
-                    options: {
-                        def: analysisDef({
-                            "type": "count-in-polygon",
-                            "id": "a0",
-                            "params": {
-                                "columnName": 'count_airbnb',
-                                "pointsSource": {
-                                    "type": "source",
-                                    "params": {
-                                        query: "select * from analysis_rent_listings"
-                                    },
-                                    dataviews: {
-                                        price_histogram: {
-                                            type: 'histogram',
-                                            options: {
-                                                column: 'price'
-                                            }
-                                        }
-                                    }
-                                },
-                                "polygonsSource": {
-                                    "id": "b1",
-                                    "type": "buffer",
-                                    "params": {
-                                        "source": {
-                                            "id": "b0",
-                                            "type": "source",
-                                            "params": {
-                                                query: "select * from analysis_banks"
-                                            }
-                                        },
-                                        "radius": 250
-                                    },
-                                    dataviews: {
-                                        bank_category: {
-                                            type: 'aggregation',
-                                            options: {
-                                                column: 'bank'
-                                            }
-                                        }
-                                    }
-                                }
-                            },
-                            dataviews: {
-                                count_histogram: {
-                                    type: 'histogram',
-                                    options: {
-                                        column: 'count_airbnb'
-                                    }
-                                }
-                            }
-                        }),
-                        cartocss: cartocss('green', 1.0)
-                    }
-                }
-            ])
-        },
-
-        {
-            skip: true,
-            desc: 'I. Distribution centers',
             mapConfig: mapConfig(
-                // layers
                 [
                     {
                         type: 'cartodb',
                         options: {
-                            "source": { id: "b0" },
-                            "cartocss": [
-                                "#distribution_centers {",
-                                "  marker-fill-opacity: 1.0;",
-                                "  marker-line-color: #FFF;",
-                                "  marker-line-width: 0.5;",
-                                "  marker-line-opacity: 0.7;",
-                                "  marker-placement: point;",
-                                "  marker-type: ellipse;",
-                                "  marker-width: 8;",
-                                "  marker-fill: blue;",
-                                "  marker-allow-overlap: true;",
-                                "}"
-                            ].join('\n'),
-                            "cartocss_version": "2.3.0"
+                            source: {
+                                id: 'a1'
+                            },
+                            cartocss: DEFAULT_MULTITYPE_STYLE,
+                            cartocss_version: '2.3.0'
                         }
                     },
                     {
                         type: 'cartodb',
                         options: {
-                            "source": { id: "a0" },
-                            "cartocss": [
-                                "#shops {",
-                                "  marker-fill-opacity: 1.0;",
-                                "  marker-line-color: #FFF;",
-                                "  marker-line-width: 0.5;",
-                                "  marker-line-opacity: 0.7;",
-                                "  marker-placement: point;",
-                                "  marker-type: ellipse;",
-                                "  marker-width: 8;",
-                                "  marker-fill: red;",
-                                "  marker-allow-overlap: true;",
-                                "}"
-                            ].join('\n'),
-                            "cartocss_version": "2.3.0"
-                        }
-                    },
-                    {
-                        type: 'cartodb',
-                        options: {
-                            "source": { id: "a1" },
-                            "cartocss": [
-                                "#routing {",
-                                "  line-color: ramp([routing_time], colorbrewer(Reds));",
-                                "  line-width: ramp([routing_time], 2, 8);",
-                                "  line-opacity: 1.0;",
-                                "}"
-                            ].join('\n'),
-                            "cartocss_version": "2.3.0"
+                            sql: 'select * from analysis_rent_listings',
+                            cartocss: DEFAULT_MULTITYPE_STYLE,
+                            cartocss_version: '2.3.0'
                         }
                     }
                 ],
-                // dataviews
-                {
-                    distribution_center_name_category: {
-                        source: { id: 'b0' },
-                        type: 'aggregation',
-                        options: {
-                            column: 'name'
-                        }
-                    },
-                    time_histogram: {
-                        source: { id: 'a1' },
-                        type: 'histogram',
-                        options: {
-                            column: 'routing_time'
-                        }
-                    },
-                    distance_histogram: {
-                        source: { id: 'a1' },
-                        type: 'histogram',
-                        options: {
-                            column: 'routing_distance'
-                        }
-                    }
-                },
-                // analysis
+                {},
                 [
-                    {
-                        id: 'a1',
-                        type: 'routing-n-to-n',
-                        params: {
-                            // distanceColumn: 'routing_distance',
-                            // timeColumn: 'routing_time',
-                            originSource: {
-                                id: 'b0',
-                                type: 'source',
-                                params: {
-                                    query: 'select * from distribution_centers'
-                                }
-                            },
-                            destinationSource: {
-                                id: 'a0',
-                                type: 'source',
-                                params: {
-                                    query: 'select * from shops'
-                                }
-                            }
-                        }
-                    }
+                    pointInPolygonDef
                 ]
             )
         },
 
         {
-            skip: true,
-            desc: 'II. Population analysis',
+            desc: 'buffer + point-in-polygon from buffer atm-machines and rent listings + rent listings',
             mapConfig: mapConfig(
-            // layers
-            [
-                {
-                    type: 'cartodb',
-                    options: {
-                        "source": { id: "a2" },
-                        "cartocss": [
-                            "#count_in_polygon {",
-                            "  polygon-opacity: 1.0",
-                            "  line-color: #FFF;",
-                            "  line-width: 0.5;",
-                            "  line-opacity: 0.7",
-                            "  polygon-fill: ramp([estimated_people], colorbrewer(Reds));",
-                            "}"
-                        ].join('\n'),
-                        "cartocss_version": "2.3.0"
-                    }
-                },
-                {
-                    type: 'cartodb',
-                    options: {
-                        "source": { id: "a0" },
-                        "cartocss": DEFAULT_MULTITYPE_STYLE,
-                        "cartocss_version": "2.3.0"
-                    }
-                }
-            ],
-            // dataviews
-            {
-                total_population_formula: {
-                    "source": { id: "a3" },
-                    type: 'formula',
-                    options: {
-                        column: 'total_population',
-                        operation: 'sum'
-                    }
-                },
-                people_histogram: { // this injects a range filter at `a2` node output
-                    "source": { id: "a2" },
-                    type: 'histogram',
-                    options: {
-                        column: 'estimated_people'
-                    }
-                },
-                subway_line_category: { // this injects a category filter at `a0` node output
-                    "source": { id: "a0" },
-                    type: 'aggregation',
-                    options: {
-                        column: 'subway_line'
-                    }
-                }
-            },
-            // analysis
-            [
-                {
-                    id: 'a3',
-                    // this will union the polygons, produce just one polygon, and calculate the total population for it
-                    type: 'total-population',
-                    params: {
-                        columnName: 'total_population',
-                        source: {
-                            id: 'a2',
-                            type: 'estimated-population',
-                            params: {
-                                columnName: 'estimated_people',
-                                source: {
-                                    id: 'a1',
-                                    type: 'trade-area',
-                                    params: {
-                                        source: {
-                                            "id": "a0",
-                                            "type": "source",
-                                            "params": {
-                                                query: "select * from subway_stops"
-                                            }
-                                        },
-                                        kind: 'walk',
-                                        time: 300
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
-            ])
-        },
-
-        {
-            skip: true,
-            desc: 'III. Point in polygon',
-            mapConfig: mapConfig(
-                // layers
                 [
                     {
                         type: 'cartodb',
                         options: {
-                            "source": { id: "a1" },
-                            "cartocss": [
-                                "#count_in_polygon {",
-                                "  polygon-opacity: 1.0",
-                                "  line-color: #FFF;",
-                                "  line-width: 0.5;",
-                                "  line-opacity: 0.7",
-                                "  polygon-fill: ramp([count_people], colorbrewer(Reds));",
-                                "}"
-                            ].join('\n'),
-                            "cartocss_version": "2.3.0"
+                            sql: 'select * from analysis_rent_listings',
+                            cartocss: DEFAULT_MULTITYPE_STYLE,
+                            cartocss_version: '2.3.0'
+                        }
+                    },
+                    {
+                        type: 'cartodb',
+                        options: {
+                            source: {
+                                id: 'a1'
+                            },
+                            cartocss: DEFAULT_MULTITYPE_STYLE,
+                            cartocss_version: '2.3.0'
+                        }
+                    },
+                    {
+                        type: 'cartodb',
+                        options: {
+                            source: {
+                                id: 'b1'
+                            },
+                            cartocss: DEFAULT_MULTITYPE_STYLE,
+                            cartocss_version: '2.3.0'
                         }
                     }
                 ],
-                // dataviews
-                {
-                    age_histogram: {
-                        "source": { id: "a0" },
-                        type: 'histogram',
-                        options: {
-                            column: 'age'
-                        }
-                    },
-                    income_histogram: {
-                        "source": { id: "a0" },
-                        type: 'histogram',
-                        options: {
-                            column: 'income'
-                        }
-                    },
-                    gender_category: {
-                        "source": { id: "a0" },
-                        type: 'aggregation',
-                        options: {
-                            column: 'gender'
-                        }
-                    }
-                },
-                // analysis
+                {},
                 [
-                    {
-                        "id": "a1",
-                        "type": "count-in-polygon",
-                        "params": {
-                            "columnName": 'count_people',
-                            "pointsSource": {
-                                "id": 'a0',
-                                "type": "source",
-                                "params": {
-                                    query: "select the_geom, age, gender, income from people"
-                                }
-                            },
-                            "polygonsSource": {
-                                "id": "b0",
-                                "type": "source",
-                                "params": {
-                                    query: "select * from postal_codes"
-                                }
-                            }
-                        }
-                    }
+                    bufferDef,
+                    pointInPolygonDef
                 ]
             )
         }
-
     ];
 
-    useCases.forEach(function(useCase, imageIdx) {
+    useCases.forEach(function (useCase) {
         if (!!useCase.skip) {
-            debug(JSON.stringify(useCase.mapConfig, null, 4));
+            return debug(JSON.stringify(useCase.mapConfig, null, 4));
         }
-        it.skip('should implement use case: "' + useCase.desc + '"', function(done) {
+        it(`should implement use case: '${useCase.desc}'`, function (done) {
 
             var testClient = new TestClient(useCase.mapConfig, 1234);
 
             var tile = useCase.tile || TILE_ANALYSIS_TABLES;
 
-            testClient.getTile(tile.z, tile.x, tile.y, function(err, res, image) {
+            testClient.getTile(tile.z, tile.x, tile.y, function (err, res, image) {
                 assert.ok(!err, err);
 
-                image.save('/tmp/tests/' + imageIdx + '---' + useCase.desc.replace(/\s/g, '-') + '.png');
+                //image.save('/tmp/tests/' + imageIdx + '---' + useCase.desc.replace(/\s/g, '-') + '.png');
 
                 assert.equal(image.width(), 256);
 

test/acceptance/analysis/named-maps.js

@@ -7,7 +7,7 @@ var serverOptions = require('../../../lib/cartodb/server_options');
 var server = new CartodbWindshaft(serverOptions);
 var TestClient = require('../../support/test-client');
 
-var LayergroupToken = require('../../support/layergroup-token');
+var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token');
 
 describe('named-maps analysis', function() {
 

test/acceptance/buffer-size-format.js

@@ -0,0 +1,473 @@
+require('../support/test_helper');
+
+var fs = require('fs');
+var assert = require('../support/assert');
+var TestClient = require('../support/test-client');
+var serverOptions = require('../../lib/cartodb/server_options');
+var mapnik = require('windshaft').mapnik;
+var IMAGE_TOLERANCE_PER_MIL = 5;
+
+var CARTOCSS_LABELS = [
+    '#layer {',
+    '  polygon-fill: #374C70;',
+    '  polygon-opacity: 0.9;',
+    '  line-width: 1;',
+    '  line-color: #FFF;',
+    '  line-opacity: 0.5;',
+    '}',
+    '#layer::labels {',
+    '  text-name: [name];',
+    '  text-face-name: \'DejaVu Sans Book\';',
+    '  text-size: 20;',
+    '  text-fill: #FFFFFF;',
+    '  text-label-position-tolerance: 0;',
+    '  text-halo-radius: 1;',
+    '  text-halo-fill: #6F808D;',
+    '  text-dy: -10;',
+    '  text-allow-overlap: true;',
+    '  text-placement: point;',
+    '  text-placement-type: dummy;',
+    '}'
+].join('\n');
+
+function createMapConfig (bufferSize, cartocss) {
+    cartocss = cartocss || CARTOCSS_LABELS;
+
+    return {
+        version: '1.6.0',
+        buffersize: bufferSize,
+        layers: [{
+            type: "cartodb",
+            options: {
+                sql: [
+                    'select',
+                    '   *',
+                    'from',
+                    '   populated_places_simple_reduced',
+                ].join('\n'),
+                cartocss: cartocss,
+                cartocss_version: '2.3.0',
+                interactivity: 'cartodb_id'
+            }
+        }]
+    };
+}
+
+describe('buffer size per format', function () {
+    var testCases = [
+        {
+            desc: 'should get png tile using buffer-size 0',
+            coords: { z: 7, x: 64, y: 48 },
+            format: 'png',
+            fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png',
+            mapConfig: createMapConfig({ png: 0, 'grid.json': 0 }),
+            assert: function (tile, callback) {
+                assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
+            }
+        },
+        {
+            desc: 'should get png tile using buffer-size 128',
+            coords: { z: 7, x: 64, y: 48 },
+            format: 'png',
+            fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.png',
+            mapConfig: createMapConfig({ png: 128, 'grid.json': 128 }),
+            assert: function (tile, callback) {
+                assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
+            }
+        },
+        {
+            desc: 'should get mvt tile using buffer-size 0',
+            coords: { z: 7, x: 64, y: 48 },
+            format: 'mvt',
+            fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.mvt',
+            mapConfig: createMapConfig({ mvt: 0 }),
+            assert: function (tile, callback) {
+                var tileJSON = tile.toJSON();
+                var features = tileJSON[0].features;
+                assert.equal(features.length, 1);
+                callback();
+            }
+        },
+        {
+            desc: 'should get mvt tile using buffer-size 128',
+            coords: { z: 7, x: 64, y: 48 },
+            format: 'mvt',
+            fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.mvt',
+            mapConfig: createMapConfig({ mvt: 128 }),
+            assert: function (tile, callback) {
+                var tileJSON = tile.toJSON();
+                var features = tileJSON[0].features;
+                assert.equal(features.length, 9);
+                callback();
+            }
+        },
+        {
+            desc: 'should get grid.json tile using buffer-size 0 overriden by template params',
+            coords: { z: 7, x: 64, y: 48 },
+            format: 'grid.json',
+            layers: [0],
+            fixturePath: './test/fixtures/buffer-size/tile-grid.json.7.64.48-buffer-size-0.grid.json',
+            mapConfig: createMapConfig({ 'grid.json': 0 }),
+            assert: function (tile, callback) {
+                assert.utfgridEqualsFile(tile, this.fixturePath, 2,callback);
+            }
+        },
+        {
+            desc: 'should get grid.json tile using buffer-size 128 overriden by template params',
+            coords: { z: 7, x: 64, y: 48 },
+            format: 'grid.json',
+            layers: [0],
+            fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.grid.json',
+            mapConfig: createMapConfig({ 'grid.json': 128 }),
+            assert: function (tile, callback) {
+                assert.utfgridEqualsFile(tile, this.fixturePath, 2, callback);
+            }
+        }
+    ];
+
+    afterEach(function(done) {
+        if (this.testClient) {
+            return this.testClient.drain(done);
+        }
+        return done();
+    });
+
+    const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS;
+    testCases.forEach(function (test) {
+        var testFn = (usePostGIS) => {
+            it(test.desc, function (done) {
+                serverOptions.renderer.mvt.usePostGIS = usePostGIS;
+                this.testClient = new TestClient(test.mapConfig, 1234);
+                serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS;
+                var coords = test.coords;
+                var options = {
+                    format: test.format,
+                    layers: test.layers
+                };
+                this.testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) {
+                    assert.ifError(err);
+                    // To generate images use:
+                    // tile.save(test.fixturePath);
+                    test.assert(tile, done);
+                });
+            });
+        };
+        if (process.env.POSTGIS_VERSION === '2.4' && test.format === 'mvt'){
+            testFn(true);
+        }
+        testFn(false);
+    });
+});
+
+function createBufferSizeTemplate (name, buffersize, placeholders, cartocss) {
+    cartocss = cartocss || CARTOCSS_LABELS;
+
+    return {
+        "version": "0.0.1",
+        "name": name,
+        "placeholders": placeholders || {
+            "buffersize": {
+                "type": "number",
+                "default": 0
+            }
+        },
+        "layergroup": createMapConfig(buffersize)
+    };
+}
+
+describe('buffer size per format for named maps', function () {
+    var testCases = [
+        {
+            desc: 'should get png tile using buffer-size 0 (default value in template)',
+            coords: { z: 7, x: 64, y: 48 },
+            format: 'png',
+            fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png',
+            template: createBufferSizeTemplate('named-default-buffer-size', {png: '<%= buffersize %>'}),
+            assert: function (tile, callback) {
+                assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
+            }
+        },
+        {
+            desc: 'should get png tile using buffer-size 128 (placehoder value)',
+            coords: { z: 7, x: 64, y: 48 },
+            format: 'png',
+            placeholders: { buffersize: 128 },
+            fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.png',
+            template: createBufferSizeTemplate('named-custom-buffer-size', { png: '<%= buffersize %>'}),
+            assert: function (tile, callback) {
+                assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
+            }
+        },
+        {
+            desc: 'should get png tile using buffer-size 0 (default value in template by format)',
+            coords: { z: 7, x: 64, y: 48 },
+            format: 'png',
+            placeholders: { buffersize_png: 0 },
+            fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png',
+            template: createBufferSizeTemplate('named-default-buffer-size-by-format', {
+                png: '<%= buffersize_png %>'
+            }, {
+                "buffersize_png": {
+                    "type": "number",
+                    "default": "0"
+                }
+            }),
+            assert: function (tile, callback) {
+                assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
+            }
+        },
+        {
+            desc: 'should get png tile using buffer-size 128 (placehoder value in template by format)',
+            coords: { z: 7, x: 64, y: 48 },
+            format: 'png',
+            placeholders: { buffersize_png: 128 },
+            fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.png',
+            template: createBufferSizeTemplate('named-custom-buffer-size-by-format', {
+                png: '<%= buffersize_png %>'
+            }, {
+                "buffersize_png": {
+                    "type": "number",
+                    "default": "0"
+                }
+            }),
+            assert: function (tile, callback) {
+                assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
+            }
+        },
+        {
+            desc: 'should get grid.json tile using buffer-size 0 overriden by template params',
+            coords: { z: 7, x: 64, y: 48 },
+            format: 'grid.json',
+            layers: [0],
+            placeholders: { buffersize_gridjson: 0 },
+            fixturePath: './test/fixtures/buffer-size/tile-grid.json.7.64.48-buffer-size-0.grid.json',
+            template: createBufferSizeTemplate('named-default-buffer-size-by-format-gridjson', {
+                'grid.json': '<%= buffersize_gridjson %>'
+            }, {
+                "buffersize_gridjson": {
+                    "type": "number",
+                    "default": "0"
+                }
+            }),
+            assert: function (tile, callback) {
+                assert.utfgridEqualsFile(tile, this.fixturePath, 2,callback);
+            }
+        },
+        {
+            desc: 'should get grid.json tile using buffer-size 128 overriden by template params',
+            coords: { z: 7, x: 64, y: 48 },
+            format: 'grid.json',
+            layers: [0],
+            placeholders: { buffersize_gridjson: 128 },
+            fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.grid.json',
+            template: createBufferSizeTemplate('named-custom-buffer-size-by-format-gridjson', {
+                'grid.json': '<%= buffersize_gridjson %>'
+            }, {
+                "buffersize_gridjson": {
+                    "type": "number",
+                    "default": "0"
+                }
+            }),
+            assert: function (tile, callback) {
+                assert.utfgridEqualsFile(tile, this.fixturePath, 2, callback);
+            }
+        }
+    ];
+
+    afterEach(function(done) {
+        if (this.testClient) {
+            return this.testClient.drain(done);
+        }
+        return done();
+    });
+
+    testCases.forEach(function (test) {
+        it(test.desc, function (done) {
+            this.testClient = new TestClient(test.template, 1234);
+            var coords = test.coords;
+            var options = {
+                format: test.format,
+                placeholders: test.placeholders,
+                layers: test.layers
+            };
+            this.testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) {
+                assert.ifError(err);
+                // To generate images use:
+                //tile.save('./test/fixtures/buffer-size/tile-7.64.48-buffer-size-0-test.png');
+                test.assert(tile, done);
+            });
+        });
+    });
+});
+
+
+describe('buffer size per format for named maps w/o placeholders', function () {
+    var testCases = [
+        {
+            desc: 'should get png tile using buffer-size 0 overriden by template params',
+            coords: { z: 7, x: 64, y: 48 },
+            format: 'png',
+            placeholders: {
+                buffersize: {
+                    png: 0
+                }
+            },
+            fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png',
+            template: createBufferSizeTemplate('named-no-buffer-size-png-0', {}, {}),
+            assert: function (tile, callback) {
+                assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
+            }
+        },
+        {
+            desc: 'should get png tile using buffer-size 128 overriden by template params',
+            coords: { z: 7, x: 64, y: 48 },
+            format: 'png',
+            placeholders: {
+                buffersize: {
+                    png: 128
+                }
+            },
+            fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.png',
+            template: createBufferSizeTemplate('named-no-buffer-size-png-128', {}, {}),
+            assert: function (tile, callback) {
+                assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
+            }
+        },
+        {
+            desc: 'should get mvt tile using buffer-size 0 overriden by template params',
+            coords: { z: 7, x: 64, y: 48 },
+            format: 'mvt',
+            placeholders: {
+                buffersize: {
+                    mvt: 0
+                }
+            },
+            fixturePath: './test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-0.mvt',
+            template: createBufferSizeTemplate('named-no-buffer-size-mvt', {}, {}),
+            assert: function (tile, callback) {
+                var tileJSON = tile.toJSON();
+                var features = tileJSON[0].features;
+
+                var dataFixture = fs.readFileSync(this.fixturePath);
+                var vtile = new mapnik.VectorTile(this.coords.z, this.coords.x, this.coords.y);
+                vtile.setDataSync(dataFixture);
+                var vtileJSON = vtile.toJSON();
+                var vtileFeatures = vtileJSON[0].features;
+
+                assert.equal(features.length, vtileFeatures.length);
+                callback();
+            }
+        },
+        {
+            desc: 'should get mvt tile using buffer-size 128 overriden by template params',
+            coords: { z: 7, x: 64, y: 48 },
+            format: 'mvt',
+            placeholders: {
+                buffersize: {
+                    mvt: 128
+                }
+            },
+            fixturePath: './test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-128.mvt',
+            template: createBufferSizeTemplate('named-no-buffer-size-mvt-128', {}, {}),
+            assert: function (tile, callback) {
+                var tileJSON = tile.toJSON();
+                var features = tileJSON[0].features;
+
+                var dataFixture = fs.readFileSync(this.fixturePath);
+                var vtile = new mapnik.VectorTile(this.coords.z, this.coords.x, this.coords.y);
+                vtile.setDataSync(dataFixture);
+                var vtileJSON = vtile.toJSON();
+                var vtileFeatures = vtileJSON[0].features;
+
+                assert.equal(features.length, vtileFeatures.length);
+                callback();
+            }
+        },
+        {
+            desc: 'should get grid.json tile using buffer-size 0 overriden by template params',
+            coords: { z: 7, x: 64, y: 48 },
+            format: 'grid.json',
+            layers: [0],
+            placeholders: {
+                buffersize: {
+                    'grid.json': 0
+                }
+            },
+            fixturePath: './test/fixtures/buffer-size/tile-grid.json.7.64.48-buffer-size-0.grid.json',
+            template: createBufferSizeTemplate('named-no-buffer-size-grid-json-0', {}, {}),
+            assert: function (tile, callback) {
+                assert.utfgridEqualsFile(tile, this.fixturePath, 2,callback);
+            }
+        },
+        {
+            desc: 'should get grid.json tile using buffer-size 128 overriden by template params',
+            coords: { z: 7, x: 64, y: 48 },
+            format: 'grid.json',
+            layers: [0],
+            placeholders: {
+                buffersize: {
+                    'grid.json': 128
+                }
+            },
+            fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.grid.json',
+            template: createBufferSizeTemplate('named-no-buffer-size-grid-json-128', {}, {}),
+            assert: function (tile, callback) {
+                assert.utfgridEqualsFile(tile, this.fixturePath, 2, callback);
+            }
+        },
+        {
+            desc: 'should get png tile using buffer-size 0' +
+                  ' overriden by template params with no buffersize in mapconfig',
+            coords: { z: 7, x: 64, y: 48 },
+            format: 'png',
+            placeholders: {
+                buffersize: {
+                    png: 0
+                }
+            },
+            fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png',
+            template: createBufferSizeTemplate('named-no-buffer-size-mapconfig-png-0', undefined, {}),
+            assert: function (tile, callback) {
+                assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
+            }
+        },
+
+    ];
+
+    afterEach(function(done) {
+        if (this.testClient) {
+            return this.testClient.drain(done);
+        }
+        return done();
+    });
+
+    const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS;
+    testCases.forEach(function (test) {
+        var testFn = (usePostGIS) => {
+                it(test.desc + `(${usePostGIS? 'PostGIS':'mapnik'})`, function (done) {
+                    serverOptions.renderer.mvt.usePostGIS = usePostGIS;
+                    test.template.name += '_1';
+                    this.testClient = new TestClient(test.template, 1234);
+                    serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS;
+                    var coords = test.coords;
+                    var options = {
+                        format: test.format,
+                        placeholders: test.placeholders,
+                        layers: test.layers
+                    };
+                    this.testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) {
+                        assert.ifError(err);
+                        // To generate images use:
+                        //tile.save(test.fixturePath);
+                        // require('fs').writeFileSync(test.fixturePath, JSON.stringify(tile));
+                        // require('fs').writeFileSync(test.fixturePath, tile.getDataSync());
+                        test.assert(tile, done);
+                    });
+                });
+        };
+        if (process.env.POSTGIS_VERSION === '2.4' && test.format === 'mvt'){
+            testFn(true);
+        }
+        testFn(false);
+    });
+});

test/acceptance/cache/cache_headers.js

@@ -8,7 +8,7 @@ var serverOptions = require('../../../lib/cartodb/server_options');
 var server = new CartodbWindshaft(serverOptions);
 server.setMaxListeners(0);
 
-var LayergroupToken = require('../../support/layergroup-token');
+var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token');
 
 describe('get requests with cache headers', function() {
 

test/acceptance/dataviews/aggregation.js

@@ -145,4 +145,182 @@ describe('aggregations happy cases', function() {
             });
         });
     });
+
+    var widgetSearchExpects = {
+        'count': [ { category: 'other_a', value: 3 } ],
+        'sum': [ { category: 'other_a', value: 6 } ],
+        'avg': [ { category: 'other_a', value: 2 } ],
+        'max': [ { category: 'other_a', value: 3 } ],
+        'min': [ { category: 'other_a', value: 1 } ]
+    };
+
+    Object.keys(operations_and_values).forEach(function (operation) {
+        var description = 'should search OTHER category using "' + operation + '"';
+
+        it(description, function (done) {
+            this.testClient = new TestClient(aggregationOperationMapConfig(operation, query_other, 'cat', 'val'));
+            this.testClient.widgetSearch('cat', 'other_a', function (err, res, searchResult) {
+                assert.ifError(err);
+
+                assert.ok(searchResult);
+                assert.equal(searchResult.type, 'aggregation');
+
+                assert.equal(searchResult.categories.length, 1);
+                assert.deepEqual(
+                    searchResult.categories,
+                    widgetSearchExpects[operation]
+                );
+                done();
+            });
+        });
+    });
+});
+
+describe('aggregation-dataview: special float values', function() {
+
+    afterEach(function(done) {
+        if (this.testClient) {
+            this.testClient.drain(done);
+        } else {
+            done();
+        }
+    });
+
+    function createMapConfig(layers, dataviews, analysis) {
+        return {
+            version: '1.5.0',
+            layers: layers,
+            dataviews: dataviews || {},
+            analyses: analysis || []
+        };
+    }
+
+    var mapConfig = createMapConfig(
+        [
+            {
+                "type": "cartodb",
+                "options": {
+                    "source": {
+                        "id": "a0"
+                    },
+                    "cartocss": "#points { marker-width: 10; marker-fill: red; }",
+                    "cartocss_version": "2.3.0"
+                }
+            }
+        ],
+        {
+            val_aggregation: {
+                source: {
+                    id: 'a0'
+                },
+                type: 'aggregation',
+                options: {
+                    column: 'cat',
+                    aggregation: 'avg',
+                    aggregationColumn: 'val'
+                }
+            },
+            sum_aggregation_numeric: {
+                source: {
+                    id: 'a1'
+                },
+                type: 'aggregation',
+                options: {
+                    column: 'cat',
+                    aggregation: 'sum',
+                    aggregationColumn: 'val'
+                }
+            }
+        },
+        [
+            {
+                "id": "a0",
+                "type": "source",
+                "params": {
+                    "query": [
+                        'SELECT',
+                        '  null::geometry the_geom_webmercator,',
+                        '  CASE',
+                        '    WHEN x % 4 = 0 THEN \'infinity\'::float',
+                        '    WHEN x % 4 = 1 THEN \'-infinity\'::float',
+                        '    WHEN x % 4 = 2 THEN \'NaN\'::float',
+                        '    ELSE x',
+                        '  END AS val,',
+                        '  CASE',
+                        '    WHEN x % 2 = 0 THEN \'category_1\'',
+                        '    ELSE \'category_2\'',
+                        '  END AS cat',
+                        'FROM generate_series(1, 1000) x'
+                    ].join('\n')
+                }
+            }, {
+                "id": "a1",
+                "type": "source",
+                "params": {
+                    "query": [
+                        'SELECT',
+                        '  null::geometry the_geom_webmercator,',
+                        '  CASE',
+                        '    WHEN x % 3 = 0 THEN \'NaN\'::numeric',
+                        '    WHEN x % 3 = 1 THEN x',
+                        '    ELSE x',
+                        '  END AS val,',
+                        '  CASE',
+                        '    WHEN x % 2 = 0 THEN \'category_1\'',
+                        '    ELSE \'category_2\'',
+                        '  END AS cat',
+                        'FROM generate_series(1, 1000) x'
+                    ].join('\n')
+                }
+            }
+        ]
+    );
+
+    // Source a0
+    // -----------------------------------------------
+    // the_geom_webmercator  |    val    |    cat
+    // ----------------------+-----------+------------
+    //                       | -Infinity | category_2
+    //                       |       NaN | category_1
+    //                       |         3 | category_2
+    //                       |  Infinity | category_1
+    //                       | -Infinity | category_2
+    //                       |       NaN | category_1
+    //                       |         7 | category_2
+    //                       |  Infinity | category_1
+    //                       | -Infinity | category_2
+    //                       |       NaN | category_1
+    //                       |        11 | category_2
+    //                       |         " |          "
+
+    var filters = [{ own_filter: 0 }, {}];
+    filters.forEach(function (filter) {
+        it('should handle special float values using filter: ' + JSON.stringify(filter), function(done) {
+            this.testClient = new TestClient(mapConfig, 1234);
+            this.testClient.getDataview('val_aggregation', { own_filter: 0 }, function(err, dataview) {
+                assert.ifError(err);
+                assert.ok(dataview.infinities === (250 + 250));
+                assert.ok(dataview.nans === 250);
+                assert.ok(dataview.categories.length === 1);
+                dataview.categories.forEach(function (category) {
+                    assert.ok(category.category === 'category_2');
+                    assert.ok(category.value === 501);
+                });
+                done();
+            });
+        });
+
+        it('should handle special numeric values using filter: ' + JSON.stringify(filter), function(done) {
+            this.testClient = new TestClient(mapConfig, 1234);
+            this.testClient.getDataview('sum_aggregation_numeric', { own_filter: 0 }, function(err, dataview) {
+                assert.ifError(err);
+                assert.ok(dataview.nans === 333);
+                assert.ok(dataview.categories.length === 2);
+                dataview.categories.forEach(function (category) {
+                    assert.ok(category.value !== null);
+                });
+                done();
+            });
+        });
+    });
 });

test/acceptance/dataviews/formula.js

@@ -0,0 +1,80 @@
+require('../../support/test_helper');
+var assert = require('../../support/assert');
+var TestClient = require('../../support/test-client');
+
+function createMapConfig(layers, dataviews, analysis) {
+    return {
+        version: '1.5.0',
+        layers: layers,
+        dataviews: dataviews || {},
+        analyses: analysis || []
+    };
+}
+
+describe('formula-dataview: special float values', function() {
+
+    afterEach(function(done) {
+        if (this.testClient) {
+            this.testClient.drain(done);
+        } else {
+            done();
+        }
+    });
+
+    var mapConfig = createMapConfig(
+        [
+            {
+                "type": "cartodb",
+                "options": {
+                    "source": {
+                        "id": "a0"
+                    },
+                    "cartocss": "#points { marker-width: 10; marker-fill: red; }",
+                    "cartocss_version": "2.3.0"
+                }
+            }
+        ],
+        {
+            val_formula: {
+                source: {
+                    id: 'a0'
+                },
+                type: 'formula',
+                options: {
+                    column: 'val',
+                    operation: 'avg'
+                }
+            }
+        },
+        [
+            {
+                "id": "a0",
+                "type": "source",
+                "params": {
+                    "query": [
+                        'SELECT',
+                        '  null::geometry the_geom_webmercator,',
+                        '  CASE',
+                        '    WHEN x % 4 = 0 THEN \'infinity\'::float',
+                        '    WHEN x % 4 = 1 THEN \'-infinity\'::float',
+                        '    WHEN x % 4 = 2 THEN \'NaN\'::float',
+                        '    ELSE x',
+                        '  END AS val',
+                        'FROM generate_series(1, 1000) x'
+                    ].join('\n')
+                }
+            }
+        ]
+    );
+
+    it('should filter infinities out and count them in the summary', function(done) {
+        this.testClient = new TestClient(mapConfig, 1234);
+        this.testClient.getDataview('val_formula', {}, function(err, dataview) {
+            assert.ok(!err, err);
+            assert.equal(dataview.result, 501);
+            assert.ok(dataview.infinities === (250 + 250));
+            assert.ok(dataview.nans === 250);
+            done();
+        });
+    });
+});

test/acceptance/dataviews/overviews.js

@@ -124,6 +124,13 @@ describe('dataviews using tables with overviews', function() {
                 params: {
                   query: 'select * from test_table_overviews'
                 }
+            },
+            {
+                id: 'data-source-special-float-values',
+                type: 'source',
+                params: {
+                  query: 'select * from test_special_float_values_table_overviews'
+                }
             }
         ],
         dataviews:  {
@@ -144,6 +151,17 @@ describe('dataviews using tables with overviews', function() {
                     aggregationColumn: 'name',
                 }
             },
+            test_categories_special_values: {
+                type: 'aggregation',
+                source: {
+                    id: 'data-source-special-float-values'
+                },
+                options: {
+                    column: 'name',
+                    aggregation: 'sum',
+                    aggregationColumn: 'value',
+                }
+            },
             test_histogram: {
                 type: 'histogram',
                 source: {id: 'data-source'},
@@ -160,6 +178,16 @@ describe('dataviews using tables with overviews', function() {
                     bins: 2
                 }
             },
+            test_histogram_special_values: {
+                type: 'histogram',
+                source: {
+                    id: 'data-source-special-float-values'
+                },
+                options: {
+                    column: 'value',
+                    bins: 2
+                }
+            },
             test_avg: {
                 type: 'formula',
                 source: {id: 'data-source'},
@@ -168,6 +196,16 @@ describe('dataviews using tables with overviews', function() {
                     operation: 'avg'
                 }
             },
+            test_formula_sum_special_values: {
+                type: 'formula',
+                source: {
+                    id: 'data-source-special-float-values'
+                },
+                options: {
+                    column: 'value',
+                    operation: 'sum'
+                }
+            },
             test_count: {
                 type: 'formula',
                 source: {id: 'data-source'},
@@ -202,6 +240,17 @@ describe('dataviews using tables with overviews', function() {
                     cartocss_version: '2.3.0',
                     source: { id: 'data-source' }
                 }
+            },
+            {
+                type: 'mapnik',
+                options: {
+                    sql: 'select * from test_special_float_values_table_overviews',
+                    cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
+                    cartocss_version: '2.3.0',
+                    source: {
+                        id: 'data-source-special-float-values'
+                    }
+                }
             }
         ]
     };
@@ -212,7 +261,14 @@ describe('dataviews using tables with overviews', function() {
             if (err) {
                 return done(err);
             }
-            assert.deepEqual(formula_result, {"operation":"sum","result":15,"nulls":0,"type":"formula"});
+            assert.deepEqual(formula_result, {
+                "operation":"sum",
+                "result":15,
+                "infinities": 0,
+                "nans": 0,
+                "nulls":0,
+                "type":"formula"
+            });
 
             testClient.drain(done);
         });
@@ -224,7 +280,14 @@ describe('dataviews using tables with overviews', function() {
             if (err) {
                 return done(err);
             }
-            assert.deepEqual(formula_result, {"operation":"avg","result":3,"nulls":0,"type":"formula"});
+            assert.deepEqual(formula_result, {
+                "operation":"avg",
+                "result":3,
+                "nulls":0,
+                "type":"formula",
+                "infinities": 0,
+                "nans": 0
+            });
 
             testClient.drain(done);
         });
@@ -236,7 +299,14 @@ describe('dataviews using tables with overviews', function() {
             if (err) {
                 return done(err);
             }
-            assert.deepEqual(formula_result, {"operation":"count","result":5,"nulls":0,"type":"formula"});
+            assert.deepEqual(formula_result, {
+                "operation":"count",
+                "result":5,
+                "nulls":0,
+                "type":"formula",
+                "infinities": 0,
+                "nans": 0
+            });
 
             testClient.drain(done);
         });
@@ -248,7 +318,14 @@ describe('dataviews using tables with overviews', function() {
             if (err) {
                 return done(err);
             }
-            assert.deepEqual(formula_result, {"operation":"max","result":5,"nulls":0,"type":"formula"});
+            assert.deepEqual(formula_result, {
+                "operation": "max",
+                "result": 5,
+                "nulls": 0,
+                "infinities": 0,
+                "nans": 0,
+                "type": "formula"
+            });
 
             testClient.drain(done);
         });
@@ -260,7 +337,14 @@ describe('dataviews using tables with overviews', function() {
             if (err) {
                 return done(err);
             }
-            assert.deepEqual(formula_result, {"operation":"min","result":1,"nulls":0,"type":"formula"});
+            assert.deepEqual(formula_result, {
+                "operation": "min",
+                "result": 1,
+                "nulls": 0,
+                "infinities": 0,
+                "nans": 0,
+                "type": "formula"
+            });
 
             testClient.drain(done);
         });
@@ -275,7 +359,14 @@ describe('dataviews using tables with overviews', function() {
             if (err) {
                 return done(err);
             }
-            assert.deepEqual(formula_result, {"operation":"sum","result":15,"nulls":0,"type":"formula"});
+            assert.deepEqual(formula_result, {
+                "operation":"sum",
+                "result":15,
+                "nulls":0,
+                "infinities": 0,
+                "nans": 0,
+                "type":"formula"
+            });
 
             testClient.drain(done);
         });
@@ -372,7 +463,14 @@ describe('dataviews using tables with overviews', function() {
                     if (err) {
                         return done(err);
                     }
-                    assert.deepEqual(formula_result, {"operation":"sum","result":1,"nulls":0,"type":"formula"});
+                    assert.deepEqual(formula_result, {
+                        "operation":"sum",
+                        "result":1,
+                        "nulls":0,
+                        "infinities": 0,
+                        "nans": 0,
+                        "type":"formula"
+                    });
                     testClient.drain(done);
                 });
             });
@@ -383,7 +481,14 @@ describe('dataviews using tables with overviews', function() {
                     if (err) {
                         return done(err);
                     }
-                    assert.deepEqual(formula_result, {"operation":"avg","result":1,"nulls":0,"type":"formula"});
+                    assert.deepEqual(formula_result, {
+                        "operation":"avg",
+                        "result":1,
+                        "nulls":0,
+                        "infinities": 0,
+                        "nans": 0,
+                        "type":"formula"
+                    });
 
                     testClient.drain(done);
                 });
@@ -395,7 +500,14 @@ describe('dataviews using tables with overviews', function() {
                     if (err) {
                         return done(err);
                     }
-                    assert.deepEqual(formula_result, {"operation":"count","result":1,"nulls":0,"type":"formula"});
+                    assert.deepEqual(formula_result, {
+                        "operation":"count",
+                        "result":1,
+                        "infinities": 0,
+                        "nans": 0,
+                        "nulls":0,
+                        "type":"formula"
+                    });
 
                     testClient.drain(done);
                 });
@@ -407,7 +519,14 @@ describe('dataviews using tables with overviews', function() {
                     if (err) {
                         return done(err);
                     }
-                    assert.deepEqual(formula_result, {"operation":"max","result":1,"nulls":0,"type":"formula"});
+                    assert.deepEqual(formula_result, {
+                        "operation": "max",
+                        "result": 1,
+                        "nulls": 0,
+                        "infinities": 0,
+                        "nans": 0,
+                        "type": "formula"
+                    });
 
                     testClient.drain(done);
                 });
@@ -419,7 +538,14 @@ describe('dataviews using tables with overviews', function() {
                     if (err) {
                         return done(err);
                     }
-                    assert.deepEqual(formula_result, {"operation":"min","result":1,"nulls":0,"type":"formula"});
+                    assert.deepEqual(formula_result, {
+                        "operation": "min",
+                        "result": 1,
+                        "nulls": 0,
+                        "infinities": 0,
+                        "nans": 0,
+                        "type": "formula"
+                    });
 
                     testClient.drain(done);
                 });
@@ -437,7 +563,14 @@ describe('dataviews using tables with overviews', function() {
                     if (err) {
                         return done(err);
                     }
-                    assert.deepEqual(formula_result, {"operation":"sum","result":1,"nulls":0,"type":"formula"});
+                    assert.deepEqual(formula_result, {
+                        "operation":"sum",
+                        "result":1,
+                        "nulls":0,
+                        "infinities": 0,
+                        "nans": 0,
+                        "type":"formula"
+                    });
                     testClient.drain(done);
                 });
             });
@@ -445,5 +578,69 @@ describe('dataviews using tables with overviews', function() {
 
         });
 
+        describe('aggregation special float values', function () {
+            var params = {};
+
+            it("should expose an aggregation dataview filtering special float values out", function (done) {
+                var testClient = new TestClient(overviewsMapConfig);
+                testClient.getDataview('test_categories_special_values', params, function (err, dataview) {
+                    if (err) {
+                        return done(err);
+                    }
+                    assert.deepEqual(dataview, {
+                        aggregation: 'sum',
+                        count: 5,
+                        nulls: 0,
+                        nans: 1,
+                        infinities: 1,
+                        min: 6,
+                        max: 6,
+                        categoriesCount: 1,
+                        categories: [ { category: 'Hawai', value: 6, agg: false } ],
+                        type: 'aggregation'
+                    });
+                    testClient.drain(done);
+                });
+            });
+
+            it('should expose a histogram dataview filtering special float values out', function (done) {
+                var testClient = new TestClient(overviewsMapConfig);
+                testClient.getDataview('test_histogram_special_values', params, function (err, dataview) {
+                    if (err) {
+                        return done(err);
+                    }
+                    assert.deepEqual(dataview, {
+                        bin_width: 0,
+                        bins_count: 1,
+                        bins_start: 3,
+                        nulls: 0,
+                        infinities: 1,
+                        nans: 1,
+                        avg: 3,
+                        bins: [ { bin: 0, min: 3, max: 3, avg: 3, freq: 2 } ],
+                        type: 'histogram'
+                    });
+                    testClient.drain(done);
+                });
+            });
+
+            it('should expose a formula (sum) dataview filtering special float values out', function (done) {
+                var testClient = new TestClient(overviewsMapConfig);
+                testClient.getDataview('test_formula_sum_special_values', params, function (err, dataview) {
+                    if (err) {
+                        return done(err);
+                    }
+                    assert.deepEqual(dataview, {
+                        operation: 'sum',
+                        result: 6,
+                        nulls: 0,
+                        nans: 1,
+                        infinities: 1,
+                        type: 'formula'
+                    });
+                    testClient.drain(done);
+                });
+            });
+        });
     });
 });

test/acceptance/dynamic-styling-named-maps.js

@@ -1,6 +1,6 @@
 var assert = require('../support/assert');
 var step = require('step');
-var LayergroupToken = require('../support/layergroup-token');
+var LayergroupToken = require('../../lib/cartodb/models/layergroup-token');
 var testHelper = require(__dirname + '/../support/test_helper');
 var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server');
 var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');

test/acceptance/geojson-renderer.js

@@ -1,343 +0,0 @@
-require('../support/test_helper');
-
-var assert = require('../support/assert');
-var TestClient = require('../support/test-client');
-
-
-describe('use only needed columns', function() {
-
-    function getFeatureByCartodbId(features, cartodbId) {
-        for (var i = 0, len = features.length; i < len; i++) {
-            if (features[i].properties.cartodb_id === cartodbId) {
-                return features[i];
-            }
-        }
-        return {};
-    }
-
-    var options = { format: 'geojson', layer: 0 };
-
-    afterEach(function(done) {
-        if (this.testClient) {
-            this.testClient.drain(done);
-        } else {
-            done();
-        }
-    });
-
-    it('with aggregation widget, interactivity and cartocss columns', function(done) {
-        var widgetMapConfig = {
-            version: '1.5.0',
-            layers: [{
-                type: 'mapnik',
-                options: {
-                    sql: 'select * from populated_places_simple_reduced',
-                    cartocss: '#layer0 { marker-fill: red; marker-width: 10; [name="Madrid"] { marker-fill: green; } }',
-                    cartocss_version: '2.0.1',
-                    widgets: {
-                        adm0name: {
-                            type: 'aggregation',
-                            options: {
-                                column: 'adm0name',
-                                aggregation: 'sum',
-                                aggregationColumn: 'pop_max'
-                            }
-                        }
-                    },
-                    interactivity: "cartodb_id,pop_min"
-                }
-            }]
-        };
-
-        this.testClient = new TestClient(widgetMapConfig);
-        this.testClient.getTile(0, 0, 0, options, function (err, res, geojsonTile) {
-            assert.ok(!err, err);
-            assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, {
-                cartodb_id: 1109,
-                name: 'Mardin',
-                adm0name: 'Turkey',
-                pop_max: 71373,
-                pop_min: 57586
-            });
-            done();
-        });
-    });
-
-    it('should not duplicate columns', function(done) {
-        var widgetMapConfig = {
-            version: '1.5.0',
-            layers: [{
-                type: 'mapnik',
-                options: {
-                    sql: 'select * from populated_places_simple_reduced',
-                    cartocss: ['#layer0 {',
-                        'marker-fill: red;',
-                        'marker-width: 10;',
-                        '[name="Madrid"] { marker-fill: green; } ',
-                        '[pop_max>100000] { marker-fill: black; } ',
-                        '}'].join('\n'),
-                    cartocss_version: '2.3.0',
-                    widgets: {
-                        adm0name: {
-                            type: 'aggregation',
-                            options: {
-                                column: 'adm0name',
-                                aggregation: 'sum',
-                                aggregationColumn: 'pop_max'
-                            }
-                        }
-                    },
-                    interactivity: "cartodb_id,pop_max"
-                }
-            }]
-        };
-
-        this.testClient = new TestClient(widgetMapConfig);
-        this.testClient.getTile(0, 0, 0, options, function (err, res, geojsonTile) {
-            assert.ok(!err, err);
-            assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, {
-                cartodb_id: 1109,
-                name: 'Mardin',
-                adm0name: 'Turkey',
-                pop_max: 71373
-            });
-            done();
-        });
-    });
-
-    it('with formula widget, no interactivity and no cartocss columns', function(done) {
-        var formulaWidgetMapConfig = {
-            version: '1.5.0',
-            layers: [{
-                type: 'mapnik',
-                options: {
-                    sql: 'select * from populated_places_simple_reduced where pop_max > 0 and pop_max < 600000',
-                    cartocss: '#layer0 { marker-fill: red; marker-width: 10; }',
-                    cartocss_version: '2.0.1',
-                    interactivity: 'cartodb_id',
-                    widgets: {
-                        pop_max_f: {
-                            type: 'formula',
-                            options: {
-                                column: 'pop_max',
-                                operation: 'count'
-                            }
-                        }
-                    }
-                }
-            }]
-        };
-
-        this.testClient = new TestClient(formulaWidgetMapConfig);
-        this.testClient.getTile(0, 0, 0, options, function (err, res, geojsonTile) {
-            assert.ok(!err, err);
-            assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, {
-                cartodb_id: 1109,
-                pop_max: 71373
-            });
-            done();
-        });
-    });
-    it('with cartocss with multiple expressions', function(done) {
-        var formulaWidgetMapConfig = {
-            version: '1.5.0',
-            layers: [{
-                type: 'mapnik',
-                options: {
-                    sql: 'select * from populated_places_simple_reduced where pop_max > 0 and pop_max < 600000',
-                    cartocss: '#layer0 { marker-fill: red; marker-width: 10; }' +
-                        '#layer0 { marker-width: 14; [name="Madrid"] { marker-width: 20; } }' +
-                        '#layer0[pop_max>1000] { marker-width: 14; [name="Madrid"] { marker-width: 20; } }' +
-                        '#layer0[adm0name=~".*Turkey*"] { marker-width: 14; [name="Madrid"] { marker-width: 20; } }',
-                    cartocss_version: '2.0.1',
-                    interactivity: 'cartodb_id'
-                }
-            }]
-        };
-
-        this.testClient = new TestClient(formulaWidgetMapConfig);
-        this.testClient.getTile(0, 0, 0, options, function (err, res, geojsonTile) {
-            assert.ok(!err, err);
-            assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, {
-                cartodb_id: 1109,
-                pop_max:71373,
-                name:"Mardin",
-                adm0name:"Turkey"
-            });
-            done();
-        });
-    });
-
-    it('should work with mapnik substitution tokens', function(done) {
-        var cartocss = [
-            "#layer {",
-            "  line-width: 2;",
-            "  line-color: #3B3B58;",
-            "  line-opacity: 1;",
-            "  polygon-opacity: 0.7;",
-            "  polygon-fill: ramp([points_count], (#E5F5F9,#99D8C9,#2CA25F))",
-            "}"
-        ].join('\n');
-
-        var sql = [
-            'WITH hgrid AS (',
-            '  SELECT CDB_HexagonGrid(',
-            '    ST_Expand(!bbox!, greatest(!pixel_width!,!pixel_height!) * 100),',
-            '    greatest(!pixel_width!,!pixel_height!) * 100',
-            '  ) as cell',
-            ')',
-            'SELECT',
-            '  hgrid.cell as the_geom_webmercator,',
-            '  count(1) as points_count,',
-            '  count(1)/power(100 * CDB_XYZ_Resolution(CDB_ZoomFromScale(!scale_denominator!)), 2) as points_density,',
-            '  1 as cartodb_id',
-            'FROM hgrid, (SELECT * FROM populated_places_simple_reduced) i',
-            'where ST_Intersects(i.the_geom_webmercator, hgrid.cell)',
-            'GROUP BY hgrid.cell'
-        ].join('\n');
-
-        var mapConfig = {
-            "version": "1.4.0",
-            "layers": [
-                {
-                    "type": 'mapnik',
-                    "options": {
-                        "cartocss_version": '2.3.0',
-                        "sql": sql,
-                        "cartocss": cartocss
-                    }
-                }
-            ]
-        };
-
-        this.testClient = new TestClient(mapConfig);
-        this.testClient.getTile(0, 0, 0, { format: 'geojson', layer: 0 }, function(err, res, geojson) {
-            assert.ok(!err, err);
-
-            assert.ok(geojson);
-            assert.equal(geojson.features.length, 5);
-
-            done();
-        });
-    });
-
-    it('should skip empty and null columns for geojson tiles', function(done) {
-
-        var mapConfig = {
-            "analyses": [
-                {
-                    "id": "a0",
-                    "params": {
-                        "query": "SELECT * FROM test_table"
-                    },
-                    "type": "source"
-                }
-            ],
-            "dataviews": {
-                "4e7b0e07-6d21-4b83-9adb-6d7e17eea6ca": {
-                    "options": {
-                        "aggregationColumn": null,
-                        "column": "cartodb_id",
-                        "operation": "avg"
-                    },
-                    "source": {
-                        "id": "a0"
-                    },
-                    "type": "formula"
-                },
-                "74f590f8-625c-4e95-922f-34ad3e9919c0": {
-                    "options": {
-                        "aggregation": "sum",
-                        "aggregationColumn": "cartodb_id",
-                        "column": "name"
-                    },
-                    "source": {
-                        "id": "a0"
-                    },
-                    "type": "aggregation"
-                },
-                "98a75757-3006-400a-b028-fb613a6c0b69": {
-                    "options": {
-                        "aggregationColumn": null,
-                        "column": "cartodb_id",
-                        "operation": "sum"
-                    },
-                    "source": {
-                        "id": "a0"
-                    },
-                    "type": "formula"
-                },
-                "ebbc97b2-87d2-4895-9e1f-2f012df3679d": {
-                    "options": {
-                        "aggregationColumn": null,
-                        "bins": "12",
-                        "column": "cartodb_id"
-                    },
-                    "source": {
-                        "id": "a0"
-                    },
-                    "type": "histogram"
-                },
-                "ebc0653f-3581-469c-8b31-c969e440a865": {
-                    "options": {
-                        "aggregationColumn": null,
-                        "column": "cartodb_id",
-                        "operation": "avg"
-                    },
-                    "source": {
-                        "id": "a0"
-                    },
-                    "type": "formula"
-                }
-            },
-            "layers": [
-                {
-                    "options": {
-                        "subdomains": "abcd",
-                        "urlTemplate": "http://{s}.basemaps.cartocdn.com/light_nolabels/{z}/{x}/{y}.png"
-                    },
-                    "type": "http"
-                },
-                {
-                    "options": {
-                        "attributes": {
-                            "columns": [
-                                "name",
-                                "address"
-                            ],
-                            "id": "cartodb_id"
-                        },
-                        "cartocss": "#layer { marker-width: 10; marker-fill: red; }",
-                        "cartocss_version": "2.3.0",
-                        "interactivity": "cartodb_id",
-                        "layer_name": "wadus",
-                        "source": {
-                            "id": "a0"
-                        }
-                    },
-                    "type": "cartodb"
-                },
-                {
-                    "options": {
-                        "subdomains": "abcd",
-                        "urlTemplate": "http://{s}.basemaps.cartocdn.com/light_only_labels/{z}/{x}/{y}.png"
-                    },
-                    "type": "http"
-                }
-            ]
-        };
-
-        this.testClient = new TestClient(mapConfig);
-        this.testClient.getTile(0, 0, 0, { format: 'geojson', layer: 0 }, function(err, res, geojson) {
-            assert.ok(!err, err);
-
-            assert.ok(geojson);
-            assert.equal(geojson.features.length, 5);
-
-            assert.deepEqual(Object.keys(geojson.features[0].properties), ['cartodb_id', 'name']);
-
-            done();
-        });
-    });
-
-});

test/acceptance/limits.js

@@ -1,311 +0,0 @@
-var testHelper = require('../support/test_helper');
-
-var assert = require('../support/assert');
-var _ = require('underscore');
-var redis = require('redis');
-
-var CartodbWindshaft = require('../../lib/cartodb/server');
-var serverOptions = require('../../lib/cartodb/server_options');
-
-var LayergroupToken = require('../support/layergroup-token');
-
-describe('render limits', function() {
-
-    var layergroupUrl = '/api/v1/map';
-
-    var redisClient = redis.createClient(global.environment.redis.port);
-
-    var server;
-    var keysToDelete;
-    beforeEach(function() {
-        keysToDelete = {};
-        server = new CartodbWindshaft(serverOptions);
-        server.setMaxListeners(0);
-    });
-
-    afterEach(function(done) {
-        testHelper.deleteRedisKeys(keysToDelete, done);
-    });
-
-    var user = 'localhost';
-
-    var pointSleepSql = "SELECT pg_sleep(0.5)," +
-        " 'SRID=3857;POINT(0 0)'::geometry the_geom_webmercator, 1 cartodb_id";
-    var pointCartoCss = '#layer { marker-fill:red; }';
-    var polygonSleepSql = "SELECT pg_sleep(0.5)," +
-        " ST_Buffer('SRID=3857;POINT(0 0)'::geometry, 100000000) the_geom_webmercator, 1 cartodb_id";
-    var polygonCartoCss = '#layer { polygon-fill:red; }';
-
-    function singleLayergroupConfig(sql, cartocss) {
-        return {
-            version: '1.0.0',
-            layers: [
-                {
-                    type: 'mapnik',
-                    options: {
-                        sql: sql,
-                        cartocss: cartocss,
-                        cartocss_version: '2.0.1'
-                    }
-                }
-            ]
-        };
-    }
-
-    function createRequest(layergroup, userHost) {
-        return {
-            url: layergroupUrl,
-            method: 'POST',
-            headers: {
-                host: userHost,
-                'Content-Type': 'application/json'
-            },
-            data: JSON.stringify(layergroup)
-        };
-    }
-
-    function withRenderLimit(user, renderLimit, callback) {
-        redisClient.SELECT(5, function(err) {
-            if (err) {
-                return callback(err);
-            }
-            var userLimitsKey = 'limits:tiler:' + user;
-            redisClient.HSET(userLimitsKey, 'render', renderLimit, function(err) {
-                if (err) {
-                    return callback(err);
-                }
-                keysToDelete[userLimitsKey] = 5;
-                return callback();
-            });
-        });
-
-    }
-
-    describe('with onTileErrorStrategy DISABLED', function() {
-        var onTileErrorStrategyEnabled;
-        before(function() {
-            onTileErrorStrategyEnabled = global.environment.enabledFeatures.onTileErrorStrategy;
-            global.environment.enabledFeatures.onTileErrorStrategy = false;
-        });
-
-        after(function() {
-            global.environment.enabledFeatures.onTileErrorStrategy = onTileErrorStrategyEnabled;
-        });
-
-        it("layergroup creation fails if test tile is slow", function(done) {
-            withRenderLimit(user, 50, function(err) {
-                if (err) {
-                    return done(err);
-                }
-
-                var layergroup = singleLayergroupConfig(polygonSleepSql, polygonCartoCss);
-                assert.response(server,
-                    createRequest(layergroup, user),
-                    {
-                        status: 400
-                    },
-                    function(res) {
-                        var parsed = JSON.parse(res.body);
-                        assert.deepEqual(parsed.errors, [ 'Render timed out' ]);
-                        done();
-                    }
-                );
-            });
-        });
-
-        it("layergroup creation does not fail if user limit is high enough even if test tile is slow", function(done) {
-            withRenderLimit(user, 5000, function(err) {
-                if (err) {
-                    return done(err);
-                }
-
-                var layergroup = singleLayergroupConfig(polygonSleepSql, polygonCartoCss);
-                assert.response(server,
-                    createRequest(layergroup, user),
-                    {
-                        status: 200
-                    },
-                    function(res) {
-                        var parsed = JSON.parse(res.body);
-                        assert.ok(parsed.layergroupid);
-                        keysToDelete['map_cfg|' + LayergroupToken.parse(parsed.layergroupid).token] = 0;
-                        keysToDelete['user:localhost:mapviews:global'] = 5;
-                        done();
-                    }
-                );
-            });
-        });
-
-
-        it("layergroup creation works if test tile is fast but tile request fails if they are slow",  function(done) {
-            withRenderLimit(user, 50, function(err) {
-                if (err) {
-                    return done(err);
-                }
-
-                var layergroup = singleLayergroupConfig(pointSleepSql, pointCartoCss);
-                assert.response(server,
-                    createRequest(layergroup, user),
-                    {
-                        status: 200
-                    },
-                    function(res) {
-                        keysToDelete['map_cfg|' + LayergroupToken.parse(JSON.parse(res.body).layergroupid).token] = 0;
-                        keysToDelete['user:localhost:mapviews:global'] = 5;
-                        assert.response(server,
-                            {
-                                url: layergroupUrl + _.template('/<%= layergroupId %>/<%= z %>/<%= x %>/<%= y %>.png', {
-                                    layergroupId: JSON.parse(res.body).layergroupid,
-                                    z: 0,
-                                    x: 0,
-                                    y: 0
-                                }),
-                                method: 'GET',
-                                headers: {
-                                    host: 'localhost'
-                                },
-                                encoding: 'binary'
-                            },
-                            {
-                                status: 400
-                            },
-                            function(res) {
-                                var parsed = JSON.parse(res.body);
-                                assert.deepEqual(parsed.errors, ['Render timed out']);
-                                done();
-                            }
-                        );
-
-                    }
-                );
-            });
-        });
-
-        it("tile request does not fail if user limit is high enough",  function(done) {
-            withRenderLimit(user, 5000, function(err) {
-                if (err) {
-                    return done(err);
-                }
-
-                var layergroup = singleLayergroupConfig(pointSleepSql, pointCartoCss);
-                assert.response(server,
-                    createRequest(layergroup, user),
-                    {
-                        status: 200
-                    },
-                    function(res) {
-                        keysToDelete['map_cfg|' + LayergroupToken.parse(JSON.parse(res.body).layergroupid).token] = 0;
-                        keysToDelete['user:localhost:mapviews:global'] = 5;
-                        assert.response(server,
-                            {
-                                url: layergroupUrl + _.template('/<%= layergroupId %>/<%= z %>/<%= x %>/<%= y %>.png', {
-                                    layergroupId: JSON.parse(res.body).layergroupid,
-                                    z: 0,
-                                    x: 0,
-                                    y: 0
-                                }),
-                                method: 'GET',
-                                headers: {
-                                    host: 'localhost'
-                                },
-                                encoding: 'binary'
-                            },
-                            {
-                                status: 200,
-                                headers: {
-                                    'Content-Type': 'image/png'
-                                }
-                            },
-                            function(res, err) {
-                                done(err);
-                            }
-                        );
-
-                    }
-                );
-            });
-        });
-
-    });
-
-    describe('with onTileErrorStrategy', function() {
-
-        it("layergroup creation works even if test tile is slow", function(done) {
-            withRenderLimit(user, 50, function(err) {
-                if (err) {
-                    return done(err);
-                }
-
-                var layergroup = singleLayergroupConfig(polygonSleepSql, polygonCartoCss);
-                assert.response(server,
-                    createRequest(layergroup, user),
-                    {
-                        status: 200
-                    },
-                    function(res) {
-                        var parsed = JSON.parse(res.body);
-                        assert.ok(parsed.layergroupid);
-                        keysToDelete['map_cfg|' + LayergroupToken.parse(parsed.layergroupid).token] = 0;
-                        keysToDelete['user:localhost:mapviews:global'] = 5;
-                        done();
-                    }
-                );
-            });
-        });
-
-        it("layergroup creation and tile requests works even if they are slow but returns fallback",  function(done) {
-            withRenderLimit(user, 50, function(err) {
-                if (err) {
-                    return done(err);
-                }
-
-                var layergroup = singleLayergroupConfig(pointSleepSql, pointCartoCss);
-                assert.response(server,
-                    createRequest(layergroup, user),
-                    {
-                        status: 200
-                    },
-                    function(res) {
-                        keysToDelete['map_cfg|' + LayergroupToken.parse(JSON.parse(res.body).layergroupid).token] = 0;
-                        keysToDelete['user:localhost:mapviews:global'] = 5;
-                        assert.response(server,
-                            {
-                                url: layergroupUrl + _.template('/<%= layergroupId %>/<%= z %>/<%= x %>/<%= y %>.png', {
-                                    layergroupId: JSON.parse(res.body).layergroupid,
-                                    z: 0,
-                                    x: 0,
-                                    y: 0
-                                }),
-                                method: 'GET',
-                                headers: {
-                                    host: 'localhost'
-                                },
-                                encoding: 'binary'
-                            },
-                            {
-                                status: 200,
-                                headers: {
-                                    'Content-Type': 'image/png'
-                                }
-                            },
-                            function(res, err) {
-                                if (err) {
-                                    done(err);
-                                }
-                                var referenceImagePath = './test/fixtures/render-timeout-fallback.png';
-                                assert.imageBufferIsSimilarToFile(res.body, referenceImagePath, 25,
-                                    function(imgErr/*, similarity*/) {
-                                        done(imgErr);
-                                    }
-                                );
-                            }
-                        );
-
-                    }
-                );
-            });
-        });
-
-    });
-
-});

test/acceptance/multilayer.js

@@ -9,7 +9,7 @@ var mapnik = require('windshaft').mapnik;
 var semver = require('semver');
 
 var helper = require(__dirname + '/../support/test_helper');
-var LayergroupToken = require('../support/layergroup-token');
+var LayergroupToken = require('../../lib/cartodb/models/layergroup-token');
 
 var windshaft_fixtures = __dirname + '/../../node_modules/windshaft/test/fixtures';
 
@@ -1041,7 +1041,7 @@ describe(suiteName, function() {
           );
         });
     }
-    
+
     // See https://github.com/CartoDB/Windshaft-cartodb/issues/91
     // and https://github.com/CartoDB/Windshaft-cartodb/issues/38
     it("tiles for private tables can be fetched with api_key", function(done) {

test/acceptance/multilayer_server.js

@@ -4,7 +4,7 @@ var assert = require('../support/assert');
 
 var _ = require('underscore');
 
-var LayergroupToken = require('../support/layergroup-token');
+var LayergroupToken = require('../../lib/cartodb/models/layergroup-token');
 
 var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner');
 var QueryTables = require('cartodb-query-tables');

test/acceptance/mvt.js

@@ -0,0 +1,507 @@
+require('../support/test_helper');
+
+var assert = require('../support/assert');
+var TestClient = require('../support/test-client');
+var serverOptions = require('../../lib/cartodb/server_options');
+
+function createMapConfig(sql = TestClient.SQL.ONE_POINT) {
+    return {
+        version: '1.6.0',
+        layers: [{
+            type: "cartodb",
+            options: {
+                sql: sql,
+                cartocss: TestClient.CARTOCSS.POINTS,
+                cartocss_version: '2.3.0',
+                interactivity: 'cartodb_id'
+            }
+        }]
+    };
+}
+
+describe('mvt (mapnik)', mvt(false));
+if (process.env.POSTGIS_VERSION === '2.4') {
+    describe('mvt (postgis)', mvt(true));
+}
+
+function mvt(usePostGIS) {
+return function () {
+    const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS;
+    before(function () {
+        serverOptions.renderer.mvt.usePostGIS = usePostGIS;
+    });
+    after(function (){
+        serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS;
+    });
+
+    describe('analysis-layers-dataviews-mvt', function () {
+
+        function createMapConfig(layers, dataviews, analysis) {
+            return {
+                version: '1.5.0',
+                layers: layers,
+                dataviews: dataviews || {},
+                analyses: analysis || []
+            };
+        }
+
+        var CARTOCSS = [
+            "#points {",
+            "  marker-fill-opacity: 1.0;",
+            "  marker-line-color: #FFF;",
+            "  marker-line-width: 0.5;",
+            "  marker-line-opacity: 1.0;",
+            "  marker-placement: point;",
+            "  marker-type: ellipse;",
+            "  marker-width: 8;",
+            "  marker-fill: red;",
+            "  marker-allow-overlap: true;",
+            "}"
+        ].join('\n');
+
+        var mapConfig = createMapConfig(
+            [
+                {
+                    "type": "cartodb",
+                    "options": {
+                        "source": {
+                            "id": "2570e105-7b37-40d2-bdf4-1af889598745"
+                        },
+                        "cartocss": CARTOCSS,
+                        "cartocss_version": "2.3.0"
+                    }
+                }
+            ],
+            {
+                pop_max_histogram: {
+                    source: {
+                        id: '2570e105-7b37-40d2-bdf4-1af889598745'
+                    },
+                    type: 'histogram',
+                    options: {
+                        column: 'pop_max'
+                    }
+                }
+            },
+            [
+                {
+                    "id": "2570e105-7b37-40d2-bdf4-1af889598745",
+                    "type": "source",
+                    "params": {
+                        "query": "select * from populated_places_simple_reduced"
+                    }
+                }
+            ]
+        );
+
+        it('should get pop_max column from dataview', function (done) {
+            var testClient = new TestClient(mapConfig);
+
+            testClient.getTile(0, 0, 0, { format: 'mvt', layers: 0 }, function (err, res, MVT) {
+                var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0));
+                assert.ok(!err, err);
+
+                assert.ok(Array.isArray(geojsonTile.features));
+                assert.ok(geojsonTile.features.length > 0);
+                var feature = geojsonTile.features[0];
+                assert.ok(feature.properties.hasOwnProperty('pop_max'), 'Missing pop_max property');
+
+                testClient.drain(done);
+            });
+        });
+
+    });
+
+
+    const testCases = [
+        {
+            desc: 'should get empty mvt with code 204 (no content)',
+            coords: { z: 0, x: 0, y: 0 },
+            format: 'mvt',
+            response: {
+                status: 204,
+                headers: {
+                    'Content-Type': undefined
+                }
+            },
+            mapConfig: createMapConfig(TestClient.SQL.EMPTY)
+        },
+        {
+            desc: 'should get mvt tile with code 200 (ok)',
+            coords: { z: 0, x: 0, y: 0 },
+            format: 'mvt',
+            response: {
+                status: 200,
+                headers: {
+                    'Content-Type': 'application/x-protobuf'
+                }
+            },
+            mapConfig: createMapConfig()
+        }
+    ];
+
+    testCases.forEach(function (test) {
+        it(test.desc, done => {
+            var testClient = new TestClient(test.mapConfig);
+            const { z, x, y } = test.coords;
+            const { format, response } = test;
+
+            testClient.getTile(z, x, y, { format, response }, err => {
+                assert.ifError(err);
+                testClient.drain(done);
+            });
+        });
+    });
+
+
+    if (usePostGIS){
+        describe('use only needed columns', onlyNeededColumns);
+    }else{
+        describe.skip('use only needed columns', onlyNeededColumns);
+    }
+
+    function onlyNeededColumns() {
+        function getFeatureByCartodbId(features, cartodbId) {
+            for (var i = 0, len = features.length; i < len; i++) {
+                if (features[i].properties.cartodb_id === cartodbId) {
+                    return features[i];
+                }
+            }
+            return {};
+        }
+
+        var options = { format: 'mvt', layer: 0 };
+
+        afterEach(function (done) {
+            if (this.testClient) {
+                this.testClient.drain(done);
+            } else {
+                done();
+            }
+        });
+
+        it('with aggregation widget, interactivity and cartocss columns', function (done) {
+            var widgetMapConfig = {
+                version: '1.5.0',
+                layers: [{
+                    type: 'mapnik',
+                    options: {
+                        sql: 'select * from populated_places_simple_reduced',
+                        cartocss:
+                            '#layer0 { marker-fill: red; marker-width: 10; [name="Madrid"] { marker-fill: green; } }',
+                        cartocss_version: '2.0.1',
+                        widgets: {
+                            adm0name: {
+                                type: 'aggregation',
+                                options: {
+                                    column: 'adm0name',
+                                    aggregation: 'sum',
+                                    aggregationColumn: 'pop_max'
+                                }
+                            }
+                        },
+                        interactivity: "cartodb_id,pop_min"
+                    }
+                }]
+            };
+
+            this.testClient = new TestClient(widgetMapConfig);
+            this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) {
+                var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0));
+                assert.ok(!err, err);
+                assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, {
+                    cartodb_id: 1109,
+                    name: 'Mardin',
+                    adm0name: 'Turkey',
+                    pop_max: 71373,
+                    pop_min: 57586
+                });
+                done();
+            });
+        });
+
+        it('should not duplicate columns', function (done) {
+            var widgetMapConfig = {
+                version: '1.5.0',
+                layers: [{
+                    type: 'mapnik',
+                    options: {
+                        sql: 'select * from populated_places_simple_reduced',
+                        cartocss: ['#layer0 {',
+                            'marker-fill: red;',
+                            'marker-width: 10;',
+                            '[name="Madrid"] { marker-fill: green; } ',
+                            '[pop_max>100000] { marker-fill: black; } ',
+                            '}'].join('\n'),
+                        cartocss_version: '2.3.0',
+                        widgets: {
+                            adm0name: {
+                                type: 'aggregation',
+                                options: {
+                                    column: 'adm0name',
+                                    aggregation: 'sum',
+                                    aggregationColumn: 'pop_max'
+                                }
+                            }
+                        },
+                        interactivity: "cartodb_id,pop_max"
+                    }
+                }]
+            };
+
+            this.testClient = new TestClient(widgetMapConfig);
+            this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) {
+                var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0));
+                assert.ok(!err, err);
+                assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, {
+                    cartodb_id: 1109,
+                    name: 'Mardin',
+                    adm0name: 'Turkey',
+                    pop_max: 71373
+                });
+                done();
+            });
+        });
+
+        it('with formula widget, no interactivity and no cartocss columns', function (done) {
+            var formulaWidgetMapConfig = {
+                version: '1.5.0',
+                layers: [{
+                    type: 'mapnik',
+                    options: {
+                        sql: 'select * from populated_places_simple_reduced where pop_max > 0 and pop_max < 600000',
+                        cartocss: '#layer0 { marker-fill: red; marker-width: 10; }',
+                        cartocss_version: '2.0.1',
+                        interactivity: 'cartodb_id',
+                        widgets: {
+                            pop_max_f: {
+                                type: 'formula',
+                                options: {
+                                    column: 'pop_max',
+                                    operation: 'count'
+                                }
+                            }
+                        }
+                    }
+                }]
+            };
+            this.testClient = new TestClient(formulaWidgetMapConfig);
+            this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) {
+                var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0));
+                assert.ok(!err, err);
+                assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, {
+                    cartodb_id: 1109,
+                    pop_max: 71373
+                });
+                done();
+            });
+        });
+        it('with cartocss with multiple expressions', function (done) {
+            var formulaWidgetMapConfig = {
+                version: '1.5.0',
+                layers: [{
+                    type: 'mapnik',
+                    options: {
+                        sql: 'select * from populated_places_simple_reduced where pop_max > 0 and pop_max < 600000',
+                        cartocss: '#layer0 { marker-fill: red; marker-width: 10; }' +
+                        '#layer0 { marker-width: 14; [name="Madrid"] { marker-width: 20; } }' +
+                        '#layer0[pop_max>1000] { marker-width: 14; [name="Madrid"] { marker-width: 20; } }' +
+                        '#layer0[adm0name=~".*Turkey*"] { marker-width: 14; [name="Madrid"] { marker-width: 20; } }',
+                        cartocss_version: '2.0.1',
+                        interactivity: 'cartodb_id'
+                    }
+                }]
+            };
+
+            this.testClient = new TestClient(formulaWidgetMapConfig);
+            this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) {
+                var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0));
+                assert.ok(!err, err);
+                assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, {
+                    cartodb_id: 1109,
+                    pop_max: 71373,
+                    name: "Mardin",
+                    adm0name: "Turkey"
+                });
+                done();
+            });
+        });
+
+        var skipOnPostGIS = usePostGIS ? it.skip: it;
+        skipOnPostGIS('should work with mapnik substitution tokens', function (done) {
+            var cartocss = [
+                "#layer {",
+                "  line-width: 2;",
+                "  line-color: #3B3B58;",
+                "  line-opacity: 1;",
+                "  polygon-opacity: 0.7;",
+                "  polygon-fill: ramp([points_count], (#E5F5F9,#99D8C9,#2CA25F))",
+                "}"
+            ].join('\n');
+
+            var sql = [
+            'WITH hgrid AS (',
+            '  SELECT CDB_HexagonGrid(',
+            '    ST_Expand(!bbox!, greatest(!pixel_width!,!pixel_height!) * 100),',
+            '    greatest(!pixel_width!,!pixel_height!) * 100',
+            '  ) as cell',
+            ')',
+            'SELECT',
+            '  hgrid.cell as the_geom_webmercator,',
+            '  count(1) as points_count,',
+            '  count(1)/power(100 * CDB_XYZ_Resolution(CDB_ZoomFromScale(!scale_denominator!)), 2) as points_density,',
+            '  1 as cartodb_id',
+            'FROM hgrid, (SELECT * FROM populated_places_simple_reduced) i',
+            'where ST_Intersects(i.the_geom_webmercator, hgrid.cell)',
+            'GROUP BY hgrid.cell'
+            ].join('\n');
+
+            var mapConfig = {
+                "version": "1.4.0",
+                "layers": [
+                    {
+                        "type": 'mapnik',
+                        "options": {
+                            "cartocss_version": '2.3.0',
+                            "sql": sql,
+                            "cartocss": cartocss
+                        }
+                    }
+                ]
+            };
+
+            this.testClient = new TestClient(mapConfig);
+            this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) {
+                var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0));
+                assert.ok(!err, err);
+
+                assert.ok(geojsonTile);
+                assert.equal(geojsonTile.features.length, 5);
+
+                done();
+            });
+        });
+
+        it('should skip empty and null columns for geojson tiles', function (done) {
+
+            var mapConfig = {
+                "analyses": [
+                    {
+                        "id": "a0",
+                        "params": {
+                            "query": "SELECT * FROM test_table"
+                        },
+                        "type": "source"
+                    }
+                ],
+                "dataviews": {
+                    "4e7b0e07-6d21-4b83-9adb-6d7e17eea6ca": {
+                        "options": {
+                            "aggregationColumn": null,
+                            "column": "cartodb_id",
+                            "operation": "avg"
+                        },
+                        "source": {
+                            "id": "a0"
+                        },
+                        "type": "formula"
+                    },
+                    "74f590f8-625c-4e95-922f-34ad3e9919c0": {
+                        "options": {
+                            "aggregation": "sum",
+                            "aggregationColumn": "cartodb_id",
+                            "column": "name"
+                        },
+                        "source": {
+                            "id": "a0"
+                        },
+                        "type": "aggregation"
+                    },
+                    "98a75757-3006-400a-b028-fb613a6c0b69": {
+                        "options": {
+                            "aggregationColumn": null,
+                            "column": "cartodb_id",
+                            "operation": "sum"
+                        },
+                        "source": {
+                            "id": "a0"
+                        },
+                        "type": "formula"
+                    },
+                    "ebbc97b2-87d2-4895-9e1f-2f012df3679d": {
+                        "options": {
+                            "aggregationColumn": null,
+                            "bins": "12",
+                            "column": "cartodb_id"
+                        },
+                        "source": {
+                            "id": "a0"
+                        },
+                        "type": "histogram"
+                    },
+                    "ebc0653f-3581-469c-8b31-c969e440a865": {
+                        "options": {
+                            "aggregationColumn": null,
+                            "column": "cartodb_id",
+                            "operation": "avg"
+                        },
+                        "source": {
+                            "id": "a0"
+                        },
+                        "type": "formula"
+                    }
+                },
+                "layers": [
+                    {
+                        "options": {
+                            "subdomains": "abcd",
+                            "urlTemplate": "http://{s}.basemaps.cartocdn.com/light_nolabels/{z}/{x}/{y}.png"
+                        },
+                        "type": "http"
+                    },
+                    {
+                        "options": {
+                            "attributes": {
+                                "columns": [
+                                    "name",
+                                    "address"
+                                ],
+                                "id": "cartodb_id"
+                            },
+                            "cartocss": "#layer { marker-width: 10; marker-fill: red; }",
+                            "cartocss_version": "2.3.0",
+                            "interactivity": "cartodb_id",
+                            "layer_name": "wadus",
+                            "source": {
+                                "id": "a0"
+                            }
+                        },
+                        "type": "cartodb"
+                    },
+                    {
+                        "options": {
+                            "subdomains": "abcd",
+                            "urlTemplate": "http://{s}.basemaps.cartocdn.com/light_only_labels/{z}/{x}/{y}.png"
+                        },
+                        "type": "http"
+                    }
+                ]
+            };
+
+            this.testClient = new TestClient(mapConfig);
+            this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) {
+                var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0));
+                assert.ok(!err, err);
+
+                assert.ok(geojsonTile);
+                assert.equal(geojsonTile.features.length, 5);
+
+                assert.deepEqual(Object.keys(geojsonTile.features[0].properties), ['cartodb_id', 'name']);
+
+                done();
+            });
+        });
+
+    }
+};
+}

test/acceptance/named_layers.js

@@ -5,7 +5,7 @@ var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server');
 var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
 var server = new CartodbWindshaft(serverOptions);
 
-var LayergroupToken = require('../support/layergroup-token');
+var LayergroupToken = require('../../lib/cartodb/models/layergroup-token');
 
 var RedisPool = require('redis-mpool');
 var TemplateMaps = require('../../lib/cartodb/backends/template_maps.js');

test/acceptance/named_maps_static_view.js

@@ -192,12 +192,36 @@ describe('named maps static view', function() {
             }
             getStaticMap({ zoom: 3 }, function(err, img) {
                 assert.ok(!err);
-                img.save('/tmp/static.png');
                 assert.imageIsSimilarToFile(img, previewFixture('override-zoom'), IMAGE_TOLERANCE, done);
             });
         });
     });
 
+    it('should return override bbox', function (done) {
+        var view = {
+            bounds: {
+                west: 0,
+                south: 0,
+                east: 45,
+                north: 45
+            },
+            zoom: 4,
+            center: {
+                lng: 40,
+                lat: 20
+            }
+        };
+        templateMaps.addTemplate(username, createTemplate(view), function (err) {
+            if (err) {
+                return done(err);
+            }
+            getStaticMap({ bbox: '0,45,90,45' }, function(err, img) {
+                assert.ok(!err);
+                assert.imageIsSimilarToFile(img, previewFixture('override-bbox'), IMAGE_TOLERANCE, done);
+            });
+        });
+    });
+
     it('should allow to select the layers to render', function (done) {
         var view = {
             bounds: {

test/acceptance/overviews_metadata.js

@@ -5,7 +5,7 @@ var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server');
 var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
 var server = new CartodbWindshaft(serverOptions);
 
-var LayergroupToken = require('../support/layergroup-token');
+var LayergroupToken = require('../../lib/cartodb/models/layergroup-token');
 
 var RedisPool = require('redis-mpool');
 

test/acceptance/overviews_metadata_named_maps.js

@@ -5,7 +5,7 @@ var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server');
 var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
 var server = new CartodbWindshaft(serverOptions);
 
-var LayergroupToken = require('../support/layergroup-token');
+var LayergroupToken = require('../../lib/cartodb/models/layergroup-token');
 
 var RedisPool = require('redis-mpool');
 

test/acceptance/ported/attributes.js

@@ -6,7 +6,7 @@ var cartodbServer = require('../../../lib/cartodb/server');
 var PortedServerOptions = require('./support/ported_server_options');
 var BaseController = require('../../../lib/cartodb/controllers/base');
 
-var LayergroupToken = require('../../support/layergroup-token');
+var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token');
 
 
 describe('attributes', function() {

test/acceptance/ported/multilayer.js

@@ -7,7 +7,7 @@ var step = require('step');
 var mapnik = require('windshaft').mapnik;
 var cartodbServer = require('../../../lib/cartodb/server');
 var ServerOptions = require('./support/ported_server_options');
-var LayergroupToken = require('../../support/layergroup-token');
+var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token');
 var BaseController = require('../../../lib/cartodb/controllers/base');
 
 describe('multilayer', function() {

test/acceptance/ported/multilayer_error_cases.js

@@ -5,6 +5,7 @@ var step = require('step');
 var cartodbServer = require('../../../lib/cartodb/server');
 var ServerOptions = require('./support/ported_server_options');
 var testClient = require('./support/test_client');
+var TestClient = require('../../support/test-client');
 
 var BaseController = require('../../../lib/cartodb/controllers/base');
 
@@ -23,6 +24,14 @@ describe('multilayer error cases', function() {
         BaseController.prototype.req2params = req2paramsFn;
     });
 
+    // var client = null;
+    afterEach(function(done) {
+        if (this.client) {
+            return this.client.drain(done);
+        }
+        return done();
+    });
+
     it("post layergroup with wrong Content-Type", function(done) {
         assert.response(server, {
             url: '/database/windshaft_test/layergroup',
@@ -153,24 +162,16 @@ describe('multilayer error cases', function() {
         ]
       };
       ServerOptions.afterLayergroupCreateCalls = 0;
-      assert.response(server, {
-          url: '/database/windshaft_test/layergroup',
-          method: 'POST',
-          headers: {'Content-Type': 'application/json' },
-          data: JSON.stringify(layergroup)
-      }, {}, function(res) {
-        try {
-          assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body);
-          // See http://github.com/CartoDB/Windshaft/issues/159
-          assert.equal(ServerOptions.afterLayergroupCreateCalls, 0);
-          var parsed = JSON.parse(res.body);
-          assert.ok(parsed);
-          assert.equal(parsed.errors.length, 1);
-          var error = parsed.errors[0];
-          assert.ok(error.match(/column "missing" does not exist/m), error);
-          // TODO: check which layer introduced the problem ?
-          done();
-        } catch (err) { done(err); }
+      this.client = new TestClient(layergroup);
+      this.client.getLayergroup({status: 400}, function(err, parsed) {
+        assert.ok(!err, err);
+        // See http://github.com/CartoDB/Windshaft/issues/159
+        assert.equal(ServerOptions.afterLayergroupCreateCalls, 0);
+        assert.ok(parsed);
+        assert.equal(parsed.errors.length, 1);
+        var error = parsed.errors[0];
+        assert.ok(error.match(/column "missing" does not exist/m), error);
+        done();
       });
     });
 

test/acceptance/ported/multilayer_interactivity.js

@@ -5,7 +5,7 @@ var _             = require('underscore');
 var cartodbServer = require('../../../lib/cartodb/server');
 var getLayerTypeFn = require('windshaft').model.MapConfig.prototype.getType;
 var PortedServerOptions = require('./support/ported_server_options');
-var LayergroupToken = require('../../support/layergroup-token');
+var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token');
 
 var BaseController = require('../../../lib/cartodb/controllers/base');
 

test/acceptance/ported/raster.js

@@ -6,7 +6,7 @@ var cartodbServer = require('../../../lib/cartodb/server');
 var ServerOptions = require('./support/ported_server_options');
 
 var BaseController = require('../../../lib/cartodb/controllers/base');
-var LayergroupToken = require('../../support/layergroup-token');
+var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token');
 
 describe('raster', function() {
 

test/acceptance/ported/retina.js

@@ -6,7 +6,7 @@ var cartodbServer = require('../../../lib/cartodb/server');
 var ServerOptions = require('./support/ported_server_options');
 
 var BaseController = require('../../../lib/cartodb/controllers/base');
-var LayergroupToken = require('../../support/layergroup-token');
+var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token');
 
 describe('retina support', function() {
 

test/acceptance/ported/server_png8_format.js

@@ -7,7 +7,7 @@ var cartodbServer = require('../../../lib/cartodb/server');
 var ServerOptions = require('./support/ported_server_options');
 
 var BaseController = require('../../../lib/cartodb/controllers/base');
-var LayergroupToken = require('../../support/layergroup-token');
+var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token');
 
 var IMAGE_EQUALS_TOLERANCE_PER_MIL = 85;
 
@@ -16,13 +16,13 @@ describe('server_png8_format', function() {
     var serverOptionsPng32 = ServerOptions;
     serverOptionsPng32.grainstore = _.clone(ServerOptions.grainstore);
     serverOptionsPng32.grainstore.mapnik_tile_format = 'png32';
-    var serverPng32 = new cartodbServer(serverOptionsPng32);
+    var serverPng32 = cartodbServer(serverOptionsPng32);
     serverPng32.setMaxListeners(0);
 
     var serverOptionsPng8 = ServerOptions;
     serverOptionsPng8.grainstore = _.clone(ServerOptions.grainstore);
     serverOptionsPng8.grainstore.mapnik_tile_format = 'png8:m=h';
-    var serverPng8 = new cartodbServer(serverOptionsPng8);
+    var serverPng8 = cartodbServer(serverOptionsPng8);
     serverPng8.setMaxListeners(0);
 
 

test/acceptance/ported/support/ported_server_options.js

@@ -1,7 +1,7 @@
 var _ = require('underscore');
 var serverOptions = require('../../../../lib/cartodb/server_options');
-var LayergroupToken = require('../../../support/layergroup-token');
 var mapnik = require('windshaft').mapnik;
+var LayergroupToken = require('../../../../lib/cartodb/models/layergroup-token');
 var OverviewsQueryRewriter = require('../../../../lib/cartodb/utils/overviews_query_rewriter');
 var overviewsQueryRewriter = new OverviewsQueryRewriter({
   zoom_level: 'CDB_ZoomFromScale(!scale_denominator!)'
@@ -48,7 +48,7 @@ module.exports = _.extend({}, serverOptions, {
     log_format: null, // do not log anything
     afterLayergroupCreateCalls: 0,
     useProfiler: true,
-    req2params: function(req, callback){
+    req2params: function(req, res, callback){
 
         if ( req.query.testUnexpectedError ) {
             return callback('test unexpected error');
@@ -56,13 +56,14 @@ module.exports = _.extend({}, serverOptions, {
 
         // this is in case you want to test sql parameters eg ...png?sql=select * from my_table limit 10
         req.params =  _.extend({}, req.params);
+
         if (req.params.token) {
             req.params.token = LayergroupToken.parse(req.params.token).token;
         }
 
         _.extend(req.params, req.query);
         req.params.user = 'localhost';
-        req.context = {user: 'localhost'};
+        res.locals.user = 'localhost';
 
         req.params.dbhost = global.environment.postgres.host;
         req.params.dbport = req.params.dbport || global.environment.postgres.port;
@@ -73,6 +74,9 @@ module.exports = _.extend({}, serverOptions, {
         }
         req.params.dbname = 'test_windshaft_cartodb_user_1_db';
 
+        // add all params to res.locals
+        res.locals = _.extend({}, req.params);
+
 
         // increment number of calls counter
         global.req2params_calls = global.req2params_calls ? global.req2params_calls + 1 : 1;

test/acceptance/ported/support/test_client.js

@@ -1,5 +1,5 @@
 var testHelper = require('../../../support/test_helper');
-var LayergroupToken = require('../../../support/layergroup-token');
+var LayergroupToken = require('../../../../lib/cartodb/models/layergroup-token');
 
 var step = require('step');
 var assert = require('../../../support/assert');