Release 2.53.2

Upgrades camshaft to 0.22.1

.travis.yml

@@ -1,9 +1,10 @@
 sudo: false
 
 addons:
-  postgresql: "9.3"
+  postgresql: "9.4"
   apt:
     packages:
+      - postgresql-plpython-9.4
       - pkg-config
       - libcairo2-dev
       - libjpeg8-dev

NEWS.md

@@ -1,5 +1,150 @@
 # Changelog
 
+## 2.53.2
+
+Released 2016-06-28
+
+Announcements:
+ - Upgrades camshaft to [0.22.2](https://github.com/CartoDB/camshaft/releases/tag/0.22.2)
+
+
+## 2.53.1
+
+Released 2016-06-28
+
+Announcements:
+ - Upgrades camshaft to [0.22.1](https://github.com/CartoDB/camshaft/releases/tag/0.22.1)
+
+
+## 2.53.0
+
+Released 2016-06-24
+
+Announcements:
+ - Upgrades camshaft to [0.22.0](https://github.com/CartoDB/camshaft/releases/tag/0.22.0)
+
+
+## 2.52.0
+
+Released 2016-06-23
+
+Announcements:
+ - Upgrades camshaft to [0.21.0](https://github.com/CartoDB/camshaft/releases/tag/0.21.0)
+
+
+## 2.51.0
+
+Released 2016-06-21
+
+Enhancements:
+ - Split turbo-carto adapter substitutions tokens query.
+ - Now errors with context have the same schema. #519
+
+Announcements:
+ - Upgrades camshaft to [0.20.0](https://github.com/CartoDB/camshaft/releases/tag/0.20.0)
+
+
+## 2.50.0
+
+Released 2016-06-21
+
+Bug fixes:
+ - Pixel size query for turbo-carto adapter using radians and degrees instead of meters.
+
+New features:
+ - Add support for min, max, and avg operations in aggregation dataview #513.
+
+Announcements:
+ - Upgrades camshaft to [0.19.0](https://github.com/CartoDB/camshaft/releases/tag/0.19.0)
+
+
+## 2.49.1
+
+Released 2016-06-20
+
+Announcements:
+ - Upgrades turbo-carto to [0.12.1](https://github.com/CartoDB/turbo-carto/releases/tag/0.12.1).
+
+Bug fixes:
+ - Use an empty array as default value for falsy ramps #512.
+ - Use the_geom for intermediate dataviews #511.
+ - Pick last update time for layergroupid from analyses results #510.
+
+
+## 2.49.0
+
+Released 2016-06-15
+
+Announcements:
+ - Upgrades camshaft to [0.17.1](https://github.com/CartoDB/camshaft/releases/tag/0.17.1)
+
+
+## 2.48.0
+
+Released 2016-06-14
+
+Announcements:
+ - Upgrades camshaft to [0.15.1](https://github.com/CartoDB/camshaft/releases/tag/0.15.1)
+ - Responses with more context info if analysis or turbo-carto fails during map creation.
+
+## 2.47.1
+
+Released 2016-06-13
+
+Announcements:
+ - Upgrades camshaft to [0.14.1](https://github.com/CartoDB/camshaft/releases/tag/0.14.1)
+
+
+## 2.47.0
+
+Released 2016-06-10
+
+Announcements:
+ - Upgrades camshaft to [0.14.0](https://github.com/CartoDB/camshaft/releases/tag/0.14.0)
+
+
+## 2.46.0
+
+Released 2016-06-09
+
+Improvements:
+ - Support for substitution tokens in geojson tiles
+ - Warn on application start about non-matching dependencies
+
+Announcements:
+ - Upgrades windshaft to [2.3.0](https://github.com/CartoDB/camshaft/releases/tag/2.3.0)
+ - Upgrades camshaft to [0.13.0](https://github.com/CartoDB/camshaft/releases/tag/0.13.0)
+ - Upgrades turbo-carto to [0.11.0](https://github.com/CartoDB/turbo-carto/releases/tag/0.11.0)
+
+Bug fixes:
+ - Column provided for geojson renderer should not be null #476
+ - Dataviews/widgets adapter working with non sql, non source, and non widgets layers
+
+
+## 2.45.0
+
+Released 2016-06-02
+
+Improvements:
+ - Removes Windshaft's widgets dependency.
+ - Makes widgets/dataviews endpoint compatible, but all using dataviews backend instead of widgets from Windshaft.
+ - Keeps adding widgets metadata in map instantiations for old clients.
+
+Announcements:
+ - Upgrades windshaft to [2.0.1](https://github.com/CartoDB/camshaft/releases/tag/2.0.1 )
+ - Upgrades camshaft to [0.12.1](https://github.com/CartoDB/camshaft/releases/tag/0.12.1)
+ - Upgrades turbo-carto to [0.10.1](https://github.com/CartoDB/turbo-carto/releases/tag/0.10.1)
+
+
+## 2.44.1
+
+Released 2016-06-01
+
+Improvements:
+ - Extend overviews support to histogram and aggregation dataviews
+ - Test improvements
+
+
 ## 2.44.0
 
 Released 2016-05-31

lib/cartodb/backends/dataview.js

@@ -16,6 +16,9 @@ var overviewsQueryRewriter = new OverviewsQueryRewriter({
     zoom_level: 'CDB_ZoomFromScale(!scale_denominator!)'
 });
 
+var dot = require('dot');
+dot.templateSettings.strip = false;
+
 function DataviewBackend(analysisBackend) {
     this.analysisBackend = analysisBackend;
 }
@@ -105,15 +108,7 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param
             var ownFilter = +params.own_filter;
             ownFilter = !!ownFilter;
 
-            var query;
-
-            if (ownFilter) {
-                query = node.getQuery();
-            } else {
-                var applyFilters = {};
-                applyFilters[dataviewName] = false;
-                query = node.getQuery(applyFilters);
-            }
+            var query = layerQuery(node, dataviewName, ownFilter);
 
             var sourceId =  dataviewDefinition.source.id; // node.id
             var layer = _.find(
@@ -137,14 +132,14 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param
             }
 
             if (params.bbox) {
-                var bboxFilter = new BBoxFilter({column: 'the_geom', srid: 4326}, {bbox: params.bbox});
+                var bboxFilter = new BBoxFilter({column: 'the_geom_webmercator', srid: 3857}, {bbox: params.bbox});
                 query = bboxFilter.sql(query);
                 if ( queryRewriteData ) {
                     var bbox_filter_definition = {
                         type: 'bbox',
                         options: {
-                            column: 'the_geom',
-                            srid: 4326,
+                            column: 'the_geom_webmercator',
+                            srid: 3857
                         },
                         params: {
                             bbox: params.bbox
@@ -260,14 +255,7 @@ DataviewBackend.prototype.search = function (mapConfigProvider, user, params, ca
             var ownFilter = +params.own_filter;
             ownFilter = !!ownFilter;
 
-            var query;
-            if (ownFilter) {
-                query = node.getQuery();
-            } else {
-                var applyFilters = {};
-                applyFilters[dataviewName] = false;
-                query = node.getQuery(applyFilters);
-            }
+            var query = layerQuery(node, dataviewName, ownFilter);
 
             if (params.bbox) {
                 var bboxFilter = new BBoxFilter({column: 'the_geom', srid: 4326}, {bbox: params.bbox});
@@ -321,3 +309,31 @@ function dbParamsFromReqParams(params) {
     }
     return dbParams;
 }
+
+var SKIP_COLUMNS = {
+    'the_geom': true,
+    'the_geom_webmercator': true
+};
+
+function skipColumns(columnNames) {
+    return columnNames
+        .filter(function(columnName) { return !SKIP_COLUMNS[columnName]; });
+}
+
+var layerQueryTemplate = dot.template([
+    'SELECT {{=it._columns}}',
+    'FROM ({{=it._query}}) _cdb_analysis_query'
+].join('\n'));
+
+function layerQuery(node, dataviewName, ownFilter) {
+    var applyFilters = {};
+    if (!ownFilter) {
+        applyFilters[dataviewName] = false;
+    }
+
+    if (node.type === 'source') {
+        return node.getQuery(applyFilters);
+    }
+    var _columns = ['ST_Transform(the_geom, 3857) the_geom_webmercator'].concat(skipColumns(node.getColumns()));
+    return layerQueryTemplate({ _query: node.getQuery(applyFilters), _columns: _columns.join(', ') });
+}

lib/cartodb/backends/turbo-carto-postgres-datasource.js

@@ -54,9 +54,8 @@ var method2strategy = {
     category: STRATEGY.EXACT
 };
 
-function PostgresDatasource (pgQueryRunner, username, query) {
-    this.pgQueryRunner = pgQueryRunner;
-    this.username = username;
+function PostgresDatasource (psql, query) {
+    this.psql = psql;
     this.query = query;
 }
 
@@ -75,13 +74,16 @@ PostgresDatasource.prototype.getRamp = function (column, buckets, method, callba
 
     var query = template({ _column: column, _sql: this.query, _buckets: buckets });
 
-    this.pgQueryRunner.run(this.username, query, function (err, result) {
+    this.psql.query(query, function (err, resultSet) {
         if (err) {
             return callback(err);
         }
 
+        resultSet = resultSet || {};
+        var result = resultSet.rows || [];
+
         var strategy = method2strategy[methodName];
-        var ramp = result[0][methodName];
+        var ramp = result[0][methodName] || [];
         if (strategy !== STRATEGY.EXACT) {
             ramp = ramp.sort(function(a, b) {
                 return a - b;
@@ -89,7 +91,7 @@ PostgresDatasource.prototype.getRamp = function (column, buckets, method, callba
         }
 
         return callback(null, { ramp: ramp, strategy: strategy });
-    });
+    }, true); // use read-only transaction
 };
 
 module.exports = PostgresDatasource;

lib/cartodb/controllers/base.js

@@ -214,15 +214,15 @@ BaseController.prototype.sendError = function(req, res, err, label) {
         statusCode = 200;
     }
 
-    var errorResponseBody = { errors: allErrors.map(errorMessage) };
+    var errorResponseBody = {
+        errors: allErrors.map(errorMessage),
+        errors_with_context: allErrors.map(errorMessageWithContext)
+    };
 
     this.send(req, res, errorResponseBody, statusCode);
 };
 
-function errorMessage(err) {
-    // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
-    var message = (_.isString(err) ? err : err.message) || 'Unknown error';
-
+function stripConnectionInfo(message) {
     // Strip connection info, if any
     return message
         // See https://github.com/CartoDB/Windshaft/issues/173
@@ -230,6 +230,24 @@ function errorMessage(err) {
         // See https://travis-ci.org/CartoDB/Windshaft/jobs/20703062#L1644
         .replace(/is the server.*encountered/im, 'encountered');
 }
+
+function errorMessage(err) {
+    // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
+    var message = (_.isString(err) ? err : err.message) || 'Unknown error';
+
+    return stripConnectionInfo(message);
+}
+
+function errorMessageWithContext(err) {
+    // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
+    var message = (_.isString(err) ? err : err.message) || 'Unknown error';
+
+    return {
+        type: err.type || 'unknown',
+        message: stripConnectionInfo(message),
+        context: err.context || 'unknown'
+    };
+}
 module.exports.errorMessage = errorMessage;
 
 function findStatusCode(err) {

lib/cartodb/controllers/layergroup.js

@@ -21,7 +21,6 @@ var QueryTables = require('cartodb-query-tables');
  * @param {TileBackend} tileBackend
  * @param {PreviewBackend} previewBackend
  * @param {AttributesBackend} attributesBackend
- * @param {WidgetBackend} widgetBackend
  * @param {SurrogateKeysCache} surrogateKeysCache
  * @param {UserLimitsApi} userLimitsApi
  * @param {LayergroupAffectedTables} layergroupAffectedTables
@@ -29,7 +28,7 @@ var QueryTables = require('cartodb-query-tables');
  * @constructor
  */
 function LayergroupController(authApi, pgConnection, mapStore, tileBackend, previewBackend, attributesBackend,
-                          widgetBackend, surrogateKeysCache, userLimitsApi, layergroupAffectedTables, analysisBackend) {
+                              surrogateKeysCache, userLimitsApi, layergroupAffectedTables, analysisBackend) {
     BaseController.call(this, authApi, pgConnection);
 
     this.pgConnection = pgConnection;
@@ -37,7 +36,6 @@ function LayergroupController(authApi, pgConnection, mapStore, tileBackend, prev
     this.tileBackend = tileBackend;
     this.previewBackend = previewBackend;
     this.attributesBackend = attributesBackend;
-    this.widgetBackend = widgetBackend;
     this.surrogateKeysCache = surrogateKeysCache;
     this.userLimitsApi = userLimitsApi;
     this.layergroupAffectedTables = layergroupAffectedTables;
@@ -78,21 +76,19 @@ LayergroupController.prototype.register = function(app) {
 
     // Undocumented/non-supported API endpoint methods.
     // Use at your own peril.
-    app.get(app.base_url_mapconfig +
-        '/:token/:layer/widget/:widgetName', cors(), userMiddleware,
-        this.widget.bind(this));
-
-    app.get(app.base_url_mapconfig +
-            '/:token/:layer/widget/:widgetName/search', cors(), userMiddleware,
-        this.widgetSearch.bind(this));
-
     app.get(app.base_url_mapconfig +
             '/:token/dataview/:dataviewName', cors(), userMiddleware,
         this.dataview.bind(this));
+    app.get(app.base_url_mapconfig +
+            '/:token/:layer/widget/:dataviewName', cors(), userMiddleware,
+        this.dataview.bind(this));
 
     app.get(app.base_url_mapconfig +
             '/:token/dataview/:dataviewName/search', cors(), userMiddleware,
         this.dataviewSearch.bind(this));
+    app.get(app.base_url_mapconfig +
+            '/:token/:layer/widget/:dataviewName/search', cors(), userMiddleware,
+        this.dataviewSearch.bind(this));
 
     app.get(app.base_url_mapconfig +
         '/:token/analysis/node/:nodeId', cors(), userMiddleware,
@@ -181,62 +177,6 @@ LayergroupController.prototype.dataviewSearch = function(req, res) {
 
 };
 
-LayergroupController.prototype.widget = function(req, res) {
-    var self = this;
-
-    step(
-        function setupParams() {
-            self.req2params(req, this);
-        },
-        function retrieveList(err) {
-            assert.ifError(err);
-
-            var mapConfigProvider = new MapStoreMapConfigProvider(
-                self.mapStore, req.context.user, self.userLimitsApi, req.params
-            );
-            self.widgetBackend.getWidget(mapConfigProvider, req.params, this);
-        },
-        function finish(err, widget, stats) {
-            req.profiler.add(stats || {});
-
-            if (err) {
-                self.sendError(req, res, err, 'GET WIDGET');
-            } else {
-                self.sendResponse(req, res, widget, 200);
-            }
-        }
-    );
-
-};
-
-LayergroupController.prototype.widgetSearch = function(req, res) {
-    var self = this;
-
-    step(
-        function setupParams() {
-            self.req2params(req, this);
-        },
-        function retrieveList(err) {
-            assert.ifError(err);
-
-            var mapConfigProvider = new MapStoreMapConfigProvider(
-                self.mapStore, req.context.user, self.userLimitsApi, req.params
-            );
-            self.widgetBackend.search(mapConfigProvider, req.params, this);
-        },
-        function finish(err, searchResult, stats) {
-            req.profiler.add(stats || {});
-
-            if (err) {
-                self.sendError(req, res, err, 'GET WIDGET');
-            } else {
-                self.sendResponse(req, res, searchResult, 200);
-            }
-        }
-    );
-
-};
-
 LayergroupController.prototype.attributes = function(req, res) {
     var self = this;
 

lib/cartodb/controllers/map.js

@@ -161,15 +161,15 @@ MapController.prototype.create = function(req, res, prepareConfigFn) {
         },
         function afterLayergroupCreate(err, layergroup) {
             assert.ifError(err);
-            var analysesResults = context.analysesResults || [];
-            self.afterLayergroupCreate(req, res, mapConfig, analysesResults, layergroup, this);
+            self.afterLayergroupCreate(req, res, mapConfig, layergroup, context.analysesResults, this);
         },
         function finish(err, layergroup) {
             if (err) {
                 self.sendError(req, res, err, 'ANONYMOUS LAYERGROUP');
             } else {
-                addWidgetsUrl(req.context.user, layergroup);
-
+                var analysesResults = context.analysesResults || [];
+                addDataviewsAndWidgetsUrls(req.context.user, layergroup, mapConfig.obj());
+                addAnalysesMetadata(req.context.user, layergroup, analysesResults, true);
                 res.set('X-Layergroup-Id', layergroup.layergroupid);
                 self.send(req, res, layergroup, 200);
             }
@@ -219,7 +219,7 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
         },
         function afterLayergroupCreate(err, layergroup) {
             assert.ifError(err);
-            self.afterLayergroupCreate(req, res, mapConfig, [], layergroup, this);
+            self.afterLayergroupCreate(req, res, mapConfig, layergroup, mapConfigProvider.analysesResults, this);
         },
         function finishTemplateInstantiation(err, layergroup) {
             if (err) {
@@ -228,8 +228,7 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
                 var templateHash = self.templateMaps.fingerPrint(mapConfigProvider.template).substring(0, 8);
                 layergroup.layergroupid = cdbuser + '@' + templateHash + '@' + layergroup.layergroupid;
 
-                addWidgetsUrl(cdbuser, layergroup);
-                addDataviewsUrls(cdbuser, layergroup, mapConfig.obj());
+                addDataviewsAndWidgetsUrls(cdbuser, layergroup, mapConfig.obj());
                 addAnalysesMetadata(cdbuser, layergroup, mapConfigProvider.analysesResults);
 
                 res.set('X-Layergroup-Id', layergroup.layergroupid);
@@ -241,8 +240,7 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
     );
 };
 
-
-MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, analysesResults, layergroup, callback) {
+MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, layergroup, analysesResults, callback) {
     var self = this;
 
     var username = req.context.user;
@@ -301,14 +299,13 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, an
             // feed affected tables cache so it can be reused from, for instance, layergroup controller
             self.layergroupAffectedTables.set(dbName, layergroupId, result);
 
-            // last update for layergroup cache buster
-            layergroup.layergroupid = layergroup.layergroupid + ':' + result.getLastUpdatedAt();
-            layergroup.last_updated = new Date(result.getLastUpdatedAt()).toISOString();
+            var lastUpdateTime = result.getLastUpdatedAt();
+            lastUpdateTime = getLastUpdatedTime(analysesResults, lastUpdateTime) || lastUpdateTime;
+
+            // last update for layergroup cache buster
+            layergroup.layergroupid = layergroup.layergroupid + ':' + lastUpdateTime;
+            layergroup.last_updated = new Date(lastUpdateTime).toISOString();
 
-            // TODO this should take into account several URL patterns
-            addWidgetsUrl(username, layergroup);
-            addDataviewsUrls(username, layergroup, mapconfig.obj());
-            addAnalysesMetadata(username, layergroup, analysesResults, true);
             if (req.method === 'GET') {
                 var ttl = global.environment.varnish.layergroupTtl || 86400;
                 res.set('Cache-Control', 'public,max-age='+ttl+',must-revalidate');
@@ -327,6 +324,19 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, an
     );
 };
 
+function getLastUpdatedTime(analysesResults, lastUpdateTime) {
+    if (!Array.isArray(analysesResults)) {
+        return lastUpdateTime;
+    }
+    return analysesResults.reduce(function(lastUpdateTime, analysis) {
+        return analysis.getSortedNodes().reduce(function(lastNodeUpdatedAtTime, node) {
+            var nodeUpdatedAtDate = node.getUpdatedAt();
+            var nodeUpdatedTimeAt = (nodeUpdatedAtDate && nodeUpdatedAtDate.getTime()) || 0;
+            return nodeUpdatedTimeAt > lastNodeUpdatedAtTime ? nodeUpdatedTimeAt : lastNodeUpdatedAtTime;
+        }, lastUpdateTime);
+    }, lastUpdateTime);
+}
+
 function addAnalysesMetadata(username, layergroup, analysesResults, includeQuery) {
     includeQuery = includeQuery || false;
     analysesResults = analysesResults || [];
@@ -353,6 +363,12 @@ function addAnalysesMetadata(username, layergroup, analysesResults, includeQuery
     });
 }
 
+// TODO this should take into account several URL patterns
+function addDataviewsAndWidgetsUrls(username, layergroup, mapConfig) {
+    addDataviewsUrls(username, layergroup, mapConfig);
+    addWidgetsUrl(username, layergroup, mapConfig);
+}
+
 function addDataviewsUrls(username, layergroup, mapConfig) {
     layergroup.metadata.dataviews = layergroup.metadata.dataviews || {};
     var dataviews = mapConfig.dataviews || {};
@@ -365,20 +381,23 @@ function addDataviewsUrls(username, layergroup, mapConfig) {
     });
 }
 
-function addWidgetsUrl(username, layergroup) {
-
-    if (layergroup.metadata && Array.isArray(layergroup.metadata.layers)) {
+function addWidgetsUrl(username, layergroup, mapConfig) {
+    if (layergroup.metadata && Array.isArray(layergroup.metadata.layers) && Array.isArray(mapConfig.layers)) {
         layergroup.metadata.layers = layergroup.metadata.layers.map(function(layer, layerIndex) {
-            if (layer.widgets) {
-                Object.keys(layer.widgets).forEach(function(widgetName) {
+            var mapConfigLayer = mapConfig.layers[layerIndex];
+            if (mapConfigLayer.options && mapConfigLayer.options.widgets) {
+                layer.widgets = layer.widgets || {};
+                Object.keys(mapConfigLayer.options.widgets).forEach(function(widgetName) {
                     var resource = layergroup.layergroupid + '/' + layerIndex + '/widget/' + widgetName;
-                    layer.widgets[widgetName].url = getUrls(username, resource);
+                    layer.widgets[widgetName] = {
+                        type: mapConfigLayer.options.widgets[widgetName].type,
+                        url: getUrls(username, resource)
+                    };
                 });
             }
             return layer;
         });
     }
-
 }
 
 function getUrls(username, resource) {

lib/cartodb/controllers/server_info.js

@@ -1,21 +1,13 @@
-var windshaft = require('windshaft');
 var HealthCheck = require('../monitoring/health_check');
 
 var WELCOME_MSG = "This is the CartoDB Maps API, " +
     "see the documentation at http://docs.cartodb.com/cartodb-platform/maps-api.html";
 
 
-var versions = {
-    windshaft: windshaft.version,
-    grainstore: windshaft.grainstore.version(),
-    node_mapnik: windshaft.mapnik.version,
-    mapnik: windshaft.mapnik.versions.mapnik,
-    windshaft_cartodb: require('../../../package.json').version
-};
-
-function ServerInfoController() {
+function ServerInfoController(versions) {
     this.healthConfig = global.environment.health || {};
     this.healthCheck = new HealthCheck(global.environment.disabled_file);
+    this.versions = versions || {};
 }
 
 module.exports = ServerInfoController;
@@ -31,7 +23,7 @@ ServerInfoController.prototype.welcome = function(req, res) {
 };
 
 ServerInfoController.prototype.version = function(req, res) {
-    res.status(200).send(versions);
+    res.status(200).send(this.versions);
 };
 
 ServerInfoController.prototype.health = function(req, res) {

lib/cartodb/models/dataview/aggregation.js

@@ -54,7 +54,10 @@ var CATEGORIES_LIMIT = 6;
 
 var VALID_OPERATIONS = {
     count: [],
-    sum: ['aggregationColumn']
+    sum: ['aggregationColumn'],
+    avg: ['aggregationColumn'],
+    min: ['aggregationColumn'],
+    max: ['aggregationColumn']
 };
 
 var TYPE = 'aggregation';
@@ -102,7 +105,7 @@ Aggregation.prototype.constructor = Aggregation;
 
 module.exports = Aggregation;
 
-Aggregation.prototype.sql = function(psql, filters, override, callback) {
+Aggregation.prototype.sql = function(psql, override, callback) {
     if (!callback) {
         callback = override;
         override = {};
@@ -198,6 +201,7 @@ Aggregation.prototype.format = function(result) {
     }
 
     return {
+        aggregation: this.aggregation,
         count: count,
         nulls: nulls,
         min: minValue,

lib/cartodb/models/dataview/formula.js

@@ -56,7 +56,7 @@ Formula.prototype.constructor = Formula;
 
 module.exports = Formula;
 
-Formula.prototype.sql = function(psql, filters, override, callback) {
+Formula.prototype.sql = function(psql, override, callback) {
     if (!callback) {
         callback = override;
         override = {};

lib/cartodb/models/dataview/histogram.js

@@ -174,8 +174,8 @@ Histogram.prototype.sql = function(psql, override, callback) {
         basicsQuery = overrideBasicsQueryTpl({
             _query: _query,
             _column: _column,
-            _start: override.start,
-            _end: override.end
+            _start: getBinStart(override),
+            _end: getBinEnd(override)
         });
 
         binsQuery = [
@@ -248,7 +248,7 @@ Histogram.prototype.format = function(result, override) {
         width = firstRow.bin_width || width;
         avg = firstRow.avg_val;
         nulls = firstRow.nulls_count;
-        binsStart = override.hasOwnProperty('start') ? override.start : firstRow.min;
+        binsStart = override.hasOwnProperty('start') ? getBinStart(override) : firstRow.min;
 
         buckets = result.rows.map(function(row) {
             return _.omit(row, 'bins_number', 'bin_width', 'nulls_count', 'avg_val');
@@ -266,9 +266,19 @@ Histogram.prototype.format = function(result, override) {
 };
 
 function getBinStart(override) {
+    if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) {
+        return Math.min(override.start, override.end);
+    }
     return override.start || 0;
 }
 
+function getBinEnd(override) {
+    if (override.hasOwnProperty('start') && override.hasOwnProperty('end')) {
+        return Math.max(override.start, override.end);
+    }
+    return override.end || 0;
+}
+
 function getBinsCount(override) {
     return override.bins || 0;
 }

lib/cartodb/models/dataview/list.js

@@ -34,7 +34,7 @@ List.prototype.constructor = List;
 
 module.exports = List;
 
-List.prototype.sql = function(psql, filters, override, callback) {
+List.prototype.sql = function(psql, override, callback) {
     if (!callback) {
         callback = override;
     }

lib/cartodb/models/dataview/overviews/aggregation.js

@@ -0,0 +1,141 @@
+var BaseOverviewsDataview = require('./base');
+var BaseDataview = require('../aggregation');
+
+var dot = require('dot');
+dot.templateSettings.strip = false;
+
+var summaryQueryTpl = dot.template([
+    'summary AS (',
+    '  SELECT',
+    '  sum(_feature_count) AS count,',
+    '  sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count',
+    '  FROM ({{=it._query}}) _cdb_aggregation_nulls',
+    ')'
+].join('\n'));
+
+var rankedCategoriesQueryTpl = dot.template([
+    'categories AS(',
+    '  SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,',
+    '    row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank',
+    '  FROM ({{=it._query}}) _cdb_aggregation_all',
+    '  GROUP BY {{=it._column}}',
+    '  ORDER BY 2 DESC',
+    ')'
+].join('\n'));
+
+var categoriesSummaryQueryTpl = dot.template([
+    'categories_summary AS(',
+    '  SELECT count(1) categories_count, max(value) max_val, min(value) min_val',
+    '  FROM categories',
+    ')'
+].join('\n'));
+
+var rankedAggregationQueryTpl = dot.template([
+    'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count',
+    '  FROM categories, summary, categories_summary',
+    '  WHERE rank < {{=it._limit}}',
+    'UNION ALL',
+    'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val, count, categories_count',
+    '  FROM categories, summary, categories_summary',
+    '  WHERE rank >= {{=it._limit}}',
+    'GROUP BY nulls_count, min_val, max_val, count, categories_count'
+].join('\n'));
+
+var aggregationQueryTpl = dot.template([
+    'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,',
+    '  nulls_count, min_val, max_val, count, categories_count',
+    'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary',
+    'GROUP BY category, nulls_count, min_val, max_val, count, categories_count',
+    'ORDER BY value DESC'
+].join('\n'));
+
+var CATEGORIES_LIMIT = 6;
+
+ function Aggregation(query, options, queryRewriter, queryRewriteData, params) {
+    BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params);
+
+    this.query = query;
+    this.column = options.column;
+    this.aggregation = options.aggregation;
+    this.aggregationColumn = options.aggregationColumn;
+}
+
+Aggregation.prototype = Object.create(BaseOverviewsDataview.prototype);
+Aggregation.prototype.constructor = Aggregation;
+
+module.exports = Aggregation;
+
+Aggregation.prototype.sql = function(psql, filters, override, callback) {
+    if (!callback) {
+        callback = override;
+        override = {};
+    }
+
+    var _query = this.rewrittenQuery(this.query);
+
+    var aggregationSql;
+    if (!!override.ownFilter) {
+        aggregationSql = [
+            "WITH",
+            [
+                summaryQueryTpl({
+                    _query: _query,
+                    _column: this.column
+                }),
+                rankedCategoriesQueryTpl({
+                    _query: _query,
+                    _column: this.column,
+                    _aggregation: this.getAggregationSql()
+                }),
+                categoriesSummaryQueryTpl({
+                    _query: _query,
+                    _column: this.column
+                })
+            ].join(',\n'),
+            aggregationQueryTpl({
+                _query: _query,
+                _column: this.column,
+                _aggregation: this.getAggregationSql(),
+                _limit: CATEGORIES_LIMIT
+            })
+        ].join('\n');
+    } else {
+        aggregationSql = [
+            "WITH",
+            [
+                summaryQueryTpl({
+                    _query: _query,
+                    _column: this.column
+                }),
+                rankedCategoriesQueryTpl({
+                    _query: _query,
+                    _column: this.column,
+                    _aggregation: this.getAggregationSql()
+                }),
+                categoriesSummaryQueryTpl({
+                    _query: _query,
+                    _column: this.column
+                })
+            ].join(',\n'),
+            rankedAggregationQueryTpl({
+                _query: _query,
+                _column: this.column,
+                _limit: CATEGORIES_LIMIT
+            })
+        ].join('\n');
+    }
+
+    return callback(null, aggregationSql);
+};
+
+var aggregationFnQueryTpl = {
+    count: dot.template('sum(_feature_count)'),
+    sum:   dot.template('sum({{=it._aggregationColumn}}*_feature_count)')
+};
+
+Aggregation.prototype.getAggregationSql = function() {
+    return aggregationFnQueryTpl[this.aggregation]({
+        _aggregationFn: this.aggregation,
+        _aggregationColumn: this.aggregationColumn || 1
+    });
+};

lib/cartodb/models/dataview/overviews/base.js

@@ -0,0 +1,88 @@
+var _ = require('underscore');
+var BaseDataview = require('../base');
+
+function BaseOverviewsDataview(query, queryOptions, BaseDataview, queryRewriter, queryRewriteData, options) {
+  this.BaseDataview = BaseDataview;
+  this.query = query;
+  this.queryOptions = queryOptions;
+  this.queryRewriter = queryRewriter;
+  this.queryRewriteData = queryRewriteData;
+  this.options = options;
+  this.baseDataview = new this.BaseDataview(this.query, this.queryOptions);
+}
+
+module.exports = BaseOverviewsDataview;
+
+BaseOverviewsDataview.prototype = new BaseDataview();
+BaseOverviewsDataview.prototype.constructor = BaseOverviewsDataview;
+
+// TODO: parameterized these settings
+var SETTINGS = {
+    // use overviews as a default fallback strategy
+    defaultOverviews: false,
+
+    // minimum ratio of bounding box size to grid size
+    // (this would ideally be based on the viewport size in pixels)
+    zoomLevelFactor: 1024.0
+};
+
+// Compute zoom level so that the the resolution grid size of the
+// selected overview is smaller (zoomLevelFactor times smaller at least)
+// than the bounding box size.
+BaseOverviewsDataview.prototype.zoomLevelForBbox = function(bbox) {
+    var pxPerTile = 256.0;
+    var earthWidth = 360.0;
+    // TODO: now we assume overviews are computed for 1-pixel tolerance;
+    // should use extended overviews metadata to compute this properly.
+    if ( bbox ) {
+      var bboxValues = _.map(bbox.split(','), function(v) { return +v; });
+      var w = Math.abs(bboxValues[2]-bboxValues[0]);
+      var h = Math.abs(bboxValues[3]-bboxValues[1]);
+      var maxDim = Math.min(w, h);
+
+      // Find minimum suitable z
+      // note that the QueryRewirter will use the minimum level overview
+      // of level >= z if it exists, and otherwise the base table
+      var z = Math.ceil(-Math.log(maxDim*pxPerTile/earthWidth/SETTINGS.zoomLevelFactor)/Math.log(2.0));
+      return Math.max(z, 0);
+    }
+    return 0;
+};
+
+BaseOverviewsDataview.prototype.rewrittenQuery = function(query) {
+    var zoom_level = this.zoomLevelForBbox(this.options.bbox);
+    return this.queryRewriter.query(query, this.queryRewriteData, { zoom_level: zoom_level });
+};
+
+// Default behaviour
+BaseOverviewsDataview.prototype.defaultSql = function(psql, filters, override, callback) {
+    var query = this.query;
+    var dataview = this.baseDataview;
+    if ( SETTINGS.defaultOverviews ) {
+        query = this.rewrittenQuery(query);
+        dataview = new this.BaseDataview(query, this.queryOptions);
+    }
+    return dataview.sql(psql, filters, override, callback);
+};
+
+// default implementation that can be override in derived classes:
+
+BaseOverviewsDataview.prototype.sql = function(psql, filters, override, callback) {
+    return this.defaultSql(psql, filters, override, callback);
+};
+
+BaseOverviewsDataview.prototype.search = function(psql, userQuery, callback) {
+    return this.baseDataview.search(psql, userQuery, callback);
+};
+
+BaseOverviewsDataview.prototype.format = function(result) {
+    return this.baseDataview.format(result);
+};
+
+BaseOverviewsDataview.prototype.getType = function() {
+    return this.baseDataview.getType();
+};
+
+BaseOverviewsDataview.prototype.toString = function() {
+    return this.baseDataview.toString();
+};

lib/cartodb/models/dataview/overviews/formula.js

@@ -1,9 +1,6 @@
-var _ = require('underscore');
-var BaseWidget = require('../base');
+var BaseOverviewsDataview = require('./base');
 var BaseDataview = require('../formula');
 
-var debug = require('debug')('windshaft:widget:formula:overviews');
-
 var dot = require('dot');
 dot.templateSettings.strip = false;
 
@@ -29,76 +26,31 @@ var formulaQueryTpls = {
 };
 
 function Formula(query, options, queryRewriter, queryRewriteData, params) {
-    this.base_dataview = new BaseDataview(query, options);
-    this.query = query;
+    BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params);
     this.column = options.column || '1';
     this.operation = options.operation;
-    this.queryRewriter = queryRewriter;
-    this.queryRewriteData = queryRewriteData;
-    this.options = params;
 }
 
-Formula.prototype = new BaseWidget();
+Formula.prototype = Object.create(BaseOverviewsDataview.prototype);
 Formula.prototype.constructor = Formula;
 
 module.exports = Formula;
 
-var zoom_level_factor = 100.0;
-
-// Compute zoom level so that the the resolution grid size of the
-// selected overview is smaller (zoom_level_factor times smaller at least)
-// than the bounding box size.
-function zoom_level_for_bbox(bbox) {
-    var px_per_tile = 256.0;
-    var earth_width = 360.0;
-    // TODO: now we assume overviews are computed for 1-pixel tolerance;
-    // should use extended overviews metadata to compute this properly.
-    if ( bbox ) {
-      var bbox_values = _.map(bbox.split(','), function(v) { return +v; });
-      var w = Math.abs(bbox_values[2]-bbox_values[0]);
-      var h = Math.abs(bbox_values[3]-bbox_values[1]);
-      var max_dim = Math.min(w, h);
-
-      // Find minimum suitable z
-      // note that the QueryRewirter will use the minimum level overview
-      // of level >= z if it exists, and otherwise the base table
-      var z = Math.ceil(-Math.log(max_dim*px_per_tile/earth_width/zoom_level_factor)/Math.log(2.0));
-      return Math.max(z, 0);
-    }
-    return 0;
-}
-
 Formula.prototype.sql = function(psql, filters, override, callback) {
-    var _query = this.query;
     var formulaQueryTpl = formulaQueryTpls[this.operation];
 
     if ( formulaQueryTpl ) {
         // supported formula for use with overviews
-        var zoom_level = zoom_level_for_bbox(this.options.bbox);
-        _query = this.queryRewriter.query(_query, this.queryRewriteData, { zoom_level: zoom_level });
         var formulaSql = formulaQueryTpl({
-            _query: _query,
+            _query:     this.rewrittenQuery(this.query),
             _operation: this.operation,
-            _column: this.column
+            _column:    this.column
         });
-        debug(formulaSql);
         callback = callback || override;
 
         return callback(null, formulaSql);
     }
 
-    // For non supported operations (min, max) we're not using overviews.
-    return this.base_dataview.sql(psql, filters, override, callback);
-};
-
-Formula.prototype.format = function(result) {
-    return this.base_dataview.format(result);
-};
-
-Formula.prototype.getType = function() {
-    return this.base_dataview.getType();
-};
-
-Formula.prototype.toString = function() {
-    return this.base_dataview.toString();
+    // default behaviour
+    return this.defaultSql(psql, filters, override, callback);
 };

lib/cartodb/models/dataview/overviews/histogram.js

@@ -0,0 +1,217 @@
+var _ = require('underscore');
+var BaseOverviewsDataview = require('./base');
+var BaseDataview = require('../histogram');
+
+var dot = require('dot');
+dot.templateSettings.strip = false;
+
+var columnTypeQueryTpl = dot.template(
+    'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1'
+);
+var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})");
+
+var BIN_MIN_NUMBER = 6;
+var BIN_MAX_NUMBER = 48;
+
+var basicsQueryTpl = dot.template([
+    'basics AS (',
+    '  SELECT',
+    '    max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,',
+    '    sum({{=it._column}}*_feature_count)/sum(_feature_count) AS avg_val, sum(_feature_count) AS total_rows',
+    '  FROM ({{=it._query}}) _cdb_basics',
+    ')'
+].join(' \n'));
+
+var overrideBasicsQueryTpl = dot.template([
+    'basics AS (',
+    '  SELECT',
+    '    max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,',
+    '    sum({{=it._column}}*_feature_count)/sum(_feature_count) AS avg_val, sum(_feature_count) AS total_rows',
+    '  FROM ({{=it._query}}) _cdb_basics',
+    ')'
+].join('\n'));
+
+var iqrQueryTpl = dot.template([
+    'iqrange AS (',
+    '  SELECT max(quartile_max) - min(quartile_max) AS iqr',
+    '  FROM (',
+    '    SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (',
+    '      SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}',
+    '    ) AS quartile',
+    '    FROM ({{=it._query}}) _cdb_rank) _cdb_quartiles',
+    '    WHERE quartile = 1 or quartile = 3',
+    '    GROUP BY quartile',
+    '  ) _cdb_iqr',
+    ')'
+].join('\n'));
+
+var binsQueryTpl = dot.template([
+    'bins AS (',
+    '  SELECT CASE WHEN total_rows = 0 OR iqr = 0',
+    '      THEN 1',
+    '      ELSE GREATEST(',
+    '        LEAST({{=it._minBins}}, CAST(total_rows AS INT)),',
+    '        LEAST(',
+    '          CAST(((max_val - min_val) / (2 * iqr * power(total_rows, 1/3))) AS INT),',
+    '          {{=it._maxBins}}',
+    '        )',
+    '      )',
+    '    END AS bins_number',
+    '  FROM basics, iqrange, ({{=it._query}}) _cdb_bins',
+    '  LIMIT 1',
+    ')'
+].join('\n'));
+
+var overrideBinsQueryTpl = dot.template([
+    'bins AS (',
+    '  SELECT {{=it._bins}} AS bins_number',
+    ')'
+].join('\n'));
+
+var nullsQueryTpl = dot.template([
+    'nulls AS (',
+    '  SELECT',
+    '  count(*) AS nulls_count',
+    '  FROM ({{=it._query}}) _cdb_histogram_nulls',
+    '  WHERE {{=it._column}} IS NULL',
+    ')'
+].join('\n'));
+
+var histogramQueryTpl = dot.template([
+    'SELECT',
+    '    (max_val - min_val) / cast(bins_number as float) AS bin_width,',
+    '    bins_number,',
+    '    nulls_count,',
+    '    avg_val,',
+    '    CASE WHEN min_val = max_val',
+    '      THEN 0',
+    '      ELSE GREATEST(1, LEAST(WIDTH_BUCKET({{=it._column}}, min_val, max_val, bins_number), bins_number)) - 1',
+    '    END AS bin,',
+    '    min({{=it._column}})::numeric AS min,',
+    '    max({{=it._column}})::numeric AS max,',
+    '    sum({{=it._column}}*_feature_count)/sum(_feature_count)::numeric AS avg,',
+    '    sum(_feature_count) AS freq',
+    'FROM ({{=it._query}}) _cdb_histogram, basics, nulls, bins',
+    'WHERE {{=it._column}} IS NOT NULL',
+    'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val',
+    'ORDER BY bin'
+].join('\n'));
+
+function Histogram(query, options, queryRewriter, queryRewriteData, params) {
+    BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params);
+
+    this.query = query;
+    this.column = options.column;
+    this.bins = options.bins;
+
+    this._columnType = null;
+}
+
+Histogram.prototype = Object.create(BaseOverviewsDataview.prototype);
+Histogram.prototype.constructor = Histogram;
+
+module.exports = Histogram;
+
+
+var DATE_OIDS = {
+    1082: true,
+    1114: true,
+    1184: true
+};
+
+Histogram.prototype.sql = function(psql, override, callback) {
+    if (!callback) {
+        callback = override;
+        override = {};
+    }
+
+    var self = this;
+
+    var _column = this.column;
+
+    var columnTypeQuery = columnTypeQueryTpl({
+        column: _column, query: this.rewrittenQuery(this.query)
+    });
+
+    if (this._columnType === null) {
+        psql.query(columnTypeQuery, function(err, result) {
+            // assume numeric, will fail later
+            self._columnType = 'numeric';
+            if (!err && !!result.rows[0]) {
+                var pgType = result.rows[0].pg_typeof;
+                if (DATE_OIDS.hasOwnProperty(pgType)) {
+                    self._columnType = 'date';
+                }
+            }
+            self.sql(psql, override, callback);
+        }, true); // use read-only transaction
+        return null;
+    }
+
+    if (this._columnType === 'date') {
+        _column = columnCastTpl({column: _column});
+    }
+
+    var _query = this.rewrittenQuery(this.query);
+
+    var basicsQuery, binsQuery;
+
+    if (override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins')) {
+        basicsQuery = overrideBasicsQueryTpl({
+            _query: _query,
+            _column: _column,
+            _start: override.start,
+            _end: override.end
+        });
+
+        binsQuery = [
+            overrideBinsQueryTpl({
+                _bins: override.bins
+            })
+        ].join(',\n');
+    } else {
+        basicsQuery = basicsQueryTpl({
+            _query: _query,
+            _column: _column
+        });
+
+        if (override && _.has(override, 'bins')) {
+            binsQuery = [
+                overrideBinsQueryTpl({
+                    _bins: override.bins
+                })
+            ].join(',\n');
+        } else {
+            binsQuery = [
+                iqrQueryTpl({
+                    _query: _query,
+                    _column: _column
+                }),
+                binsQueryTpl({
+                    _query: _query,
+                    _minBins: BIN_MIN_NUMBER,
+                    _maxBins: BIN_MAX_NUMBER
+                })
+            ].join(',\n');
+        }
+    }
+
+
+    var histogramSql = [
+        "WITH",
+        [
+            basicsQuery,
+            binsQuery,
+            nullsQueryTpl({
+                _query: _query,
+                _column: _column
+            })
+        ].join(',\n'),
+        histogramQueryTpl({
+            _query: _query,
+            _column: _column
+        })
+    ].join('\n');
+
+    return callback(null, histogramSql);
+};

lib/cartodb/models/dataview/overviews/index.js

@@ -1,3 +1,6 @@
 module.exports = {
-    Formula: require('./formula')
+  Aggregation: require('./aggregation'),
+  Formula: require('./formula'),
+  Histogram: require('./histogram'),
+  List: require('./list')
 };

lib/cartodb/models/dataview/overviews/list.js

@@ -0,0 +1,11 @@
+var BaseOverviewsDataview = require('./base');
+var BaseDataview = require('../list');
+
+function List(query, options, queryRewriter, queryRewriteData, params) {
+    BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params);
+}
+
+List.prototype = Object.create(BaseOverviewsDataview.prototype);
+List.prototype.constructor = List;
+
+module.exports = List;

lib/cartodb/models/mapconfig/adapter/analysis-mapconfig-adapter.js

@@ -58,13 +58,27 @@ AnalysisMapConfigAdapter.prototype.getMapConfig = function(user, requestMapConfi
 
     requestMapConfig = appendFiltersToNodes(requestMapConfig, dataviewsFiltersBySourceId);
 
-    function createAnalysis(analysisDefinition, done) {
-        self.analysisBackend.create(analysisConfiguration, analysisDefinition, done);
+    function createAnalysis(analysisDefinition, index, done) {
+        self.analysisBackend.create(analysisConfiguration, analysisDefinition, function (err, analysis) {
+            if (err) {
+                var error = new Error(err.message);
+                error.type = 'analysis';
+                error.context = {};
+                error.context.layer = {
+                    index: index,
+                    id: analysisDefinition.id,
+                    type: analysisDefinition.type
+                };
+                return done(error);
+            }
+
+            done(null, analysis);
+        });
     }
 
     var analysesQueue = queue(requestMapConfig.analyses.length);
-    requestMapConfig.analyses.forEach(function(analysis) {
-        analysesQueue.defer(createAnalysis, analysis);
+    requestMapConfig.analyses.forEach(function(analysis, index) {
+        analysesQueue.defer(createAnalysis, analysis, index);
     });
 
     analysesQueue.awaitAll(function(err, analysesResults) {
@@ -101,10 +115,7 @@ AnalysisMapConfigAdapter.prototype.getMapConfig = function(user, requestMapConfi
                         analysisSql = sqlQueryWrap.replace(/<%=\s*sql\s*%>/g, analysisSql);
                     }
                     layer.options.sql = analysisSql;
-                    var layerDataviews = getLayerDataviews(layer, dataviews);
-                    layer.options.columns = layerDataviews.reduce(function(columns, dataview) {
-                        return columns.concat(getDataviewColumns(dataview));
-                    }, []);
+                    layer.options.columns = getDataviewsColumns(getLayerDataviews(layer, dataviews));
                 } else {
                     missingNodesErrors.push(
                         new Error('Missing analysis node.id="' + layerSourceId +'" for layer='+layerIndex)
@@ -114,10 +125,12 @@ AnalysisMapConfigAdapter.prototype.getMapConfig = function(user, requestMapConfi
             return layer;
         });
 
+
         debug('mapconfig output', JSON.stringify(requestMapConfig, null, 4));
 
-        if (missingNodesErrors.length > 0) {
-            return callback(missingNodesErrors);
+        var missingDataviewsNodesErrors = getMissingDataviewsSourceIds(dataviews, sourceId2Node);
+        if (missingNodesErrors.length > 0 || missingDataviewsNodesErrors.length > 0) {
+            return callback(missingNodesErrors.concat(missingDataviewsNodesErrors));
         }
 
         context.analysesResults = analysesResults;
@@ -166,8 +179,9 @@ function appendFiltersToNodes(requestMapConfig, dataviewsFiltersBySourceId) {
 }
 
 function shouldAdaptLayers(requestMapConfig) {
-    return Array.isArray(requestMapConfig.layers) &&
-        Array.isArray(requestMapConfig.analyses) && requestMapConfig.analyses.length > 0;
+    return Array.isArray(requestMapConfig.layers) && requestMapConfig.layers.some(getLayerSourceId) ||
+        (Array.isArray(requestMapConfig.analyses) && requestMapConfig.analyses.length > 0) ||
+        requestMapConfig.dataviews;
 }
 
 var DATAVIEW_TYPE_2_FILTER_TYPE = {
@@ -208,11 +222,22 @@ function getLayerDataviews(layer, dataviews) {
     return layerDataviews;
 }
 
+function getDataviewsColumns(dataviews) {
+    return Object.keys(dataviews.reduce(function(columnsDict, dataview) {
+        getDataviewColumns(dataview).forEach(function(columnName) {
+            if (!!columnName) {
+                columnsDict[columnName] = true;
+            }
+        });
+        return columnsDict;
+    }, {}));
+}
+
 function getDataviewColumns(dataview) {
     var columns = [];
     var options = dataview.options;
     ['column', 'aggregationColumn'].forEach(function(opt) {
-        if (options.hasOwnProperty(opt)) {
+        if (options.hasOwnProperty(opt) && !!options[opt]) {
             columns.push(options[opt]);
         }
     });
@@ -224,6 +249,15 @@ function getDataviewsList(dataviews) {
 }
 
 function getDataviewsErrors(dataviews) {
+    var dataviewType = typeof dataviews;
+    if (dataviewType !== 'object') {
+        return [new Error('"dataviews" must be a valid JSON object: "' + dataviewType + '" type found')];
+    }
+
+    if (Array.isArray(dataviews)) {
+        return [new Error('"dataviews" must be a valid JSON object: "array" type found')];
+    }
+
     var errors = [];
 
     Object.keys(dataviews).forEach(function(dataviewName) {
@@ -239,3 +273,26 @@ function getDataviewsErrors(dataviews) {
 
     return errors;
 }
+
+function getMissingDataviewsSourceIds(dataviews, sourceId2Node) {
+    var missingDataviewsSourceIds = [];
+    Object.keys(dataviews).forEach(function(dataviewName) {
+        var dataview = dataviews[dataviewName];
+        var dataviewSourceId = getDataviewSourceId(dataview);
+        if (!sourceId2Node.hasOwnProperty(dataviewSourceId)) {
+            missingDataviewsSourceIds.push(new AnalysisError('Node with `source.id="' + dataviewSourceId +'"`' +
+                ' not found in analyses for dataview "' + dataviewName + '"'));
+        }
+    });
+
+    return missingDataviewsSourceIds;
+}
+
+function AnalysisError(message) {
+    Error.captureStackTrace(this, this.constructor);
+    this.name = this.constructor.name;
+    this.type = 'analysis';
+    this.message = message;
+}
+
+require('util').inherits(AnalysisError, Error);

lib/cartodb/models/mapconfig/adapter/dataviews-widgets-adapter.js

@@ -0,0 +1,98 @@
+function DataviewsWidgetsMapConfigAdapter() {
+}
+
+module.exports = DataviewsWidgetsMapConfigAdapter;
+
+
+DataviewsWidgetsMapConfigAdapter.prototype.getMapConfig = function(user, requestMapConfig, params, context, callback) {
+    if (!shouldAdapt(requestMapConfig)) {
+        return callback(null, requestMapConfig);
+    }
+
+    // prepare placeholders for new dataviews created from widgets
+    requestMapConfig.analyses = requestMapConfig.analyses || [];
+    requestMapConfig.dataviews = requestMapConfig.dataviews || {};
+
+    requestMapConfig.layers.forEach(function(layer, index) {
+        var layerSourceId = getLayerSourceId(layer);
+
+        if (!layer.options.widgets) {
+            return;
+        }
+
+        if (!layerSourceId && !layer.options.sql) {
+            return;
+        }
+
+        var dataviewSourceId = layerSourceId || 'cdb-layer-source-' + index;
+        // Append a new analysis if layer has no source id but sql.
+        if (!layerSourceId) {
+            requestMapConfig.analyses.push(
+                {
+                    id: dataviewSourceId,
+                    type: 'source',
+                    params: {
+                        query: layer.options.sql
+                    }
+                }
+            );
+        }
+        var source = { id: dataviewSourceId };
+        var layerWidgets = layer.options.widgets || {};
+        Object.keys(layerWidgets).forEach(function(widgetId) {
+            var dataview = layerWidgets[widgetId];
+            requestMapConfig.dataviews[widgetId] = {
+                source: source,
+                type: dataview.type,
+                options: dataview.options
+            };
+        });
+
+        layer.options.source = source;
+
+        delete layer.options.sql;
+        // don't delete widgets for now as it might be useful for old clients
+        //delete layer.options.widgets;
+    });
+
+    // filters have to be rewritten also
+    var filters = getFilters(params);
+    var layersFilters = filters.layers || [];
+    filters.dataviews = filters.dataviews || {};
+
+    layersFilters.forEach(function(layerFilters) {
+        Object.keys(layerFilters).forEach(function(filterName) {
+            if (!filters.dataviews.hasOwnProperty(filterName)) {
+                filters.dataviews[filterName] = layerFilters[filterName];
+            }
+        });
+    });
+
+    delete filters.layers;
+
+    params.filters = JSON.stringify(filters);
+
+    return callback(null, requestMapConfig);
+};
+
+function shouldAdapt(requestMapConfig) {
+    return Array.isArray(requestMapConfig.layers) && requestMapConfig.layers.some(function hasWidgets(layer) {
+        return layer.options && layer.options.widgets && Object.keys(layer.options.widgets).length > 0;
+    });
+}
+
+function getLayerSourceId(layer) {
+    return layer.options.source && layer.options.source.id;
+}
+
+function getFilters(params) {
+    var filters = {};
+    if (params.filters) {
+        try {
+            filters = JSON.parse(params.filters);
+        } catch (e) {
+            // ignore
+        }
+    }
+    return filters;
+}

lib/cartodb/models/mapconfig/adapter/mapconfig-overviews-adapter.js

@@ -72,7 +72,7 @@ MapConfigOverviewsAdapter.prototype.getMapConfig = function(user, requestMapConf
                            layer = _.extend({}, layer);
                            layer.options = _.extend({}, layer.options, { query_rewrite_data: query_rewrite_data });
                        }
-                       done(err, layer);
+                       done(null, layer);
                    }
                );
            }

lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js

@@ -3,10 +3,13 @@
 var dot = require('dot');
 dot.templateSettings.strip = false;
 var queue = require('queue-async');
-var SubstitutionTokens = require('../../../utils/substitution-tokens');
+var PSQL = require('cartodb-psql');
+var turboCarto = require('turbo-carto');
 
-function TurboCartoAdapter(turboCartoParser) {
-    this.turboCartoParser = turboCartoParser;
+var SubstitutionTokens = require('../../../utils/substitution-tokens');
+var PostgresDatasource = require('../../../backends/turbo-carto-postgres-datasource');
+
+function TurboCartoAdapter() {
 }
 
 module.exports = TurboCartoAdapter;
@@ -22,8 +25,8 @@ TurboCartoAdapter.prototype.getMapConfig = function (user, requestMapConfig, par
 
     var parseCartoQueue = queue(layers.length);
 
-    layers.forEach(function(layer) {
-        parseCartoQueue.defer(self._parseCartoCss.bind(self), user, layer);
+    layers.forEach(function(layer, index) {
+        parseCartoQueue.defer(self._parseCartoCss.bind(self), user, params, layer, index);
     });
 
     parseCartoQueue.awaitAll(function (err, layers) {
@@ -37,25 +40,87 @@ TurboCartoAdapter.prototype.getMapConfig = function (user, requestMapConfig, par
     });
 };
 
-var bboxTemplate = dot.template('(select ST_SetSRID(st_extent(the_geom_webmercator), 3857) from ({{=it._sql}}) __c)');
-var zoomTemplate = dot.template([
-    'GREATEST(',
-    'ceil(log(40075017000 / 256 / GREATEST(',
-    '  st_xmax({{=it._bbox}}) - st_xmin({{=it._bbox}}),',
-    '  st_ymax({{=it._bbox}}) - st_ymin({{=it._bbox}})',
-    '))/log(2)),',
-    '0',
-    ')'
+var tokensQueryTpl = dot.template([
+    'WITH input_query AS (',
+    '  {{=it._sql}}',
+    '),',
+    'bbox_query AS (',
+    '  SELECT ST_SetSRID(ST_Extent(the_geom_webmercator), 3857) as bbox from input_query',
+    '),',
+    'zoom_query as (',
+    '  SELECT GREATEST(',
+    '  ceil(log(40075017000 / 256 / GREATEST(ST_XMax(bbox) - ST_XMin(bbox), ST_YMax(bbox) - ST_YMin(bbox)))/log(2)),',
+    '  0) as zoom',
+    '  FROM bbox_query',
+    '),',
+    'pixel_size_query as (',
+    '  SELECT 40075017 * cos(radians(ST_Y(ST_Transform(ST_Centroid(bbox), 4326)))) / 2 ^ ((zoom) + 8) as pixel_size',
+    '  FROM bbox_query, zoom_query',
+    '),',
+    'scale_denominator_query as (',
+    '  SELECT (pixel_size / 0.00028)::numeric as scale_denominator',
+    '  FROM pixel_size_query',
+    ')',
+    'select ST_AsText(bbox) bbox, pixel_size, scale_denominator, zoom',
+    'from bbox_query, pixel_size_query, scale_denominator_query, zoom_query'
 ].join('\n'));
-var pixelSizeTemplate = dot.template('40075017 * cos(ST_Y(ST_Centroid({{=it._bbox}}))) / 2 ^ (({{=it._zoom}}) + 8)');
-var scaleDenominatorTemplate = dot.template('({{=it._pixelSize}} / 0.00028)::numeric');
 
-
-TurboCartoAdapter.prototype._parseCartoCss = function (username, layer, callback) {
+TurboCartoAdapter.prototype._parseCartoCss = function (username, params, layer, index, callback) {
     if (!shouldParseLayerCartocss(layer)) {
         return callback(null, layer);
     }
 
+    var pg = new PSQL(dbParamsFromReqParams(params));
+    function processCallback(err, cartocss) {
+        // Only return turbo-carto errors
+        if (err && err.name === 'TurboCartoError') {
+            var error = new Error('turbo-carto: ' + err.message);
+            error.http_status = 400;
+            error.type = 'turbo-carto';
+            error.context = err.context;
+            error.context.layer = {
+                index: index,
+                type: layer.type
+            };
+
+            return callback(error);
+        }
+
+        // Try to continue in the rest of the cases
+        if (cartocss) {
+            layer.options.cartocss = cartocss;
+        }
+        return callback(null, layer);
+    }
+
+    var layerSql = layer.options.sql;
+    var layerRawSql = layer.options.sql_raw;
+    if (SubstitutionTokens.hasTokens(layerSql) && layerRawSql) {
+        var self = this;
+        var tokensQuery = tokensQueryTpl({_sql: layerRawSql});
+        return pg.query(tokensQuery, function(err, resultSet) {
+            if (err) {
+                var error = new Error('turbo-carto: ' + err.message);
+                error.type = 'turbo-carto';
+                return callback(error);
+            }
+
+            resultSet = resultSet || {};
+            var rows = resultSet.rows || [];
+            var result = rows[0] || {};
+
+            var tokens = {
+                bbox: 'ST_SetSRID(ST_GeomFromText(\'' + result.bbox + '\'), 3857)',
+                scale_denominator: result.scale_denominator,
+                pixel_width: result.pixel_size,
+                pixel_height: result.pixel_size
+            };
+
+            var sql = SubstitutionTokens.replace(layerSql, tokens);
+            self.process(pg, layer.options.cartocss, sql, processCallback);
+        }, true); // use read-only transaction
+    }
+
     var tokens = {
         bbox: 'ST_MakeEnvelope(-20037508.34,-20037508.34,20037508.34,20037508.34,3857)',
         scale_denominator: '500000001',
@@ -63,40 +128,35 @@ TurboCartoAdapter.prototype._parseCartoCss = function (username, layer, callback
         pixel_height: '156412'
     };
 
-    var layerSql = layer.options.sql;
-    var layerRawSql = layer.options.sql_raw;
-    if (SubstitutionTokens.hasTokens(layerSql) && layerRawSql) {
-        var bbox = bboxTemplate({ _sql: layerRawSql });
-        var zoom = zoomTemplate({ _bbox: bbox });
-        var pixelSize = pixelSizeTemplate({ _bbox: bbox, _zoom: zoom });
-        var scaleDenominator = scaleDenominatorTemplate({ _pixelSize: pixelSize });
-
-        tokens = {
-            bbox: bbox,
-            scale_denominator: scaleDenominator,
-            pixel_width: pixelSize,
-            pixel_height: pixelSize
-        };
-    }
-
     var sql = SubstitutionTokens.replace(layerSql, tokens);
+    this.process(pg, layer.options.cartocss, sql, processCallback);
+};
 
-    this.turboCartoParser.process(username, layer.options.cartocss, sql, function (err, cartocss) {
-        // Only return turbo-carto errors
-        if (err && err.name === 'TurboCartoError') {
-            err = new Error('turbo-carto: ' + err.message);
-            err.http_status = 400;
-            return callback(err);
-        }
-
-        // Try to continue in the rest of the cases
-        if (cartocss) {
-            layer.options.cartocss = cartocss;
-        }
-        return callback(null, layer);
-    });
+TurboCartoAdapter.prototype.process = function (psql, cartocss, sql, callback) {
+    var datasource = new PostgresDatasource(psql, sql);
+    turboCarto(cartocss, datasource, callback);
 };
 
 function shouldParseLayerCartocss(layer) {
     return layer && layer.options && layer.options.cartocss && layer.options.sql;
 }
+
+function dbParamsFromReqParams(params) {
+    var dbParams = {};
+    if ( params.dbuser ) {
+        dbParams.user = params.dbuser;
+    }
+    if ( params.dbpassword ) {
+        dbParams.pass = params.dbpassword;
+    }
+    if ( params.dbhost ) {
+        dbParams.host = params.dbhost;
+    }
+    if ( params.dbport ) {
+        dbParams.port = params.dbport;
+    }
+    if ( params.dbname ) {
+        dbParams.dbname = params.dbname;
+    }
+    return dbParams;
+}

lib/cartodb/server.js

@@ -3,7 +3,6 @@ var bodyParser = require('body-parser');
 var RedisPool = require('redis-mpool');
 var cartodbRedis = require('cartodb-redis');
 var _ = require('underscore');
-var debug = require('debug')('windshaft:cartodb');
 
 var controller = require('./controllers');
 
@@ -34,13 +33,12 @@ var AnalysisBackend = require('./backends/analysis');
 var timeoutErrorTilePath = __dirname + '/../../assets/render-timeout-fallback.png';
 var timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, {encoding: null});
 
-var TurboCartoParser = require('./utils/style/turbo-carto-parser');
-
 var SqlWrapMapConfigAdapter = require('./models/mapconfig/adapter/sql-wrap-mapconfig-adapter');
 var MapConfigNamedLayersAdapter = require('./models/mapconfig/adapter/mapconfig-named-layers-adapter');
 var AnalysisMapConfigAdapter = require('./models/mapconfig/adapter/analysis-mapconfig-adapter');
 var MapConfigOverviewsAdapter = require('./models/mapconfig/adapter/mapconfig-overviews-adapter');
 var TurboCartoAdapter = require('./models/mapconfig/adapter/turbo-carto-adapter');
+var DataviewsWidgetsAdapter = require('./models/mapconfig/adapter/dataviews-widgets-adapter');
 var MapConfigAdapter = require('./models/mapconfig/adapter');
 
 module.exports = function(serverOptions) {
@@ -154,14 +152,13 @@ module.exports = function(serverOptions) {
     var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache();
     app.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
 
-    var turboCartoParser = new TurboCartoParser(pgQueryRunner);
-
     var mapConfigAdapter = new MapConfigAdapter(
         new MapConfigNamedLayersAdapter(templateMaps, pgConnection),
         new SqlWrapMapConfigAdapter(),
+        new DataviewsWidgetsAdapter(),
         new AnalysisMapConfigAdapter(analysisBackend),
         new MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi),
-        new TurboCartoAdapter(turboCartoParser)
+        new TurboCartoAdapter()
     );
 
     var namedMapProviderCache = new NamedMapProviderCache(
@@ -181,6 +178,8 @@ module.exports = function(serverOptions) {
     var TablesExtentApi = require('./api/tables_extent_api');
     var tablesExtentApi = new TablesExtentApi(pgQueryRunner);
 
+    var versions = getAndValidateVersions(serverOptions);
+
     /*******************************************************************************************************************
      * Routing
      ******************************************************************************************************************/
@@ -192,7 +191,6 @@ module.exports = function(serverOptions) {
         tileBackend,
         previewBackend,
         attributesBackend,
-        new windshaft.backend.Widget(),
         surrogateKeysCache,
         userLimitsApi,
         layergroupAffectedTablesCache,
@@ -224,7 +222,7 @@ module.exports = function(serverOptions) {
 
     new controller.NamedMapsAdmin(authApi, pgConnection, templateMaps).register(app);
 
-    new controller.ServerInfo().register(app);
+    new controller.ServerInfo(versions).register(app);
 
     /*******************************************************************************************************************
      * END Routing
@@ -237,12 +235,45 @@ function validateOptions(opts) {
     if (!_.isString(opts.base_url) || !_.isString(opts.base_url_mapconfig) || !_.isString(opts.base_url_templated)) {
         throw new Error("Must initialise server with: 'base_url'/'base_url_mapconfig'/'base_url_templated' URLs");
     }
+}
 
-    // Be nice and warn if configured mapnik version is != instaled mapnik version
-    if (mapnik.versions.mapnik !== opts.grainstore.mapnik_version) {
-        debug('WARNING: detected mapnik version (' + mapnik.versions.mapnik + ')' +
-            ' != configured mapnik version (' + opts.grainstore.mapnik_version + ')');
+function getAndValidateVersions(options) {
+    // jshint undef:false
+    var warn = console.warn.bind(console);
+    // jshint undef:true
+
+    var packageDefinition = require('../../package.json');
+
+    var declaredDependencies = packageDefinition.dependencies || {};
+    var installedDependenciesVersions = {
+        camshaft: require('camshaft').version,
+        grainstore: windshaft.grainstore.version(),
+        mapnik: windshaft.mapnik.versions.mapnik,
+        node_mapnik: windshaft.mapnik.version,
+        'turbo-carto': require('turbo-carto').version,
+        windshaft: windshaft.version,
+        windshaft_cartodb: packageDefinition.version
+    };
+
+    var dependenciesToValidate = ['camshaft', 'turbo-carto', 'windshaft'];
+    dependenciesToValidate.forEach(function(depName) {
+        var declaredDependencyVersion = declaredDependencies[depName];
+        var installedDependencyVersion = installedDependenciesVersions[depName];
+        if (declaredDependencyVersion !== installedDependencyVersion) {
+            warn(
+                'Dependency="%s" installed version="%s" does not match declared version="%s". Check your installation.',
+                depName, installedDependencyVersion, declaredDependencyVersion
+            );
+        }
+    });
+
+    // Be nice and warn if configured mapnik version is != installed mapnik version
+    if (mapnik.versions.mapnik !== options.grainstore.mapnik_version) {
+        warn('WARNING: detected mapnik version (' + mapnik.versions.mapnik + ')' +
+            ' != configured mapnik version (' + options.grainstore.mapnik_version + ')');
     }
+
+    return installedDependenciesVersions;
 }
 
 function bootstrapFonts(opts) {

lib/cartodb/utils/style/turbo-carto-parser.js

@@ -1,15 +0,0 @@
-'use strict';
-
-var turboCarto = require('turbo-carto');
-var PostgresDatasource = require('./postgres-datasource');
-
-function TurboCartoParser (pgQueryRunner) {
-    this.pgQueryRunner = pgQueryRunner;
-}
-
-module.exports = TurboCartoParser;
-
-TurboCartoParser.prototype.process = function (username, cartocss, sql, callback) {
-    var datasource = new PostgresDatasource(this.pgQueryRunner, username, sql);
-    turboCarto(cartocss, datasource, callback);
-};

npm-shrinkwrap.json

@@ -1,6 +1,6 @@
 {
   "name": "windshaft-cartodb",
-  "version": "2.44.0",
+  "version": "2.53.2",
   "dependencies": {
     "body-parser": {
       "version": "1.14.2",
@@ -62,25 +62,25 @@
           "resolved": "https://registry.npmjs.org/qs/-/qs-5.2.0.tgz"
         },
         "raw-body": {
-          "version": "2.1.6",
+          "version": "2.1.7",
           "from": "raw-body@>=2.1.5 <2.2.0",
-          "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.1.6.tgz",
+          "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.1.7.tgz",
           "dependencies": {
             "bytes": {
-              "version": "2.3.0",
-              "from": "bytes@2.3.0",
-              "resolved": "https://registry.npmjs.org/bytes/-/bytes-2.3.0.tgz"
+              "version": "2.4.0",
+              "from": "bytes@2.4.0",
+              "resolved": "https://registry.npmjs.org/bytes/-/bytes-2.4.0.tgz"
             },
             "unpipe": {
               "version": "1.0.0",
-              "from": "unpipe@>=1.0.0 <1.1.0",
+              "from": "unpipe@1.0.0",
               "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz"
             }
           }
         },
         "type-is": {
           "version": "1.6.13",
-          "from": "type-is@>=1.6.6 <1.7.0",
+          "from": "type-is@>=1.6.10 <1.7.0",
           "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.13.tgz",
           "dependencies": {
             "media-typer": {
@@ -105,13 +105,13 @@
       }
     },
     "camshaft": {
-      "version": "0.11.0",
-      "from": "camshaft@0.11.0",
-      "resolved": "https://registry.npmjs.org/camshaft/-/camshaft-0.11.0.tgz",
+      "version": "0.22.2",
+      "from": "camshaft@0.22.2",
+      "resolved": "https://registry.npmjs.org/camshaft/-/camshaft-0.22.2.tgz",
       "dependencies": {
         "async": {
           "version": "1.5.2",
-          "from": "async@>=1.0.0 <2.0.0",
+          "from": "async@>=1.5.2 <2.0.0",
           "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz"
         },
         "request": {
@@ -354,9 +354,9 @@
                   "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-0.2.0.tgz"
                 },
                 "jsprim": {
-                  "version": "1.2.2",
+                  "version": "1.3.0",
                   "from": "jsprim@>=1.2.2 <2.0.0",
-                  "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.2.2.tgz",
+                  "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.3.0.tgz",
                   "dependencies": {
                     "extsprintf": {
                       "version": "1.0.2",
@@ -391,9 +391,9 @@
                       "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz"
                     },
                     "dashdash": {
-                      "version": "1.13.1",
+                      "version": "1.14.0",
                       "from": "dashdash@>=1.12.0 <2.0.0",
-                      "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.13.1.tgz"
+                      "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.0.tgz"
                     },
                     "getpass": {
                       "version": "0.1.6",
@@ -757,9 +757,50 @@
           }
         },
         "serve-static": {
-          "version": "1.10.2",
+          "version": "1.10.3",
           "from": "serve-static@>=1.10.2 <1.11.0",
-          "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.10.2.tgz"
+          "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.10.3.tgz",
+          "dependencies": {
+            "send": {
+              "version": "0.13.2",
+              "from": "send@0.13.2",
+              "resolved": "https://registry.npmjs.org/send/-/send-0.13.2.tgz",
+              "dependencies": {
+                "destroy": {
+                  "version": "1.0.4",
+                  "from": "destroy@>=1.0.4 <1.1.0",
+                  "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz"
+                },
+                "http-errors": {
+                  "version": "1.3.1",
+                  "from": "http-errors@>=1.3.1 <1.4.0",
+                  "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.3.1.tgz",
+                  "dependencies": {
+                    "inherits": {
+                      "version": "2.0.1",
+                      "from": "inherits@>=2.0.1 <2.1.0",
+                      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz"
+                    }
+                  }
+                },
+                "mime": {
+                  "version": "1.3.4",
+                  "from": "mime@1.3.4",
+                  "resolved": "https://registry.npmjs.org/mime/-/mime-1.3.4.tgz"
+                },
+                "ms": {
+                  "version": "0.7.1",
+                  "from": "ms@0.7.1",
+                  "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz"
+                },
+                "statuses": {
+                  "version": "1.2.1",
+                  "from": "statuses@>=1.2.1 <1.3.0",
+                  "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.2.1.tgz"
+                }
+              }
+            }
+          }
         },
         "type-is": {
           "version": "1.6.13",
@@ -1216,9 +1257,9 @@
       "resolved": "https://registry.npmjs.org/step-profiler/-/step-profiler-0.3.0.tgz"
     },
     "turbo-carto": {
-      "version": "0.10.0",
-      "from": "turbo-carto@0.10.0",
-      "resolved": "https://registry.npmjs.org/turbo-carto/-/turbo-carto-0.10.0.tgz",
+      "version": "0.12.1",
+      "from": "turbo-carto@0.12.1",
+      "resolved": "https://registry.npmjs.org/turbo-carto/-/turbo-carto-0.12.1.tgz",
       "dependencies": {
         "colorbrewer": {
           "version": "1.0.0",
@@ -1277,14 +1318,14 @@
       "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz"
     },
     "windshaft": {
-      "version": "1.19.0",
-      "from": "windshaft@1.19.0",
-      "resolved": "https://registry.npmjs.org/windshaft/-/windshaft-1.19.0.tgz",
+      "version": "2.3.0",
+      "from": "windshaft@2.3.0",
+      "resolved": "https://registry.npmjs.org/windshaft/-/windshaft-2.3.0.tgz",
       "dependencies": {
         "abaculus": {
-          "version": "1.1.0-cdb4",
-          "from": "https://github.com/CartoDB/abaculus/tarball/1.1.0-cdb4",
-          "resolved": "https://github.com/CartoDB/abaculus/tarball/1.1.0-cdb4"
+          "version": "1.1.0-cdb5",
+          "from": "https://github.com/CartoDB/abaculus/tarball/1.1.0-cdb5",
+          "resolved": "https://github.com/CartoDB/abaculus/tarball/1.1.0-cdb5"
         },
         "canvas": {
           "version": "1.2.7-cdb1",
@@ -1328,9 +1369,9 @@
           }
         },
         "grainstore": {
-          "version": "1.1.1",
-          "from": "grainstore@1.1.1",
-          "resolved": "https://registry.npmjs.org/grainstore/-/grainstore-1.1.1.tgz",
+          "version": "1.2.0",
+          "from": "grainstore@1.2.0",
+          "resolved": "https://registry.npmjs.org/grainstore/-/grainstore-1.2.0.tgz",
           "dependencies": {
             "carto": {
               "version": "0.9.5-cdb2",
@@ -2916,9 +2957,9 @@
                   "resolved": "https://registry.npmjs.org/sqlite3/-/sqlite3-3.1.4.tgz",
                   "dependencies": {
                     "nan": {
-                      "version": "2.3.3",
+                      "version": "2.3.5",
                       "from": "nan@>=2.3.3 <2.4.0",
-                      "resolved": "https://registry.npmjs.org/nan/-/nan-2.3.3.tgz"
+                      "resolved": "https://registry.npmjs.org/nan/-/nan-2.3.5.tgz"
                     },
                     "node-pre-gyp": {
                       "version": "0.6.28",
@@ -2935,11 +2976,6 @@
                       "from": "ansi@>=0.3.1 <0.4.0",
                       "resolved": "https://registry.npmjs.org/ansi/-/ansi-0.3.1.tgz"
                     },
-                    "ansi-regex": {
-                      "version": "2.0.0",
-                      "from": "ansi-regex@>=2.0.0 <3.0.0",
-                      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.0.0.tgz"
-                    },
                     "ansi-styles": {
                       "version": "2.2.1",
                       "from": "ansi-styles@>=2.2.1 <3.0.0",
@@ -2950,21 +2986,26 @@
                       "from": "are-we-there-yet@>=1.1.2 <1.2.0",
                       "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.2.tgz"
                     },
+                    "ansi-regex": {
+                      "version": "2.0.0",
+                      "from": "ansi-regex@>=2.0.0 <3.0.0",
+                      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.0.0.tgz"
+                    },
                     "asn1": {
                       "version": "0.2.3",
                       "from": "asn1@>=0.2.3 <0.3.0",
                       "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.3.tgz"
                     },
-                    "async": {
-                      "version": "1.5.2",
-                      "from": "async@>=1.5.2 <2.0.0",
-                      "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz"
-                    },
                     "assert-plus": {
                       "version": "0.2.0",
                       "from": "assert-plus@>=0.2.0 <0.3.0",
                       "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-0.2.0.tgz"
                     },
+                    "async": {
+                      "version": "1.5.2",
+                      "from": "async@>=1.5.2 <2.0.0",
+                      "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz"
+                    },
                     "aws-sign2": {
                       "version": "0.6.0",
                       "from": "aws-sign2@>=0.6.0 <0.7.0",
@@ -2995,41 +3036,41 @@
                       "from": "brace-expansion@>=1.0.0 <2.0.0",
                       "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.4.tgz"
                     },
-                    "chalk": {
-                      "version": "1.1.3",
-                      "from": "chalk@>=1.1.1 <2.0.0",
-                      "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz"
-                    },
                     "caseless": {
                       "version": "0.11.0",
                       "from": "caseless@>=0.11.0 <0.12.0",
                       "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.11.0.tgz"
                     },
+                    "chalk": {
+                      "version": "1.1.3",
+                      "from": "chalk@>=1.1.1 <2.0.0",
+                      "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz"
+                    },
                     "combined-stream": {
                       "version": "1.0.5",
                       "from": "combined-stream@>=1.0.5 <1.1.0",
                       "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.5.tgz"
                     },
-                    "commander": {
-                      "version": "2.9.0",
-                      "from": "commander@>=2.9.0 <3.0.0",
-                      "resolved": "https://registry.npmjs.org/commander/-/commander-2.9.0.tgz"
-                    },
                     "concat-map": {
                       "version": "0.0.1",
                       "from": "concat-map@0.0.1",
                       "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz"
                     },
-                    "core-util-is": {
-                      "version": "1.0.2",
-                      "from": "core-util-is@>=1.0.0 <1.1.0",
-                      "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz"
+                    "commander": {
+                      "version": "2.9.0",
+                      "from": "commander@>=2.9.0 <3.0.0",
+                      "resolved": "https://registry.npmjs.org/commander/-/commander-2.9.0.tgz"
                     },
                     "cryptiles": {
                       "version": "2.0.5",
                       "from": "cryptiles@>=2.0.0 <3.0.0",
                       "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-2.0.5.tgz"
                     },
+                    "core-util-is": {
+                      "version": "1.0.2",
+                      "from": "core-util-is@>=1.0.0 <1.1.0",
+                      "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz"
+                    },
                     "debug": {
                       "version": "2.2.0",
                       "from": "debug@>=2.2.0 <2.3.0",
@@ -3060,10 +3101,10 @@
                       "from": "escape-string-regexp@>=1.0.2 <2.0.0",
                       "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz"
                     },
-                    "extend": {
-                      "version": "3.0.0",
-                      "from": "extend@>=3.0.0 <3.1.0",
-                      "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.0.tgz"
+                    "forever-agent": {
+                      "version": "0.6.1",
+                      "from": "forever-agent@>=0.6.1 <0.7.0",
+                      "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz"
                     },
                     "extsprintf": {
                       "version": "1.0.2",
@@ -3075,10 +3116,10 @@
                       "from": "form-data@>=1.0.0-rc3 <1.1.0",
                       "resolved": "https://registry.npmjs.org/form-data/-/form-data-1.0.0-rc4.tgz"
                     },
-                    "forever-agent": {
-                      "version": "0.6.1",
-                      "from": "forever-agent@>=0.6.1 <0.7.0",
-                      "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz"
+                    "extend": {
+                      "version": "3.0.0",
+                      "from": "extend@>=3.0.0 <3.1.0",
+                      "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.0.tgz"
                     },
                     "fstream": {
                       "version": "1.0.9",
@@ -3130,16 +3171,16 @@
                       "from": "has-ansi@>=2.0.0 <3.0.0",
                       "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz"
                     },
-                    "hawk": {
-                      "version": "3.1.3",
-                      "from": "hawk@>=3.1.3 <3.2.0",
-                      "resolved": "https://registry.npmjs.org/hawk/-/hawk-3.1.3.tgz"
-                    },
                     "has-unicode": {
                       "version": "2.0.0",
                       "from": "has-unicode@>=2.0.0 <3.0.0",
                       "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.0.tgz"
                     },
+                    "hawk": {
+                      "version": "3.1.3",
+                      "from": "hawk@>=3.1.3 <3.2.0",
+                      "resolved": "https://registry.npmjs.org/hawk/-/hawk-3.1.3.tgz"
+                    },
                     "hoek": {
                       "version": "2.16.3",
                       "from": "hoek@>=2.0.0 <3.0.0",
@@ -3205,16 +3246,16 @@
                       "from": "json-schema@0.2.2",
                       "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.2.tgz"
                     },
-                    "json-stringify-safe": {
-                      "version": "5.0.1",
-                      "from": "json-stringify-safe@>=5.0.1 <5.1.0",
-                      "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz"
-                    },
                     "jsonpointer": {
                       "version": "2.0.0",
                       "from": "jsonpointer@2.0.0",
                       "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-2.0.0.tgz"
                     },
+                    "json-stringify-safe": {
+                      "version": "5.0.1",
+                      "from": "json-stringify-safe@>=5.0.1 <5.1.0",
+                      "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz"
+                    },
                     "jsprim": {
                       "version": "1.2.2",
                       "from": "jsprim@>=1.2.2 <2.0.0",
@@ -3250,11 +3291,6 @@
                       "from": "lodash.tostring@>=4.0.0 <5.0.0",
                       "resolved": "https://registry.npmjs.org/lodash.tostring/-/lodash.tostring-4.1.3.tgz"
                     },
-                    "mime-db": {
-                      "version": "1.23.0",
-                      "from": "mime-db@>=1.23.0 <1.24.0",
-                      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.23.0.tgz"
-                    },
                     "mime-types": {
                       "version": "2.1.11",
                       "from": "mime-types@>=2.1.7 <2.2.0",
@@ -3265,16 +3301,21 @@
                       "from": "minimatch@>=2.0.0 <3.0.0||>=3.0.0 <4.0.0",
                       "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.0.tgz"
                     },
-                    "mkdirp": {
-                      "version": "0.5.1",
-                      "from": "mkdirp@>=0.5.0 <0.6.0",
-                      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz"
-                    },
                     "minimist": {
                       "version": "0.0.8",
                       "from": "minimist@0.0.8",
                       "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz"
                     },
+                    "mime-db": {
+                      "version": "1.23.0",
+                      "from": "mime-db@>=1.23.0 <1.24.0",
+                      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.23.0.tgz"
+                    },
+                    "mkdirp": {
+                      "version": "0.5.1",
+                      "from": "mkdirp@>=0.5.0 <0.6.0",
+                      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz"
+                    },
                     "ms": {
                       "version": "0.7.1",
                       "from": "ms@0.7.1",
@@ -3285,16 +3326,16 @@
                       "from": "node-uuid@>=1.4.7 <1.5.0",
                       "resolved": "https://registry.npmjs.org/node-uuid/-/node-uuid-1.4.7.tgz"
                     },
-                    "nopt": {
-                      "version": "3.0.6",
-                      "from": "nopt@>=3.0.1 <3.1.0",
-                      "resolved": "https://registry.npmjs.org/nopt/-/nopt-3.0.6.tgz"
-                    },
                     "npmlog": {
                       "version": "2.0.3",
                       "from": "npmlog@>=2.0.0 <2.1.0",
                       "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-2.0.3.tgz"
                     },
+                    "nopt": {
+                      "version": "3.0.6",
+                      "from": "nopt@>=3.0.1 <3.1.0",
+                      "resolved": "https://registry.npmjs.org/nopt/-/nopt-3.0.6.tgz"
+                    },
                     "oauth-sign": {
                       "version": "0.8.2",
                       "from": "oauth-sign@>=0.8.1 <0.9.0",
@@ -3330,41 +3371,41 @@
                       "from": "qs@>=6.1.0 <6.2.0",
                       "resolved": "https://registry.npmjs.org/qs/-/qs-6.1.0.tgz"
                     },
-                    "request": {
-                      "version": "2.72.0",
-                      "from": "request@>=2.0.0 <3.0.0",
-                      "resolved": "https://registry.npmjs.org/request/-/request-2.72.0.tgz"
-                    },
                     "readable-stream": {
                       "version": "2.1.2",
                       "from": "readable-stream@>=2.0.0 <3.0.0||>=1.1.13 <2.0.0",
                       "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.1.2.tgz"
                     },
+                    "request": {
+                      "version": "2.72.0",
+                      "from": "request@>=2.0.0 <3.0.0",
+                      "resolved": "https://registry.npmjs.org/request/-/request-2.72.0.tgz"
+                    },
                     "rimraf": {
                       "version": "2.5.2",
                       "from": "rimraf@>=2.5.0 <2.6.0",
                       "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.5.2.tgz"
                     },
-                    "semver": {
-                      "version": "5.1.0",
-                      "from": "semver@>=5.1.0 <5.2.0",
-                      "resolved": "https://registry.npmjs.org/semver/-/semver-5.1.0.tgz"
-                    },
                     "sntp": {
                       "version": "1.0.9",
                       "from": "sntp@>=1.0.0 <2.0.0",
                       "resolved": "https://registry.npmjs.org/sntp/-/sntp-1.0.9.tgz"
                     },
-                    "string_decoder": {
-                      "version": "0.10.31",
-                      "from": "string_decoder@>=0.10.0 <0.11.0",
-                      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz"
+                    "semver": {
+                      "version": "5.1.0",
+                      "from": "semver@>=5.1.0 <5.2.0",
+                      "resolved": "https://registry.npmjs.org/semver/-/semver-5.1.0.tgz"
                     },
                     "stringstream": {
                       "version": "0.0.5",
                       "from": "stringstream@>=0.0.4 <0.1.0",
                       "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz"
                     },
+                    "string_decoder": {
+                      "version": "0.10.31",
+                      "from": "string_decoder@>=0.10.0 <0.11.0",
+                      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz"
+                    },
                     "strip-ansi": {
                       "version": "3.0.1",
                       "from": "strip-ansi@>=3.0.0 <4.0.0",
@@ -3385,16 +3426,16 @@
                       "from": "tar@>=2.2.0 <2.3.0",
                       "resolved": "https://registry.npmjs.org/tar/-/tar-2.2.1.tgz"
                     },
-                    "tunnel-agent": {
-                      "version": "0.4.3",
-                      "from": "tunnel-agent@>=0.4.1 <0.5.0",
-                      "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.4.3.tgz"
-                    },
                     "tough-cookie": {
                       "version": "2.2.2",
                       "from": "tough-cookie@>=2.2.0 <2.3.0",
                       "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.2.2.tgz"
                     },
+                    "tunnel-agent": {
+                      "version": "0.4.3",
+                      "from": "tunnel-agent@>=0.4.1 <0.5.0",
+                      "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.4.3.tgz"
+                    },
                     "tweetnacl": {
                       "version": "0.13.3",
                       "from": "tweetnacl@>=0.13.0 <0.14.0",
@@ -3538,9 +3579,9 @@
           }
         },
         "mapnik": {
-          "version": "1.4.15-cdb6",
-          "from": "https://github.com/CartoDB/node-mapnik/tarball/1.4.15-cdb6",
-          "resolved": "https://github.com/CartoDB/node-mapnik/tarball/1.4.15-cdb6",
+          "version": "1.4.15-cdb8",
+          "from": "https://github.com/CartoDB/node-mapnik/tarball/1.4.15-cdb8",
+          "resolved": "https://github.com/CartoDB/node-mapnik/tarball/1.4.15-cdb8",
           "dependencies": {
             "nan": {
               "version": "1.2.0",
@@ -3957,14 +3998,14 @@
           }
         },
         "tilelive-bridge": {
-          "version": "1.3.0-cdb4",
-          "from": "https://github.com/CartoDB/tilelive-bridge/tarball/1.3.0-cdb4",
-          "resolved": "https://github.com/CartoDB/tilelive-bridge/tarball/1.3.0-cdb4",
+          "version": "1.3.0-cdb5",
+          "from": "https://github.com/CartoDB/tilelive-bridge/tarball/1.3.0-cdb5",
+          "resolved": "https://github.com/CartoDB/tilelive-bridge/tarball/1.3.0-cdb5",
           "dependencies": {
             "mapnik-pool": {
-              "version": "0.1.1-cdb4",
-              "from": "https://github.com/CartoDB/mapnik-pool/tarball/0.1.1-cdb4",
-              "resolved": "https://github.com/CartoDB/mapnik-pool/tarball/0.1.1-cdb4",
+              "version": "0.1.1-cdb5",
+              "from": "https://github.com/CartoDB/mapnik-pool/tarball/0.1.1-cdb5",
+              "resolved": "https://github.com/CartoDB/mapnik-pool/tarball/0.1.1-cdb5",
               "dependencies": {
                 "generic-pool": {
                   "version": "2.1.1",
@@ -3981,9 +4022,9 @@
           }
         },
         "tilelive-mapnik": {
-          "version": "0.6.15-cdb5",
-          "from": "https://github.com/CartoDB/tilelive-mapnik/tarball/0.6.15-cdb5",
-          "resolved": "https://github.com/CartoDB/tilelive-mapnik/tarball/0.6.15-cdb5",
+          "version": "0.6.15-cdb7",
+          "from": "https://github.com/CartoDB/tilelive-mapnik/tarball/0.6.15-cdb7",
+          "resolved": "https://github.com/CartoDB/tilelive-mapnik/tarball/0.6.15-cdb7",
           "dependencies": {
             "generic-pool": {
               "version": "2.1.1",

package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "name": "windshaft-cartodb",
-  "version": "2.44.0",
+  "version": "2.53.2",
   "description": "A map tile server for CartoDB",
   "keywords": [
     "cartodb"
@@ -20,7 +20,7 @@
   ],
   "dependencies": {
     "body-parser": "~1.14.0",
-    "camshaft": "0.11.0",
+    "camshaft": "0.22.2",
     "cartodb-psql": "~0.6.1",
     "cartodb-query-tables": "~0.1.0",
     "cartodb-redis": "~0.13.0",
@@ -37,9 +37,9 @@
     "request": "~2.62.0",
     "step": "~0.0.6",
     "step-profiler": "~0.3.0",
-    "turbo-carto": "0.10.0",
+    "turbo-carto": "0.12.1",
     "underscore": "~1.6.0",
-    "windshaft": "1.19.0"
+    "windshaft": "2.3.0"
   },
   "devDependencies": {
     "istanbul": "~0.4.3",

run_tests.sh

@@ -5,6 +5,7 @@ OPT_CREATE_PGSQL=yes # create the PostgreSQL test environment
 OPT_DROP_REDIS=yes   # drop the redis test environment
 OPT_DROP_PGSQL=yes   # drop the PostgreSQL test environment
 OPT_COVERAGE=no      # run tests with coverage
+OPT_DOWNLOAD_SQL=yes # download a fresh copy of sql files
 
 export PGAPPNAME=cartodb_tiler_tester
 
@@ -73,6 +74,10 @@ while [ -n "$1" ]; do
                 OPT_CREATE_REDIS=no
                 shift
                 continue
+        elif test "$1" = "--no-sql-download"; then
+                OPT_DOWNLOAD_SQL=no
+                shift
+                continue
         elif test "$1" = "--with-coverage"; then
                 OPT_COVERAGE=yes
                 shift
@@ -113,6 +118,9 @@ fi
 if test x"$OPT_CREATE_REDIS" != xyes; then
   PREPARE_DB_OPTS="$PREPARE_DB_OPTS --skip-redis"
 fi
+if test x"$OPT_DOWNLOAD_SQL" != xyes; then
+  PREPARE_DB_OPTS="$PREPARE_DB_OPTS --no-sql-download"
+fi
 
 echo "Preparing the environment"
 cd ${BASEDIR}/test/support

test/acceptance/analysis/error-cases.js

@@ -20,6 +20,13 @@ describe('analysis-layers error cases', function() {
         }
     };
 
+    var AUTH_ERROR_RESPONSE = {
+        status: 403,
+        headers: {
+            'Content-Type': 'application/json; charset=utf-8'
+        }
+    };
+
     it('should handle missing analysis nodes for layers', function(done) {
         var mapConfig = createMapConfig(
             [
@@ -64,4 +71,200 @@ describe('analysis-layers error cases', function() {
             testClient.drain(done);
         });
     });
+
+    it('should handle missing analyses when layers point to nonexistent one', function(done) {
+        var mapConfig = createMapConfig(
+            [
+                {
+                    "type": "http",
+                    "options": {
+                        "urlTemplate": "http://{s}.basemaps.cartocdn.com/light_only_labels/{z}/{x}/{y}.png",
+                        "subdomains": "abcd"
+                    }
+                },
+                {
+                    "type": "cartodb",
+                    "options": {
+                        "source": {
+                            "id": "ID-FOR-NONEXISTENT-ANALYSIS"
+                        },
+                        "cartocss": '#polygons { polygon-fill: red; }',
+                        "cartocss_version": "2.3.0"
+                    }
+                }
+            ]
+        );
+
+        var testClient = new TestClient(mapConfig, 1234);
+
+        testClient.getLayergroup(ERROR_RESPONSE, function(err, layergroupResult) {
+            assert.ok(!err, err);
+
+            assert.equal(layergroupResult.errors.length, 1);
+            assert.equal(
+                layergroupResult.errors[0],
+                'Missing analysis node.id="ID-FOR-NONEXISTENT-ANALYSIS" for layer=1'
+            );
+
+            testClient.drain(done);
+        });
+    });
+
+    it('should handle missing analyses when dataviews point to nonexistent one', function(done) {
+        var mapConfig = createMapConfig(
+            [
+                {
+                    "type": "http",
+                    "options": {
+                        "urlTemplate": "http://{s}.basemaps.cartocdn.com/light_only_labels/{z}/{x}/{y}.png",
+                        "subdomains": "abcd"
+                    }
+                },
+                {
+                    "type": "cartodb",
+                    "options": {
+                        "sql": "select * from populated_places_simple_reduced",
+                        "cartocss": '#polygons { polygon-fill: red; }',
+                        "cartocss_version": "2.3.0"
+                    }
+                }
+            ],
+            {
+                pop_max_histogram: {
+                    source: {
+                        id: 'ID-FOR-NONEXISTENT-ANALYSIS'
+                    },
+                    type: 'histogram',
+                    options: {
+                        column: 'pop_max'
+                    }
+                }
+            }
+        );
+
+        var testClient = new TestClient(mapConfig, 1234);
+
+        testClient.getLayergroup(ERROR_RESPONSE, function(err, layergroupResult) {
+            assert.ok(!err, err);
+
+            assert.equal(layergroupResult.errors.length, 1);
+            assert.equal(layergroupResult.errors[0], 'Node with `source.id="ID-FOR-NONEXISTENT-ANALYSIS"`' +
+                ' not found in analyses for dataview "pop_max_histogram"');
+
+            testClient.drain(done);
+        });
+    });
+
+    it('camshaft: should return error missing analysis nodes for layers with some context', function(done) {
+        var mapConfig = createMapConfig(
+            [
+                {
+                    "type": "cartodb",
+                    "options": {
+                        "source": {
+                            "id": "HEAD"
+                        },
+                        "cartocss": '#polygons { polygon-fill: red; }',
+                        "cartocss_version": "2.3.0"
+                    }
+                }
+            ],
+            {},
+            [
+                {
+                    "id": "HEAD",
+                    "type": "buffer",
+                    "params": {
+                        "source": {
+                            "id": "HEAD",
+                            "type": "source",
+                            "params": {
+                                "query": "select * from populated_places_simple_reduced"
+                            }
+                        },
+                        "radius": 50000
+                    }
+                }
+            ]
+        );
+
+        var testClient = new TestClient(mapConfig, 11111);
+
+        testClient.getLayergroup(AUTH_ERROR_RESPONSE, function(err, layergroupResult) {
+            assert.ok(!err, err);
+
+            assert.equal(layergroupResult.errors.length, 1);
+            assert.equal(
+                layergroupResult.errors[0],
+                'Analysis requires authentication with API key: permission denied.'
+            );
+
+            assert.equal(layergroupResult.errors_with_context[0].type, 'analysis');
+            assert.equal(
+                layergroupResult.errors_with_context[0].message,
+                'Analysis requires authentication with API key: permission denied.'
+            );
+            assert.equal(layergroupResult.errors_with_context[0].context.layer.index, 0);
+            assert.equal(layergroupResult.errors_with_context[0].context.layer.id, 'HEAD');
+            assert.equal(layergroupResult.errors_with_context[0].context.layer.type, 'buffer');
+
+            testClient.drain(done);
+        });
+    });
+
+
+    it('camshaft: should return error: Missing required param "radius"; with context', function(done) {
+        var mapConfig = createMapConfig(
+            [
+                {
+                    "type": "cartodb",
+                    "options": {
+                        "source": {
+                            "id": "HEAD"
+                        },
+                        "cartocss": '#polygons { polygon-fill: red; }',
+                        "cartocss_version": "2.3.0"
+                    }
+                }
+            ],
+            {},
+            [
+                {
+                    "id": "HEAD",
+                    "type": "buffer",
+                    "params": {
+                        "source": {
+                            "id": "HEAD",
+                            "type": "source",
+                            "params": {
+                                "query": "select * from populated_places_simple_reduced"
+                            }
+                        }
+                    }
+                }
+            ]
+        );
+
+        var testClient = new TestClient(mapConfig, 1234);
+
+        testClient.getLayergroup(ERROR_RESPONSE, function(err, layergroupResult) {
+            assert.ok(!err, err);
+
+            assert.equal(layergroupResult.errors.length, 1);
+            assert.equal(
+                layergroupResult.errors[0],
+                'Missing required param "radius"'
+            );
+
+            assert.equal(layergroupResult.errors_with_context[0].type, 'analysis');
+            assert.equal(layergroupResult.errors_with_context[0].message, 'Missing required param "radius"');
+            assert.equal(layergroupResult.errors_with_context[0].context.layer.index, 0);
+            assert.equal(layergroupResult.errors_with_context[0].context.layer.id, 'HEAD');
+            assert.equal(layergroupResult.errors_with_context[0].context.layer.type, 'buffer');
+
+            testClient.drain(done);
+        });
+    });
+
+
 });

test/acceptance/dataviews/aggregation.js

@@ -0,0 +1,59 @@
+require('../../support/test_helper');
+
+var assert = require('../../support/assert');
+var TestClient = require('../../support/test-client');
+
+describe('aggregations', function() {
+
+    afterEach(function(done) {
+        if (this.testClient) {
+            this.testClient.drain(done);
+        } else {
+            done();
+        }
+    });
+
+    function aggregationOperationMapConfig(operation) {
+        return {
+            version: '1.5.0',
+            layers: [
+                {
+                    type: 'mapnik',
+                    options: {
+                        sql: 'select * from populated_places_simple_reduced',
+                        cartocss: '#layer0 { marker-fill: red; marker-width: 10; }',
+                        cartocss_version: '2.0.1',
+                        widgets: {
+                            adm0name: {
+                                type: 'aggregation',
+                                options: {
+                                    column: 'adm0name',
+                                    aggregation: operation,
+                                    aggregationColumn: 'pop_max'
+                                }
+                            }
+                        }
+                    }
+                }
+            ]
+        };
+    }
+
+    var operations = ['count', 'sum', 'avg', 'max', 'min'];
+
+    operations.forEach(function(operation) {
+        it('should be able to use "' + operation + '" as aggregation operation', function(done) {
+
+            this.testClient = new TestClient(aggregationOperationMapConfig(operation));
+            this.testClient.getDataview('adm0name', { own_filter: 0 }, function (err, aggregation) {
+                assert.ok(!err, err);
+                assert.ok(aggregation);
+
+                assert.equal(aggregation.type, 'aggregation');
+                assert.equal(aggregation.aggregation, operation);
+
+                done();
+            });
+        });
+    });
+});

test/acceptance/dataviews/error-cases.js

@@ -0,0 +1,87 @@
+require('../../support/test_helper');
+
+var assert = require('../../support/assert');
+var TestClient = require('../../support/test-client');
+
+describe('histogram-dataview', function() {
+
+    afterEach(function(done) {
+        if (this.testClient) {
+            this.testClient.drain(done);
+        } else {
+            done();
+        }
+    });
+
+    var ERROR_RESPONSE = {
+        status: 400,
+        headers: {
+            'Content-Type': 'application/json; charset=utf-8'
+        }
+    };
+
+    function createMapConfig(dataviews) {
+        return {
+            version: '1.5.0',
+            layers: [
+                {
+                    "type": "cartodb",
+                    "options": {
+                        "source": {
+                            "id": "HEAD"
+                        },
+                        "cartocss": "#points { marker-width: 10; marker-fill: red; }",
+                        "cartocss_version": "2.3.0"
+                    }
+                }
+            ],
+            dataviews: dataviews,
+            analyses: [
+                {
+                    "id": "HEAD",
+                    "type": "source",
+                    "params": {
+                        "query": "select null::geometry the_geom_webmercator, x from generate_series(0,1000) x"
+                    }
+                }
+            ]
+        };
+    }
+
+    it('should fail when invalid dataviews object is provided, string case', function(done) {
+        var mapConfig = createMapConfig("wadus-string");
+        this.testClient = new TestClient(mapConfig, 1234);
+        this.testClient.getLayergroup(ERROR_RESPONSE, function(err, errObj) {
+            assert.ok(!err, err);
+
+            assert.deepEqual(errObj.errors, [ '"dataviews" must be a valid JSON object: "string" type found' ]);
+
+            done();
+        });
+    });
+
+    it('should fail when invalid dataviews object is provided, array case', function(done) {
+        var mapConfig = createMapConfig([]);
+        this.testClient = new TestClient(mapConfig, 1234);
+        this.testClient.getLayergroup(ERROR_RESPONSE, function(err, errObj) {
+            assert.ok(!err, err);
+
+            assert.deepEqual(errObj.errors, [ '"dataviews" must be a valid JSON object: "array" type found' ]);
+
+            done();
+        });
+    });
+
+    it('should work with empty but valid objects', function(done) {
+        var mapConfig = createMapConfig({});
+        this.testClient = new TestClient(mapConfig, 1234);
+        this.testClient.getLayergroup(function(err, layergroup) {
+            assert.ok(!err, err);
+
+            assert.ok(layergroup);
+            assert.ok(layergroup.layergroupid);
+
+            done();
+        });
+    });
+});

test/acceptance/dataviews/histogram.js

@@ -0,0 +1,80 @@
+require('../../support/test_helper');
+
+var assert = require('../../support/assert');
+var TestClient = require('../../support/test-client');
+
+describe('histogram-dataview', function() {
+
+    afterEach(function(done) {
+        if (this.testClient) {
+            this.testClient.drain(done);
+        } else {
+            done();
+        }
+    });
+
+    function createMapConfig(layers, dataviews, analysis) {
+        return {
+            version: '1.5.0',
+            layers: layers,
+            dataviews: dataviews || {},
+            analyses: analysis || []
+        };
+    }
+
+    var mapConfig = createMapConfig(
+        [
+            {
+                "type": "cartodb",
+                "options": {
+                    "source": {
+                        "id": "2570e105-7b37-40d2-bdf4-1af889598745"
+                    },
+                    "cartocss": "#points { marker-width: 10; marker-fill: red; }",
+                    "cartocss_version": "2.3.0"
+                }
+            }
+        ],
+        {
+            pop_max_histogram: {
+                source: {
+                    id: '2570e105-7b37-40d2-bdf4-1af889598745'
+                },
+                type: 'histogram',
+                options: {
+                    column: 'x'
+                }
+            }
+        },
+        [
+            {
+                "id": "2570e105-7b37-40d2-bdf4-1af889598745",
+                "type": "source",
+                "params": {
+                    "query": "select null::geometry the_geom_webmercator, x from generate_series(0,1000) x"
+                }
+            }
+        ]
+    );
+
+    it('should get bin_width right when max > min in filter', function(done) {
+        var params = {
+            bins: 10,
+            start: 1e3,
+            end: 0
+        };
+
+        this.testClient = new TestClient(mapConfig, 1234);
+        this.testClient.getDataview('pop_max_histogram', params, function(err, dataview) {
+            assert.ok(!err, err);
+
+            assert.equal(dataview.type, 'histogram');
+            assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width);
+            dataview.bins.forEach(function(bin) {
+                assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin));
+            });
+
+            done();
+        });
+    });
+});

test/acceptance/geojson-renderer.js

@@ -0,0 +1,343 @@
+require('../support/test_helper');
+
+var assert = require('../support/assert');
+var TestClient = require('../support/test-client');
+
+
+describe('use only needed columns', function() {
+
+    function getFeatureByCartodbId(features, cartodbId) {
+        for (var i = 0, len = features.length; i < len; i++) {
+            if (features[i].properties.cartodb_id === cartodbId) {
+                return features[i];
+            }
+        }
+        return {};
+    }
+
+    var options = { format: 'geojson', layer: 0 };
+
+    afterEach(function(done) {
+        if (this.testClient) {
+            this.testClient.drain(done);
+        } else {
+            done();
+        }
+    });
+
+    it('with aggregation widget, interactivity and cartocss columns', function(done) {
+        var widgetMapConfig = {
+            version: '1.5.0',
+            layers: [{
+                type: 'mapnik',
+                options: {
+                    sql: 'select * from populated_places_simple_reduced',
+                    cartocss: '#layer0 { marker-fill: red; marker-width: 10; [name="Madrid"] { marker-fill: green; } }',
+                    cartocss_version: '2.0.1',
+                    widgets: {
+                        adm0name: {
+                            type: 'aggregation',
+                            options: {
+                                column: 'adm0name',
+                                aggregation: 'sum',
+                                aggregationColumn: 'pop_max'
+                            }
+                        }
+                    },
+                    interactivity: "cartodb_id,pop_min"
+                }
+            }]
+        };
+
+        this.testClient = new TestClient(widgetMapConfig);
+        this.testClient.getTile(0, 0, 0, options, function (err, res, geojsonTile) {
+            assert.ok(!err, err);
+            assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, {
+                cartodb_id: 1109,
+                name: 'Mardin',
+                adm0name: 'Turkey',
+                pop_max: 71373,
+                pop_min: 57586
+            });
+            done();
+        });
+    });
+
+    it('should not duplicate columns', function(done) {
+        var widgetMapConfig = {
+            version: '1.5.0',
+            layers: [{
+                type: 'mapnik',
+                options: {
+                    sql: 'select * from populated_places_simple_reduced',
+                    cartocss: ['#layer0 {',
+                        'marker-fill: red;',
+                        'marker-width: 10;',
+                        '[name="Madrid"] { marker-fill: green; } ',
+                        '[pop_max>100000] { marker-fill: black; } ',
+                        '}'].join('\n'),
+                    cartocss_version: '2.3.0',
+                    widgets: {
+                        adm0name: {
+                            type: 'aggregation',
+                            options: {
+                                column: 'adm0name',
+                                aggregation: 'sum',
+                                aggregationColumn: 'pop_max'
+                            }
+                        }
+                    },
+                    interactivity: "cartodb_id,pop_max"
+                }
+            }]
+        };
+
+        this.testClient = new TestClient(widgetMapConfig);
+        this.testClient.getTile(0, 0, 0, options, function (err, res, geojsonTile) {
+            assert.ok(!err, err);
+            assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, {
+                cartodb_id: 1109,
+                name: 'Mardin',
+                adm0name: 'Turkey',
+                pop_max: 71373
+            });
+            done();
+        });
+    });
+
+    it('with formula widget, no interactivity and no cartocss columns', function(done) {
+        var formulaWidgetMapConfig = {
+            version: '1.5.0',
+            layers: [{
+                type: 'mapnik',
+                options: {
+                    sql: 'select * from populated_places_simple_reduced where pop_max > 0 and pop_max < 600000',
+                    cartocss: '#layer0 { marker-fill: red; marker-width: 10; }',
+                    cartocss_version: '2.0.1',
+                    interactivity: 'cartodb_id',
+                    widgets: {
+                        pop_max_f: {
+                            type: 'formula',
+                            options: {
+                                column: 'pop_max',
+                                operation: 'count'
+                            }
+                        }
+                    }
+                }
+            }]
+        };
+
+        this.testClient = new TestClient(formulaWidgetMapConfig);
+        this.testClient.getTile(0, 0, 0, options, function (err, res, geojsonTile) {
+            assert.ok(!err, err);
+            assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, {
+                cartodb_id: 1109,
+                pop_max: 71373
+            });
+            done();
+        });
+    });
+    it('with cartocss with multiple expressions', function(done) {
+        var formulaWidgetMapConfig = {
+            version: '1.5.0',
+            layers: [{
+                type: 'mapnik',
+                options: {
+                    sql: 'select * from populated_places_simple_reduced where pop_max > 0 and pop_max < 600000',
+                    cartocss: '#layer0 { marker-fill: red; marker-width: 10; }' +
+                        '#layer0 { marker-width: 14; [name="Madrid"] { marker-width: 20; } }' +
+                        '#layer0[pop_max>1000] { marker-width: 14; [name="Madrid"] { marker-width: 20; } }' +
+                        '#layer0[adm0name=~".*Turkey*"] { marker-width: 14; [name="Madrid"] { marker-width: 20; } }',
+                    cartocss_version: '2.0.1',
+                    interactivity: 'cartodb_id'
+                }
+            }]
+        };
+
+        this.testClient = new TestClient(formulaWidgetMapConfig);
+        this.testClient.getTile(0, 0, 0, options, function (err, res, geojsonTile) {
+            assert.ok(!err, err);
+            assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, {
+                cartodb_id: 1109,
+                pop_max:71373,
+                name:"Mardin",
+                adm0name:"Turkey"
+            });
+            done();
+        });
+    });
+
+    it('should work with mapnik substitution tokens', function(done) {
+        var cartocss = [
+            "#layer {",
+            "  line-width: 2;",
+            "  line-color: #3B3B58;",
+            "  line-opacity: 1;",
+            "  polygon-opacity: 0.7;",
+            "  polygon-fill: ramp([points_count], (#E5F5F9,#99D8C9,#2CA25F))",
+            "}"
+        ].join('\n');
+
+        var sql = [
+            'WITH hgrid AS (',
+            '  SELECT CDB_HexagonGrid(',
+            '    ST_Expand(!bbox!, greatest(!pixel_width!,!pixel_height!) * 100),',
+            '    greatest(!pixel_width!,!pixel_height!) * 100',
+            '  ) as cell',
+            ')',
+            'SELECT',
+            '  hgrid.cell as the_geom_webmercator,',
+            '  count(1) as points_count,',
+            '  count(1)/power(100 * CDB_XYZ_Resolution(CDB_ZoomFromScale(!scale_denominator!)), 2) as points_density,',
+            '  1 as cartodb_id',
+            'FROM hgrid, (SELECT * FROM populated_places_simple_reduced) i',
+            'where ST_Intersects(i.the_geom_webmercator, hgrid.cell)',
+            'GROUP BY hgrid.cell'
+        ].join('\n');
+
+        var mapConfig = {
+            "version": "1.4.0",
+            "layers": [
+                {
+                    "type": 'mapnik',
+                    "options": {
+                        "cartocss_version": '2.3.0',
+                        "sql": sql,
+                        "cartocss": cartocss
+                    }
+                }
+            ]
+        };
+
+        this.testClient = new TestClient(mapConfig);
+        this.testClient.getTile(0, 0, 0, { format: 'geojson', layer: 0 }, function(err, res, geojson) {
+            assert.ok(!err, err);
+
+            assert.ok(geojson);
+            assert.equal(geojson.features.length, 5);
+
+            done();
+        });
+    });
+
+    it('should skip empty and null columns for geojson tiles', function(done) {
+
+        var mapConfig = {
+            "analyses": [
+                {
+                    "id": "a0",
+                    "params": {
+                        "query": "SELECT * FROM test_table"
+                    },
+                    "type": "source"
+                }
+            ],
+            "dataviews": {
+                "4e7b0e07-6d21-4b83-9adb-6d7e17eea6ca": {
+                    "options": {
+                        "aggregationColumn": null,
+                        "column": "cartodb_id",
+                        "operation": "avg"
+                    },
+                    "source": {
+                        "id": "a0"
+                    },
+                    "type": "formula"
+                },
+                "74f590f8-625c-4e95-922f-34ad3e9919c0": {
+                    "options": {
+                        "aggregation": "sum",
+                        "aggregationColumn": "cartodb_id",
+                        "column": "name"
+                    },
+                    "source": {
+                        "id": "a0"
+                    },
+                    "type": "aggregation"
+                },
+                "98a75757-3006-400a-b028-fb613a6c0b69": {
+                    "options": {
+                        "aggregationColumn": null,
+                        "column": "cartodb_id",
+                        "operation": "sum"
+                    },
+                    "source": {
+                        "id": "a0"
+                    },
+                    "type": "formula"
+                },
+                "ebbc97b2-87d2-4895-9e1f-2f012df3679d": {
+                    "options": {
+                        "aggregationColumn": null,
+                        "bins": "12",
+                        "column": "cartodb_id"
+                    },
+                    "source": {
+                        "id": "a0"
+                    },
+                    "type": "histogram"
+                },
+                "ebc0653f-3581-469c-8b31-c969e440a865": {
+                    "options": {
+                        "aggregationColumn": null,
+                        "column": "cartodb_id",
+                        "operation": "avg"
+                    },
+                    "source": {
+                        "id": "a0"
+                    },
+                    "type": "formula"
+                }
+            },
+            "layers": [
+                {
+                    "options": {
+                        "subdomains": "abcd",
+                        "urlTemplate": "http://{s}.basemaps.cartocdn.com/light_nolabels/{z}/{x}/{y}.png"
+                    },
+                    "type": "http"
+                },
+                {
+                    "options": {
+                        "attributes": {
+                            "columns": [
+                                "name",
+                                "address"
+                            ],
+                            "id": "cartodb_id"
+                        },
+                        "cartocss": "#layer { marker-width: 10; marker-fill: red; }",
+                        "cartocss_version": "2.3.0",
+                        "interactivity": "cartodb_id",
+                        "layer_name": "wadus",
+                        "source": {
+                            "id": "a0"
+                        }
+                    },
+                    "type": "cartodb"
+                },
+                {
+                    "options": {
+                        "subdomains": "abcd",
+                        "urlTemplate": "http://{s}.basemaps.cartocdn.com/light_only_labels/{z}/{x}/{y}.png"
+                    },
+                    "type": "http"
+                }
+            ]
+        };
+
+        this.testClient = new TestClient(mapConfig);
+        this.testClient.getTile(0, 0, 0, { format: 'geojson', layer: 0 }, function(err, res, geojson) {
+            assert.ok(!err, err);
+
+            assert.ok(geojson);
+            assert.equal(geojson.features.length, 5);
+
+            assert.deepEqual(Object.keys(geojson.features[0].properties), ['cartodb_id', 'name']);
+
+            done();
+        });
+    });
+
+});

test/acceptance/limits.js

@@ -106,7 +106,7 @@ describe('render limits', function() {
                     },
                     function(res) {
                         var parsed = JSON.parse(res.body);
-                        assert.deepEqual(parsed, { errors: [ 'Render timed out' ] });
+                        assert.deepEqual(parsed.errors, [ 'Render timed out' ]);
                         done();
                     }
                 );
@@ -171,7 +171,7 @@ describe('render limits', function() {
                             },
                             function(res) {
                                 var parsed = JSON.parse(res.body);
-                                assert.deepEqual(parsed, { errors: ['Render timed out'] });
+                                assert.deepEqual(parsed.errors, ['Render timed out']);
                                 done();
                             }
                         );

test/acceptance/multilayer_server.js

@@ -228,7 +228,7 @@ describe('tests from old api translated to multilayer', function() {
                 },
                 function(res) {
                     var parsed = JSON.parse(res.body);
-                    assert.deepEqual(parsed, { errors: [ 'Unexpected token W' ] });
+                    assert.deepEqual(parsed.errors, [ 'Unexpected token W' ]);
 
                     done();
                 }
@@ -334,9 +334,7 @@ describe('tests from old api translated to multilayer', function() {
                 assert.ok(!res.headers.hasOwnProperty('x-cache-channel'));
 
                 var parsed = JSON.parse(res.body);
-                assert.deepEqual(parsed, {
-                    errors: ["fake error message"]
-                });
+                assert.deepEqual(parsed.errors, ["fake error message"]);
 
                 done();
             }

test/acceptance/named_layers.js

@@ -181,7 +181,7 @@ describe('named_layers', function() {
                 }
 
                 var parsedBody = JSON.parse(response.body);
-                assert.deepEqual(parsedBody, { errors: ["Template 'nonexistent' of user 'localhost' not found"] });
+                assert.deepEqual(parsedBody.errors, ["Template 'nonexistent' of user 'localhost' not found"]);
 
                 return null;
             },
@@ -234,10 +234,7 @@ describe('named_layers', function() {
                 }
 
                 var parsedBody = JSON.parse(response.body);
-                assert.deepEqual(
-                    parsedBody,
-                    { errors: [ "Unauthorized 'auth_valid_template' template instantiation" ] }
-                );
+                assert.deepEqual(parsedBody.errors, [ "Unauthorized 'auth_valid_template' template instantiation" ]);
 
                 return null;
             },
@@ -347,7 +344,7 @@ describe('named_layers', function() {
                 }
 
                 var parsedBody = JSON.parse(response.body);
-                assert.deepEqual(parsedBody, { errors: [ 'Nested named layers are not allowed' ] });
+                assert.deepEqual(parsedBody.errors, ['Nested named layers are not allowed' ]);
 
                 return null;
             },

test/acceptance/named_maps_authentication.js

@@ -169,8 +169,8 @@ describe('named maps authentication', function() {
             getNamedTile(nonexistentName, 0, 0, 0, { status: 404 }, function(err, res) {
                 assert.ok(!err);
                 assert.deepEqual(
-                    JSON.parse(res.body),
-                    { errors: ["Template '" + nonexistentName + "' of user '" + username + "' not found"] }
+                    JSON.parse(res.body).errors,
+                    ["Template '" + nonexistentName + "' of user '" + username + "' not found"]
                 );
                 done();
             });
@@ -179,7 +179,7 @@ describe('named maps authentication', function() {
         it('should return 403 if not properly authorized', function(done) {
             getNamedTile(tokenAuthTemplateName, 0, 0, 0, { status: 403 }, function(err, res) {
                 assert.ok(!err);
-                assert.deepEqual(JSON.parse(res.body), { errors: ['Unauthorized template instantiation'] });
+                assert.deepEqual(JSON.parse(res.body).errors, ['Unauthorized template instantiation']);
                 done();
             });
         });
@@ -238,8 +238,8 @@ describe('named maps authentication', function() {
             getStaticMap(nonexistentName, { status: 404 }, function(err, res) {
                 assert.ok(!err);
                 assert.deepEqual(
-                    JSON.parse(res.body),
-                    { errors: ["Template '" + nonexistentName + "' of user '" + username + "' not found"] }
+                    JSON.parse(res.body).errors,
+                    ["Template '" + nonexistentName + "' of user '" + username + "' not found"]
                 );
                 done();
             });
@@ -248,7 +248,7 @@ describe('named maps authentication', function() {
         it('should return 403 if not properly authorized', function(done) {
             getStaticMap(tokenAuthTemplateName, { status: 403 }, function(err, res) {
                 assert.ok(!err);
-                assert.deepEqual(JSON.parse(res.body), { errors: ['Unauthorized template instantiation'] });
+                assert.deepEqual(JSON.parse(res.body).errors, ['Unauthorized template instantiation']);
                 done();
             });
         });

test/acceptance/named_maps_cache.js

@@ -124,8 +124,8 @@ describe('named maps provider cache', function() {
                         getNamedTile({ statusCode: 404 }, function(err, res) {
                             assert.ok(!err);
                             assert.deepEqual(
-                                JSON.parse(res.body),
-                                { errors: ["Template 'template_with_color' of user 'localhost' not found"] }
+                                JSON.parse(res.body).errors,
+                                ["Template 'template_with_color' of user 'localhost' not found"]
                             );
 
                             // add template again so it's clean in afterEach

test/acceptance/ported/attributes.js

@@ -231,7 +231,11 @@ describe('attributes', function() {
                 assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
                 assert.equal(
                     res.body,
-                    '/**/ typeof test === \'function\' && test({"errors":["Layer 0 has no exposed attributes"]});'
+                    '/**/ typeof test === \'function\' && ' +
+                    'test({"errors":["Layer 0 has no exposed attributes"],' +
+                    '"errors_with_context":[{' +
+                    '"type":"unknown","message":"Layer 0 has no exposed attributes","context":"unknown"' +
+                    '}]});'
                 );
                 return null;
             },

test/acceptance/ported/blend_http_fallback.js

@@ -138,11 +138,9 @@ describe('blend http fallback', function() {
         testClient.getTileLayer(mapConfig, tileRequest, expectedResponse, function(err, res) {
             assert.ok(!err);
             var parsedBody = JSON.parse(res.body);
-            assert.deepEqual(parsedBody, {
-                errors: [
-                    "Unable to fetch http tile: http://127.0.0.1:8033/error404/1/0/0.png [404]"
-                ]
-            });
+            assert.deepEqual(parsedBody.errors, [
+                "Unable to fetch http tile: http://127.0.0.1:8033/error404/1/0/0.png [404]"
+            ]);
             done();
         });
     });

test/acceptance/ported/external_resources.js

@@ -119,12 +119,11 @@ describe('external resources', function() {
         var mapConfig = testClient.defaultTableMapConfig('test_table_3', style);
 
         testClient.createLayergroup(mapConfig, { statusCode: 400 }, function(err, res) {
-            assert.deepEqual(JSON.parse(res.body), {
-                errors: ["Unable to download '" + url + "' for 'style0' (server returned 404)"]
-            });
+            assert.deepEqual(JSON.parse(res.body).errors, [
+                "Unable to download '" + url + "' for 'style0' (server returned 404)"]
+            );
             done();
         });
     });
 
 });
-

test/acceptance/ported/multilayer_error_cases.js

@@ -31,7 +31,7 @@ describe('multilayer error cases', function() {
         }, {}, function(res) {
             assert.equal(res.statusCode, 400, res.body);
             var parsedBody = JSON.parse(res.body);
-            assert.deepEqual(parsedBody, {"errors":["layergroup POST data must be of type application/json"]});
+            assert.deepEqual(parsedBody.errors, ["layergroup POST data must be of type application/json"]);
             done();
         });
     });
@@ -44,7 +44,7 @@ describe('multilayer error cases', function() {
         }, {}, function(res) {
             assert.equal(res.statusCode, 400, res.body);
             var parsedBody = JSON.parse(res.body);
-            assert.deepEqual(parsedBody, {"errors":["Missing layers array from layergroup config"]});
+            assert.deepEqual(parsedBody.errors, ["Missing layers array from layergroup config"]);
             done();
         });
     });
@@ -58,7 +58,10 @@ describe('multilayer error cases', function() {
             assert.equal(res.statusCode, 200);
             assert.equal(
                 res.body,
-                '/**/ typeof test === \'function\' && test({"errors":["Missing layers array from layergroup config"]});'
+                '/**/ typeof test === \'function\' && ' +
+                'test({"errors":["Missing layers array from layergroup config"],' +
+                '"errors_with_context":[{"type":"unknown",' +
+                '"message":"Missing layers array from layergroup config","context":"unknown"}]});'
             );
             done();
         });
@@ -83,7 +86,7 @@ describe('multilayer error cases', function() {
       }, {}, function(res) {
           assert.equal(res.statusCode, 400, res.body);
           var parsedBody = JSON.parse(res.body);
-          assert.deepEqual(parsedBody, {errors:["Missing cartocss_version for layer 0 options"]});
+          assert.deepEqual(parsedBody.errors, ["Missing cartocss_version for layer 0 options"]);
           done();
       });
     });
@@ -355,7 +358,7 @@ describe('multilayer error cases', function() {
         var mapConfig = testClient.singleLayerMapConfig('select * from test_table', null, null, 'name');
 
         testClient.getGrid(mapConfig, 1, 13, 4011, 3088, defaultErrorExpectedResponse, function(err, res) {
-            assert.deepEqual(JSON.parse(res.body), { errors: ["Layer '1' not found in layergroup"] });
+            assert.deepEqual(JSON.parse(res.body).errors, ["Layer '1' not found in layergroup"]);
             done();
         });
     });
@@ -383,7 +386,7 @@ describe('multilayer error cases', function() {
           // FIXME: should be 404
           assert.equal(res.statusCode, 400, res.statusCode + ':' + res.body);
           var parsed = JSON.parse(res.body);
-          assert.deepEqual(parsed, {"errors": ["Invalid or nonexistent map configuration token 'deadbeef'"]});
+          assert.deepEqual(parsed.errors, ["Invalid or nonexistent map configuration token 'deadbeef'"]);
           return null;
         },
         function finish(err) {

test/acceptance/ported/raster.js

@@ -148,11 +148,10 @@ describe('raster', function() {
                 assert.ok(!err);
                 checkCORSHeaders(res);
                 var parsedBody = JSON.parse(res.body);
-                assert.deepEqual(parsedBody, { errors: [ 'Mapnik raster layers do not support interactivity' ] });
+                assert.deepEqual(parsedBody.errors, [ 'Mapnik raster layers do not support interactivity' ]);
                 done();
             }
         );
     });
 
 });
-

test/acceptance/ported/regressions.js

@@ -29,7 +29,7 @@ describe('regressions', function() {
                 contentType: 'application/json; charset=utf-8'
             };
             requestTile('/0/0/0.png?testUnexpectedError=1', options, function(err, res) {
-                assert.deepEqual(JSON.parse(res.body), { "errors": ["test unexpected error"] });
+                assert.deepEqual(JSON.parse(res.body).errors, ["test unexpected error"]);
                 finish(done);
             });
         });

test/acceptance/ported/retina.js

@@ -129,7 +129,7 @@ describe('retina support', function() {
             },
             function(res, err) {
                 assert.ok(!err, 'Failed to request 0/0/0' + scaleFactor + '.png tile');
-                assert.deepEqual(JSON.parse(res.body), { errors: ["Tile with specified resolution not found"] } );
+                assert.deepEqual(JSON.parse(res.body).errors, ["Tile with specified resolution not found"]);
 
                 done();
             }

test/acceptance/ported/server.js

@@ -112,7 +112,7 @@ describe('server', function() {
             }
         };
         testClient.getGrid(mapConfig, 0, 13, 4011, 3088, expectedResponse, function(err, res) {
-            assert.deepEqual(JSON.parse(res.body), {"errors":["Tileset has no interactivity"]});
+            assert.deepEqual(JSON.parse(res.body).errors, ["Tileset has no interactivity"]);
             done();
         });
     });

test/acceptance/templates.js

@@ -1927,9 +1927,8 @@ describe('template_api', function() {
                 if (err) {
                     return done(err);
                 }
-                assert.deepEqual(JSON.parse(res.body), {
-                    errors: ["Invalid or nonexistent map configuration token '" + nonexistentToken + "'"]
-                });
+                assert.deepEqual(JSON.parse(res.body).errors,
+                    ["Invalid or nonexistent map configuration token '" + nonexistentToken + "'"]);
 
                 done();
             };

test/acceptance/turbo-cartocss/error-cases.js

@@ -106,4 +106,34 @@ describe('turbo-carto error cases', function() {
             done();
         });
     });
+
+    it('turbo-carto: should return error invalid column from datasource with some context', function(done) {
+        this.testClient = new TestClient(makeMapconfig(null, 'ramp([wadus_column], (red, green, blue))'));
+        this.testClient.getLayergroup(ERROR_RESPONSE, function(err, layergroup) {
+            assert.ok(!err, err);
+
+            assert.ok(layergroup.hasOwnProperty('errors'));
+            assert.equal(layergroup.errors_with_context.length, 1);
+            assert.equal(layergroup.errors_with_context[0].type, 'turbo-carto');
+            assert.ok(layergroup.errors_with_context[0].message.match(/^turbo-carto/));
+            assert.ok(layergroup.errors_with_context[0].message.match(/unable\sto\scompute\sramp/i));
+            assert.ok(layergroup.errors_with_context[0].message.match(/wadus_column/));
+
+            assert.equal(layergroup.errors_with_context[0].context.layer.index, 0);
+            assert.equal(layergroup.errors_with_context[0].context.layer.type, 'mapnik');
+
+            assert.equal(layergroup.errors_with_context[0].context.selector, '#populated_places_simple_reduced');
+            assert.deepEqual(layergroup.errors_with_context[0].context.source, {
+                start: {
+                    line: 10,
+                    column: 3
+                },
+                end: {
+                    line: 10,
+                    column: 56
+                }
+            });
+            done();
+        });
+    });
 });

test/acceptance/turbo-cartocss/named-maps.js

@@ -22,215 +22,231 @@ describe('turbo-carto for named maps', function() {
 
     var templateId = 'turbo-carto-template-1';
 
-    var template = {
-        version: '0.0.1',
-        name: templateId,
-        auth: { method: 'open' },
-        placeholders: {
-            color: {
-                type: "css_color",
-                default: "Reds"
-            }
-        },
-        layergroup:  {
-            version: '1.0.0',
-                layers: [{
-                    options: {
-                        sql: [
-                            'SELECT test_table.*, _prices.price FROM test_table JOIN (' +
-                            '  SELECT 1 AS cartodb_id, 10.00 AS price',
-                            '  UNION',
-                            '  SELECT 2, 10.50',
-                            '  UNION',
-                            '  SELECT 3, 11.00',
-                            '  UNION',
-                            '  SELECT 4, 12.00',
-                            '  UNION',
-                            '  SELECT 5, 21.00',
-                            ') _prices ON _prices.cartodb_id = test_table.cartodb_id'
-                        ].join('\n'),
-                        cartocss: [
-                            '#layer {',
-                            '  marker-fill: ramp([price], colorbrewer(<%= color %>));',
-                            '  marker-allow-overlap:true;',
-                            '}'
-                        ].join('\n'),
-                        cartocss_version: '2.0.2'
-                    }
+    function template(table) {
+        return {
+            version: '0.0.1',
+            name: templateId,
+            auth: { method: 'open' },
+            placeholders: {
+                color: {
+                    type: "css_color",
+                    default: "Reds"
                 }
-            ]
-        }
-    };
+            },
+            layergroup: {
+                version: '1.0.0',
+                layers: [
+                    {
+                        options: {
+                            sql: [
+                                'SELECT ' + table + '.*, _prices.price FROM ' + table + ' JOIN (' +
+                                '  SELECT 1 AS cartodb_id, 10.00 AS price',
+                                '  UNION',
+                                '  SELECT 2, 10.50',
+                                '  UNION',
+                                '  SELECT 3, 11.00',
+                                '  UNION',
+                                '  SELECT 4, 12.00',
+                                '  UNION',
+                                '  SELECT 5, 21.00',
+                                ') _prices ON _prices.cartodb_id = ' + table + '.cartodb_id'
+                            ].join('\n'),
+                            cartocss: [
+                                '#layer {',
+                                '  marker-fill: ramp([price], colorbrewer(<%= color %>));',
+                                '  marker-allow-overlap:true;',
+                                '}'
+                            ].join('\n'),
+                            cartocss_version: '2.0.2'
+                        }
+                    }
+                ]
+            }
+        };
+    }
 
     var templateParamsReds = { color: 'Reds' };
     var templateParamsBlues = { color: 'Blues' };
 
-    it('should create a template with turbo-carto parsed properly', function (done) {
-        step(
-            function postTemplate() {
-                var next = this;
+    var scenarios = [
+        {
+            desc: 'with public tables',
+            table: 'test_table'
+        },
+        {
+            desc: 'with private tables',
+            table: 'test_table_private_1'
+        }
+    ];
 
-                assert.response(server, {
-                    url: '/api/v1/map/named?api_key=1234',
-                    method: 'POST',
-                    headers: { host: 'localhost', 'Content-Type': 'application/json' },
-                    data: JSON.stringify(template)
-                }, {},
-                function (res, err) {
-                    next(err, res);
-                });
-            },
-            function checkTemplate(err, res) {
-                assert.ifError(err);
-                assert.equal(res.statusCode, 200);
-                assert.deepEqual(JSON.parse(res.body), {
-                    template_id: templateId
-                });
+    scenarios.forEach(function(scenario) {
+        it('should create a template with turbo-carto parsed properly: ' + scenario.desc, function (done) {
+            step(
+                function postTemplate() {
+                    var next = this;
 
-                return null;
-            },
-            function instantiateTemplateWithReds(err) {
-                assert.ifError(err);
+                    assert.response(server, {
+                        url: '/api/v1/map/named?api_key=1234',
+                        method: 'POST',
+                        headers: { host: 'localhost', 'Content-Type': 'application/json' },
+                        data: JSON.stringify(template(scenario.table))
+                    }, {},
+                    function (res, err) {
+                        next(err, res);
+                    });
+                },
+                function checkTemplate(err, res) {
+                    assert.ifError(err);
+                    assert.equal(res.statusCode, 200);
+                    assert.deepEqual(JSON.parse(res.body), {
+                        template_id: templateId
+                    });
 
-                var next = this;
-                assert.response(server, {
-                    url: '/api/v1/map/named/' + templateId,
-                    method: 'POST',
-                    headers: {
-                        host: 'localhost',
-                        'Content-Type': 'application/json'
-                    },
-                    data: JSON.stringify(templateParamsReds)
-                }, {},
-                function(res, err) {
-                    return next(err, res);
-                });
-            },
-            function checkInstanciationWithReds(err, res) {
-                assert.ifError(err);
+                    return null;
+                },
+                function instantiateTemplateWithReds(err) {
+                    assert.ifError(err);
 
-                assert.equal(res.statusCode, 200);
+                    var next = this;
+                    assert.response(server, {
+                        url: '/api/v1/map/named/' + templateId,
+                        method: 'POST',
+                        headers: {
+                            host: 'localhost',
+                            'Content-Type': 'application/json'
+                        },
+                        data: JSON.stringify(templateParamsReds)
+                    }, {},
+                    function(res, err) {
+                        return next(err, res);
+                    });
+                },
+                function checkInstanciationWithReds(err, res) {
+                    assert.ifError(err);
 
-                var parsedBody = JSON.parse(res.body);
+                    assert.equal(res.statusCode, 200);
 
-                keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0;
-                keysToDelete['user:localhost:mapviews:global'] = 5;
+                    var parsedBody = JSON.parse(res.body);
 
-                assert.ok(parsedBody.layergroupid);
-                assert.ok(parsedBody.last_updated);
+                    keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0;
+                    keysToDelete['user:localhost:mapviews:global'] = 5;
 
-                return parsedBody.layergroupid;
-            },
-            function requestTileReds(err, layergroupId) {
-                assert.ifError(err);
+                    assert.ok(parsedBody.layergroupid);
+                    assert.ok(parsedBody.last_updated);
 
-                var next = this;
+                    return parsedBody.layergroupid;
+                },
+                function requestTileReds(err, layergroupId) {
+                    assert.ifError(err);
 
-                assert.response(server, {
-                    url: '/api/v1/map/' + layergroupId + '/0/0/0.png',
-                    method: 'GET',
-                    headers: { host: 'localhost' },
-                    encoding: 'binary'
-                }, {},
-                function(res, err) {
-                    next(err, res);
-                });
-            },
-            function checkTileReds(err, res) {
-                assert.ifError(err);
+                    var next = this;
 
-                var next = this;
+                    assert.response(server, {
+                        url: '/api/v1/map/' + layergroupId + '/0/0/0.png',
+                        method: 'GET',
+                        headers: { host: 'localhost' },
+                        encoding: 'binary'
+                    }, {},
+                    function(res, err) {
+                        next(err, res);
+                    });
+                },
+                function checkTileReds(err, res) {
+                    assert.ifError(err);
 
-                assert.equal(res.statusCode, 200);
-                assert.equal(res.headers['content-type'], 'image/png');
+                    var next = this;
 
-                var fixturePath = './test/fixtures/turbo-carto-named-maps-reds.png';
-                var image = mapnik.Image.fromBytes(new Buffer(res.body, 'binary'));
+                    assert.equal(res.statusCode, 200);
+                    assert.equal(res.headers['content-type'], 'image/png');
 
-                assert.imageIsSimilarToFile(image, fixturePath, IMAGE_TOLERANCE_PER_MIL, next);
-            },
-            function instantiateTemplateWithBlues(err) {
-                assert.ifError(err);
+                    var fixturePath = './test/fixtures/turbo-carto-named-maps-reds.png';
+                    var image = mapnik.Image.fromBytes(new Buffer(res.body, 'binary'));
 
-                var next = this;
-                assert.response(server, {
-                    url: '/api/v1/map/named/' + templateId,
-                    method: 'POST',
-                    headers: {
-                        host: 'localhost',
-                        'Content-Type': 'application/json'
-                    },
-                    data: JSON.stringify(templateParamsBlues)
-                }, {},
-                function(res, err) {
-                    return next(err, res);
-                });
-            },
-            function checkInstanciationWithBlues(err, res) {
-                assert.ifError(err);
-                assert.equal(res.statusCode, 200);
+                    assert.imageIsSimilarToFile(image, fixturePath, IMAGE_TOLERANCE_PER_MIL, next);
+                },
+                function instantiateTemplateWithBlues(err) {
+                    assert.ifError(err);
 
-                var parsedBody = JSON.parse(res.body);
+                    var next = this;
+                    assert.response(server, {
+                        url: '/api/v1/map/named/' + templateId,
+                        method: 'POST',
+                        headers: {
+                            host: 'localhost',
+                            'Content-Type': 'application/json'
+                        },
+                        data: JSON.stringify(templateParamsBlues)
+                    }, {},
+                    function(res, err) {
+                        return next(err, res);
+                    });
+                },
+                function checkInstanciationWithBlues(err, res) {
+                    assert.ifError(err);
+                    assert.equal(res.statusCode, 200);
 
-                keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0;
-                keysToDelete['user:localhost:mapviews:global'] = 5;
+                    var parsedBody = JSON.parse(res.body);
 
-                assert.ok(parsedBody.layergroupid);
-                assert.ok(parsedBody.last_updated);
+                    keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0;
+                    keysToDelete['user:localhost:mapviews:global'] = 5;
 
-                return parsedBody.layergroupid;
-            },
-            function requestTileBlues(err, layergroupId) {
-                assert.ifError(err);
+                    assert.ok(parsedBody.layergroupid);
+                    assert.ok(parsedBody.last_updated);
 
-                var next = this;
+                    return parsedBody.layergroupid;
+                },
+                function requestTileBlues(err, layergroupId) {
+                    assert.ifError(err);
 
-                assert.response(server, {
-                    url: '/api/v1/map/' + layergroupId + '/0/0/0.png',
-                    method: 'GET',
-                    headers: { host: 'localhost' },
-                    encoding: 'binary'
-                }, {},
-                function(res, err) {
-                    next(err, res);
-                });
-            },
-            function checkTileBlues(err, res) {
-                assert.ifError(err);
+                    var next = this;
 
-                var next = this;
+                    assert.response(server, {
+                        url: '/api/v1/map/' + layergroupId + '/0/0/0.png',
+                        method: 'GET',
+                        headers: { host: 'localhost' },
+                        encoding: 'binary'
+                    }, {},
+                    function(res, err) {
+                        next(err, res);
+                    });
+                },
+                function checkTileBlues(err, res) {
+                    assert.ifError(err);
 
-                assert.equal(res.statusCode, 200);
-                assert.equal(res.headers['content-type'], 'image/png');
+                    var next = this;
 
-                var fixturePath = './test/fixtures/turbo-carto-named-maps-blues.png';
-                var image = mapnik.Image.fromBytes(new Buffer(res.body, 'binary'));
+                    assert.equal(res.statusCode, 200);
+                    assert.equal(res.headers['content-type'], 'image/png');
 
-                assert.imageIsSimilarToFile(image, fixturePath, IMAGE_TOLERANCE_PER_MIL, next);
-            },
-            function deleteTemplate(err) {
-                assert.ifError(err);
+                    var fixturePath = './test/fixtures/turbo-carto-named-maps-blues.png';
+                    var image = mapnik.Image.fromBytes(new Buffer(res.body, 'binary'));
 
-                var next = this;
+                    assert.imageIsSimilarToFile(image, fixturePath, IMAGE_TOLERANCE_PER_MIL, next);
+                },
+                function deleteTemplate(err) {
+                    assert.ifError(err);
 
-                assert.response(server, {
-                    url: '/api/v1/map/named/' + templateId + '?api_key=1234',
-                    method: 'DELETE',
-                    headers: { host: 'localhost' }
-                }, {}, function (res, err) {
-                    next(err, res);
-                });
-            },
-            function checkDeleteTemplate(err, res) {
-                assert.ifError(err);
-                assert.equal(res.statusCode, 204);
-                assert.ok(!res.body);
+                    var next = this;
 
-                return null;
-            },
-            function finish(err) {
-                done(err);
-            }
-        );
+                    assert.response(server, {
+                        url: '/api/v1/map/named/' + templateId + '?api_key=1234',
+                        method: 'DELETE',
+                        headers: { host: 'localhost' }
+                    }, {}, function (res, err) {
+                        next(err, res);
+                    });
+                },
+                function checkDeleteTemplate(err, res) {
+                    assert.ifError(err);
+                    assert.equal(res.statusCode, 204);
+                    assert.ok(!res.body);
+
+                    return null;
+                },
+                function finish(err) {
+                    done(err);
+                }
+            );
+        });
     });
 });

test/acceptance/turbo-cartocss/regressions.js

@@ -24,6 +24,8 @@ describe('turbo-carto regressions', function() {
     afterEach(function (done) {
         if (this.testClient) {
             this.testClient.drain(done);
+        } else {
+            done();
         }
     });
 
@@ -64,6 +66,57 @@ describe('turbo-carto regressions', function() {
         });
     });
 
+    it('should fail for private tables', function(done) {
+        var cartocss = [
+            "#private_table {",
+            "  marker-placement: point;",
+            "  marker-allow-overlap: true;",
+            "  marker-line-width: 0;",
+            "  marker-fill-opacity: 1.0;",
+            "  marker-width: ramp([cartodb_id], 10, 20);",
+            "  marker-fill: red;",
+            "}"
+        ].join('\n');
+
+        this.testClient = new TestClient(makeMapconfig('SELECT * FROM test_table_private_1', cartocss));
+        this.testClient.getLayergroup(TestClient.RESPONSE.ERROR, function(err, layergroup) {
+            assert.ok(!err, err);
+
+            assert.ok(!layergroup.hasOwnProperty('layergroupid'));
+            assert.ok(layergroup.hasOwnProperty('errors'));
+
+            var turboCartoError = layergroup.errors_with_context[0];
+            assert.ok(turboCartoError);
+            assert.equal(turboCartoError.type, 'turbo-carto');
+            assert.ok(turboCartoError.message.match(/permission\sdenied\sfor\srelation\stest_table_private_1/));
+
+            done();
+        });
+    });
+
+    it('should work for private tables with api key', function(done) {
+        var cartocss = [
+            "#private_table {",
+            "  marker-placement: point;",
+            "  marker-allow-overlap: true;",
+            "  marker-line-width: 0;",
+            "  marker-fill-opacity: 1.0;",
+            "  marker-width: ramp([cartodb_id], 10, 20);",
+            "  marker-fill: red;",
+            "}"
+        ].join('\n');
+
+        this.testClient = new TestClient(makeMapconfig('SELECT * FROM test_table_private_1', cartocss), 1234);
+        this.testClient.getLayergroup(function(err, layergroup) {
+            assert.ok(!err, err);
+
+            assert.ok(layergroup.hasOwnProperty('layergroupid'));
+            assert.ok(!layergroup.hasOwnProperty('errors'));
+
+            done();
+        });
+    });
+
     it('should work with mapnik substitution tokens', function(done) {
         var cartocss = [
             "#layer {",
@@ -167,4 +220,83 @@ describe('turbo-carto regressions', function() {
             done();
         });
     });
+
+    describe('empty datasource results', function() {
+
+        afterEach(function (done) {
+            if (this.testClient) {
+                this.testClient.drain(done);
+            } else {
+                done();
+            }
+        });
+
+        function emptyResultMapConfig(markerFillRule) {
+            var cartocss = [
+                "#county_points_with_population {",
+                "  marker-placement: point;",
+                "  marker-allow-overlap: true;",
+                "  marker-fill-opacity: 1.0;",
+                "  marker-fill: " + markerFillRule + ';',
+                "  marker-line-width: 0;",
+                "}"
+            ].join('\n');
+
+            return {
+                "version": "1.5.0",
+                "layers": [
+                    {
+                        "type": 'mapnik',
+                        "options": {
+                            "cartocss_version": '2.3.0',
+                            "source": {
+                                "id": "head"
+                            },
+                            "cartocss": cartocss
+                        }
+                    }
+                ],
+                "analyses": [
+                    {
+                        "id": "head",
+                        "type": "source",
+                        "params": {
+                            "query": "SELECT * FROM populated_places_simple_reduced limit 0"
+                        }
+                    }
+                ]
+            };
+        }
+
+        it('should work for numeric ramps', function(done) {
+
+            var makerFillRule = 'ramp([pop_max], (#E5F5F9,#99D8C9,#2CA25F), jenks)';
+
+            this.testClient = new TestClient(emptyResultMapConfig(makerFillRule), 1234);
+            this.testClient.getLayergroup(function(err, layergroup) {
+                assert.ok(!err, err);
+
+                assert.ok(layergroup.hasOwnProperty('layergroupid'));
+                assert.ok(!layergroup.hasOwnProperty('errors'));
+
+                done();
+            });
+        });
+
+        it('should work for category ramps', function(done) {
+
+            var makerFillRule = 'ramp([adm0name], (#E5F5F9,#99D8C9,#2CA25F), category)';
+
+            this.testClient = new TestClient(emptyResultMapConfig(makerFillRule), 1234);
+            this.testClient.getLayergroup(function(err, layergroup) {
+                assert.ok(!err, err);
+
+                assert.ok(layergroup.hasOwnProperty('layergroupid'));
+                assert.ok(!layergroup.hasOwnProperty('errors'));
+
+                done();
+            });
+        });
+    });
+
 });

test/acceptance/widgets/ported/aggregation.js

@@ -0,0 +1,329 @@
+require('../../../support/test_helper');
+
+var assert = require('../../../support/assert');
+var TestClient = require('../../../support/test-client');
+
+describe('widgets', function() {
+
+    describe('aggregations', function() {
+
+        afterEach(function(done) {
+            if (this.testClient) {
+                this.testClient.drain(done);
+            } else {
+                done();
+            }
+        });
+
+        var aggregationMapConfig = {
+            version: '1.5.0',
+            layers: [
+                {
+                    type: 'mapnik',
+                    options: {
+                        sql: 'select * from populated_places_simple_reduced',
+                        cartocss: '#layer0 { marker-fill: red; marker-width: 10; }',
+                        cartocss_version: '2.0.1',
+                        widgets: {
+                            adm0name: {
+                                type: 'aggregation',
+                                options: {
+                                    column: 'adm0name',
+                                    aggregation: 'count'
+                                }
+                            }
+                        }
+                    }
+                }
+            ]
+        };
+
+        it('can be fetched from a valid aggregation', function(done) {
+            this.testClient = new TestClient(aggregationMapConfig);
+            this.testClient.getWidget('adm0name', { own_filter: 0 }, function (err, res, aggregation) {
+                assert.ok(!err, err);
+                assert.ok(aggregation);
+                assert.equal(aggregation.type, 'aggregation');
+
+                assert.equal(aggregation.categories.length, 6);
+
+                assert.deepEqual(
+                    aggregation.categories[0],
+                    { category: 'United States of America', value: 769, agg: false }
+                );
+
+                assert.deepEqual(
+                    aggregation.categories[aggregation.categories.length - 1],
+                    { category: 'Other', value: 4914, agg: true }
+                );
+
+                done();
+            });
+        });
+
+        var filteredCategoriesScenarios = [
+            { accept: ['Canada'], values: [256] },
+            { accept: ['Canada', 'Spain', 'Chile', 'Thailand'], values: [256, 49, 83, 79] },
+            { accept: ['Canada', 'Spain', 'Chile', 'Thailand', 'Japan'], values: [256, 49, 83, 79, 69] },
+            { accept: ['Canada', 'Spain', 'Chile', 'Thailand', 'Japan', 'France'], values: [256, 49, 83, 79, 69, 71] },
+            {
+                accept: ['United States of America', 'Canada', 'Spain', 'Chile', 'Thailand', 'Japan', 'France'],
+                values: [769, 256, 49, 83, 79, 69, 71]
+            }
+        ];
+
+        filteredCategoriesScenarios.forEach(function(scenario) {
+            it('can filter some categories: ' + scenario.accept.join(', '), function(done) {
+                this.testClient = new TestClient(aggregationMapConfig);
+                var adm0nameFilter = {
+                    adm0name: {
+                        accept: scenario.accept
+                    }
+                };
+                var params = {
+                    own_filter: 1,
+                    filters: {
+                        layers: [
+                            adm0nameFilter
+                        ]
+                    }
+                };
+                this.testClient.getWidget('adm0name', params, function (err, res, aggregation) {
+                    assert.ok(!err, err);
+                    assert.ok(aggregation);
+                    assert.equal(aggregation.type, 'aggregation');
+
+                    assert.equal(aggregation.categories.length, scenario.accept.length);
+
+                    var categoriesByCategory = aggregation.categories.reduce(function(byCategory, row) {
+                        byCategory[row.category] = row;
+                        return byCategory;
+                    }, {});
+
+                    var scenarioByCategory = scenario.accept.reduce(function(byCategory, category, index) {
+                        byCategory[category] = { category: category, value: scenario.values[index], agg: false };
+                        return byCategory;
+                    }, {});
+
+                    Object.keys(categoriesByCategory).forEach(function(category) {
+                        assert.deepEqual(categoriesByCategory[category], scenarioByCategory[category]);
+                    });
+
+                    done();
+                });
+            });
+        });
+
+        var aggregationSumMapConfig = {
+            version: '1.5.0',
+            layers: [
+                {
+                    type: 'mapnik',
+                    options: {
+                        sql: 'select * from populated_places_simple_reduced',
+                        cartocss: '#layer0 { marker-fill: red; marker-width: 10; }',
+                        cartocss_version: '2.0.1',
+                        widgets: {
+                            adm0name: {
+                                type: 'aggregation',
+                                options: {
+                                    column: 'adm0name',
+                                    aggregation: 'sum',
+                                    aggregationColumn: 'pop_max'
+                                }
+                            }
+                        }
+                    }
+                }
+            ]
+        };
+
+        it('can sum other column for aggregation value', function(done) {
+
+            this.testClient = new TestClient(aggregationSumMapConfig);
+            this.testClient.getWidget('adm0name', { own_filter: 0 }, function (err, res, aggregation) {
+                assert.ok(!err, err);
+                assert.ok(aggregation);
+                assert.equal(aggregation.type, 'aggregation');
+
+                assert.equal(aggregation.categories.length, 6);
+
+                assert.deepEqual(
+                    aggregation.categories[0],
+                    { category: 'China', value: 374537585, agg: false }
+                );
+
+                assert.deepEqual(
+                    aggregation.categories[aggregation.categories.length - 1],
+                    { category: 'Other', value: 1412626289, agg: true }
+                );
+
+                done();
+            });
+        });
+
+        var filteredCategoriesSumScenarios = [
+            { accept: [], values: [] },
+            { accept: ['Canada'], values: [23955084] },
+            { accept: ['Canada', 'Spain', 'Chile', 'Thailand'], values: [23955084, 22902774, 14356263, 17492483] },
+            {
+                accept: ['United States of America', 'Canada', 'Spain', 'Chile', 'Thailand', 'Japan', 'France'],
+                values: [239098994, 23955084, 22902774, 14356263, 17492483, 93577001, 25473876]
+            }
+        ];
+
+        filteredCategoriesSumScenarios.forEach(function(scenario) {
+            it('can filter some categories with sum aggregation: ' + scenario.accept.join(', '), function(done) {
+                this.testClient = new TestClient(aggregationSumMapConfig);
+                var adm0nameFilter = {
+                    adm0name: {
+                        accept: scenario.accept
+                    }
+                };
+                var params = {
+                    own_filter: 1,
+                    filters: {
+                        layers: [
+                            adm0nameFilter
+                        ]
+                    }
+                };
+                this.testClient.getWidget('adm0name', params, function (err, res, aggregation) {
+                    assert.ok(!err, err);
+                    assert.ok(aggregation);
+                    assert.equal(aggregation.type, 'aggregation');
+
+                    assert.equal(aggregation.categories.length, scenario.accept.length);
+
+                    var categoriesByCategory = aggregation.categories.reduce(function(byCategory, row) {
+                        byCategory[row.category] = row;
+                        return byCategory;
+                    }, {});
+
+                    var scenarioByCategory = scenario.accept.reduce(function(byCategory, category, index) {
+                        byCategory[category] = { category: category, value: scenario.values[index], agg: false };
+                        return byCategory;
+                    }, {});
+
+                    Object.keys(categoriesByCategory).forEach(function(category) {
+                        assert.deepEqual(categoriesByCategory[category], scenarioByCategory[category]);
+                    });
+
+                    done();
+                });
+            });
+        });
+
+        var numericAggregationMapConfig = {
+            version: '1.5.0',
+            layers: [
+                {
+                    type: 'mapnik',
+                    options: {
+                        sql: 'select * from populated_places_simple_reduced',
+                        cartocss: '#layer0 { marker-fill: red; marker-width: 10; }',
+                        cartocss_version: '2.3.0',
+                        widgets: {
+                            scalerank: {
+                                type: 'aggregation',
+                                options: {
+                                    column: 'scalerank',
+                                    aggregation: 'count'
+                                }
+                            }
+                        }
+                    }
+                }
+            ]
+        };
+
+        ['1', 1].forEach(function(filterValue) {
+            it('can filter numeric categories: ' + (typeof filterValue), function(done) {
+                this.testClient = new TestClient(numericAggregationMapConfig);
+                var scalerankFilter = {
+                    scalerank: {
+                        accept: [filterValue]
+                    }
+                };
+                var params = {
+                    own_filter: 1,
+                    filters: {
+                        layers: [scalerankFilter]
+                    }
+                };
+                this.testClient.getWidget('scalerank', params, function (err, res, aggregation) {
+                    assert.ok(!err, err);
+                    assert.ok(aggregation);
+                    assert.equal(aggregation.type, 'aggregation');
+
+                    assert.equal(aggregation.categories.length, 1);
+                    assert.deepEqual(aggregation.categories[0], { category: '1', value: 179, agg: false });
+
+                    done();
+                });
+            });
+        });
+
+        describe('search', function() {
+            afterEach(function(done) {
+                if (this.testClient) {
+                    this.testClient.drain(done);
+                } else {
+                    done();
+                }
+            });
+
+            ['1', 1].forEach(function(userQuery) {
+                it('can search numeric categories: ' + (typeof userQuery), function(done) {
+                    this.testClient = new TestClient(numericAggregationMapConfig);
+                    var scalerankFilter = {
+                        scalerank: {
+                            accept: [userQuery]
+                        }
+                    };
+                    var params = {
+                        own_filter: 0,
+                        filters: {
+                            layers: [scalerankFilter]
+                        }
+                    };
+                    this.testClient.widgetSearch('scalerank', userQuery, params, function (err, res, searchResult) {
+                        assert.ok(!err, err);
+                        assert.ok(searchResult);
+                        assert.equal(searchResult.type, 'aggregation');
+
+                        assert.equal(searchResult.categories.length, 2);
+                        assert.deepEqual(
+                            searchResult.categories,
+                            [{ category: 10, value: 515 }, { category: 1, value: 179 }]
+                        );
+
+                        done();
+                    });
+                });
+            });
+
+            var adm0name = 'Argentina';
+            [adm0name, adm0name.toLowerCase(), adm0name.toUpperCase()].forEach(function(userQuery) {
+                it('should search with case insensitive: ' + userQuery, function(done) {
+                    this.testClient = new TestClient(aggregationMapConfig);
+                    this.testClient.widgetSearch('adm0name', userQuery, function (err, res, searchResult) {
+                        assert.ok(!err, err);
+                        assert.ok(searchResult);
+                        assert.equal(searchResult.type, 'aggregation');
+
+                        assert.equal(searchResult.categories.length, 1);
+                        assert.deepEqual(
+                            searchResult.categories,
+                            [{ category:"Argentina", value:159 }]
+                        );
+
+                        done();
+                    });
+                });
+            });
+        });
+
+    });
+
+});

test/acceptance/widgets/ported/formula.js

@@ -0,0 +1,89 @@
+require('../../../support/test_helper');
+
+var assert        = require('../../../support/assert');
+var TestClient = require('../../../support/test-client');
+
+describe('widgets', function() {
+
+    describe('formula', function() {
+
+        afterEach(function(done) {
+            if (this.testClient) {
+                this.testClient.drain(done);
+            } else {
+                done();
+            }
+        });
+
+        function widgetsMapConfig(widgets) {
+            return {
+                version: '1.5.0',
+                layers: [
+                    {
+                        type: 'mapnik',
+                        options: {
+                            sql: 'select * from populated_places_simple_reduced where pop_max > 0 and pop_max < 600000',
+                            cartocss: '#layer0 { marker-fill: red; marker-width: 10; }',
+                            cartocss_version: '2.0.1',
+                            widgets: widgets
+                        }
+                    }
+                ]
+            };
+        }
+
+        var operations = {
+            min: [10, 0],
+            max: [599579, 0],
+            count: [5822, 0],
+            avg: [112246.00893163861, 0],
+            sum: [653496264, 0]
+        };
+
+        Object.keys(operations).forEach(function(operation) {
+            it('should do ' + operation + ' over column', function(done) {
+                var widgets = {
+                    pop_max_f: {
+                        type: 'formula',
+                        options: {
+                            column: 'pop_max',
+                            operation: operation
+                        }
+                    }
+                };
+                this.testClient = new TestClient(widgetsMapConfig(widgets));
+                this.testClient.getWidget('pop_max_f', function (err, res, result) {
+                    assert.ok(!err, err);
+                    assert.equal(result.operation, operation);
+                    assert.equal(result.result, operations[operation][0]);
+                    assert.equal(result.nulls, operations[operation][1]);
+
+                    done();
+                });
+            });
+        });
+
+        it('does not require column for count formula', function(done) {
+           var operation = 'count';
+            var widgets = {
+                pop_max_count_f: {
+                    type: 'formula',
+                    options: {
+                        operation: operation
+                    }
+                }
+            };
+            this.testClient = new TestClient(widgetsMapConfig(widgets));
+            this.testClient.getWidget('pop_max_count_f', function (err, res, result) {
+                assert.ok(!err, err);
+                assert.equal(result.operation, operation);
+                assert.equal(result.result, operations[operation][0]);
+                assert.equal(result.nulls, operations[operation][1]);
+
+                done();
+            });
+        });
+
+    });
+
+});

test/acceptance/widgets/ported/histogram.js

@@ -0,0 +1,373 @@
+require('../../../support/test_helper');
+
+var assert        = require('../../../support/assert');
+var TestClient = require('../../../support/test-client');
+
+describe('widgets', function() {
+
+    describe('histograms', function() {
+
+        afterEach(function(done) {
+            if (this.testClient) {
+                this.testClient.drain(done);
+            } else {
+                done();
+            }
+        });
+
+        function histogramsMapConfig(widgets) {
+            return {
+                version: '1.5.0',
+                layers: [
+                    {
+                        type: 'mapnik',
+                        options: {
+                            sql: 'select * from populated_places_simple_reduced',
+                            cartocss: '#layer0 { marker-fill: red; marker-width: 10; }',
+                            cartocss_version: '2.0.1',
+                            widgets: widgets || {
+                                scalerank: {
+                                    type: 'histogram',
+                                    options: {
+                                        column: 'scalerank'
+                                    }
+                                },
+                                pop_max: {
+                                    type: 'histogram',
+                                    options: {
+                                        column: 'pop_max'
+                                    }
+                                }
+                            }
+                        }
+                    }
+                ]
+            };
+        }
+
+        it('can be fetched from a valid histogram', function(done) {
+            this.testClient = new TestClient(histogramsMapConfig());
+            this.testClient.getWidget('scalerank', function (err, res, histogram) {
+                assert.ok(!err, err);
+                assert.ok(histogram);
+                assert.equal(histogram.type, 'histogram');
+                validateHistogramBins(histogram);
+
+                assert.ok(histogram.bins.length);
+
+                assert.deepEqual(histogram.bins[0], { bin: 0, freq: 179, min: 1, max: 1, avg: 1 });
+
+                done();
+            });
+        });
+
+        it('can be fetched from a valid histogram', function(done) {
+            this.testClient = new TestClient(histogramsMapConfig());
+            this.testClient.getWidget('pop_max', function (err, res, histogram) {
+                assert.ok(!err, err);
+                assert.ok(histogram);
+                assert.equal(histogram.type, 'histogram');
+                validateHistogramBins(histogram);
+
+                assert.ok(histogram.bins.length);
+
+                assert.deepEqual(
+                    histogram.bins[histogram.bins.length - 1],
+                    { bin: 47, freq: 1, min: 35676000, max: 35676000, avg: 35676000 }
+                );
+
+                done();
+            });
+        });
+
+        it('can be fetched from a valid filtered histogram', function(done) {
+            this.testClient = new TestClient(histogramsMapConfig());
+            var popMaxFilter = {
+                pop_max: {
+                    min: 1e5,
+                    max: 1e7
+                }
+            };
+            var params = {
+                own_filter: 1,
+                filters: {
+                    layers: [popMaxFilter]
+                }
+            };
+            this.testClient.getWidget('pop_max', params, function (err, res, histogram) {
+                assert.ok(!err, err);
+                assert.ok(histogram);
+                assert.equal(histogram.type, 'histogram');
+                validateHistogramBins(histogram);
+
+                assert.ok(histogram.bins.length);
+
+                assert.deepEqual(
+                    histogram.bins[histogram.bins.length - 1],
+                    { bin: 7, min: 8829000, max: 9904000, avg: 9340914.714285715, freq: 7 }
+                );
+
+                done();
+            });
+        });
+
+        it('returns array with freq=0 entries for empty bins', function(done) {
+            var histogram20binsMapConfig = {
+                version: '1.5.0',
+                layers: [
+                    {
+                        type: 'mapnik',
+                        options: {
+                            sql: 'select * from populated_places_simple_reduced',
+                            cartocss: '#layer0 { marker-fill: red; marker-width: 10; }',
+                            cartocss_version: '2.0.1',
+                            widgets: {
+                                pop_max: {
+                                    type: 'histogram',
+                                    options: {
+                                        column: 'pop_max'
+                                    }
+                                }
+                            }
+                        }
+                    }
+                ]
+            };
+
+            this.testClient = new TestClient(histogram20binsMapConfig);
+            this.testClient.getWidget('pop_max', { start: 0, end: 35676000, bins: 20 }, function (err, res, histogram) {
+                assert.ok(!err, err);
+                assert.equal(histogram.type, 'histogram');
+                validateHistogramBins(histogram);
+                assert.ok(histogram.bins.length);
+                assert.deepEqual(
+                    histogram.bins[histogram.bins.length - 1],
+                    { bin: 19, freq: 1, min: 35676000, max: 35676000, avg: 35676000 }
+                );
+
+                var emptyBin = histogram.bins[18];
+                assert.ok(!emptyBin);
+
+                done();
+            });
+        });
+
+        it('can use a fixed number of bins', function(done) {
+            var fixedBinsHistogramMapConfig = histogramsMapConfig({
+                pop_max: {
+                    type: 'histogram',
+                    options: {
+                        column: 'pop_max'
+                    }
+                }
+            });
+
+            this.testClient = new TestClient(fixedBinsHistogramMapConfig);
+            this.testClient.getWidget('pop_max', { bins: 5 }, function (err, res, histogram) {
+                assert.ok(!err, err);
+                assert.equal(histogram.type, 'histogram');
+
+                assert.equal(histogram.bins_count, 5);
+
+                validateHistogramBins(histogram);
+
+                assert.ok(histogram.bins.length);
+                assert.deepEqual(
+                    histogram.bins[0],
+                    { bin: 0, min: 0, max: 7067423, avg: 280820.0057731959, freq: 7275 }
+                );
+                assert.deepEqual(
+                    histogram.bins[histogram.bins.length - 1],
+                    { bin: 4, freq: 1, min: 35676000, max: 35676000, avg: 35676000 }
+                );
+
+                done();
+            });
+        });
+
+        function validateHistogramBins(histogram) {
+            var binWidth = histogram.bin_width;
+            var start = histogram.bins_start;
+            var end = start + (histogram.bins_count * binWidth);
+
+            var firstBin = histogram.bins[0];
+            assert.equal(firstBin.min, start,
+                'First bin does not match min and start ' + JSON.stringify({
+                    min: firstBin.min,
+                    start: start
+                })
+            );
+
+            var lastBin = histogram.bins[histogram.bins.length - 1];
+            assert.equal(lastBin.max, end,
+                'Last bin does not match max and end ' + JSON.stringify({
+                    max: lastBin.max,
+                    end: end
+                })
+            );
+
+            function getBinStartEnd(binIndex) {
+                return {
+                    start: start + (binIndex * binWidth),
+                    end: start + ((binIndex + 1) * binWidth)
+                };
+            }
+
+            histogram.bins.forEach(function(bin) {
+                var binStartEnd = getBinStartEnd(bin.bin);
+
+                assert.ok(binStartEnd.start <= bin.min,
+                    'Bin start bigger than bin min ' + JSON.stringify({
+                        bin: bin.bin,
+                        min: bin.min,
+                        start: binStartEnd.start
+                    })
+                );
+
+                assert.ok(binStartEnd.end >= bin.max,
+                    'Bin end smaller than bin max ' + JSON.stringify({
+                        bin: bin.bin,
+                        max: bin.max,
+                        end: binStartEnd.end
+                    })
+                );
+
+                assert.ok(bin.avg >= bin.min && bin.avg <= bin.max,
+                        'Bin avg not between min and max values' + JSON.stringify({
+                        bin: bin.bin,
+                        avg: bin.avg,
+                        min: bin.min,
+                        max: bin.max
+                    })
+                );
+            });
+        }
+
+        describe('datetime column', function() {
+            afterEach(function(done) {
+                if (this.testClient) {
+                    this.testClient.drain(done);
+                } else {
+                    done();
+                }
+            });
+
+            var updatedAtFilter = {
+                updated_at: {
+                    min: 0
+                }
+            };
+
+            it('can use a datetime column', function(done) {
+                this.testClient = new TestClient(histogramsMapConfig({
+                    updated_at: {
+                        type: 'histogram',
+                        options: {
+                            column: 'updated_at'
+                        }
+                    }
+                }));
+                this.testClient.getWidget('updated_at', function (err, res, histogram) {
+                    assert.ok(!err, err);
+                    assert.ok(histogram);
+                    assert.equal(histogram.type, 'histogram');
+
+                    assert.ok(histogram.bins.length);
+
+                    done();
+                });
+            });
+
+            it('can use a datetime filtered column', function(done) {
+                this.testClient = new TestClient(histogramsMapConfig({
+                    updated_at: {
+                        type: 'histogram',
+                        options: {
+                            column: 'updated_at'
+                        }
+                    }
+                }));
+                var params = {
+                    own_filter: 1,
+                    filters: {
+                        layers: [updatedAtFilter]
+                    }
+                };
+                this.testClient.getWidget('updated_at', params, function (err, res, histogram) {
+                    assert.ok(!err, err);
+                    assert.ok(histogram);
+                    assert.equal(histogram.type, 'histogram');
+
+                    assert.ok(histogram.bins.length);
+
+                    done();
+                });
+            });
+
+            it('can getTile with datetime filtered column', function(done) {
+                this.testClient = new TestClient(histogramsMapConfig({
+                    updated_at: {
+                        type: 'histogram',
+                        options: {
+                            column: 'updated_at'
+                        }
+                    }
+                }));
+                var params = {
+                    own_filter: 1,
+                    filters: {
+                        layers: [updatedAtFilter]
+                    }
+                };
+                this.testClient.getTile(0, 0, 0, params, function (err, res, tile) {
+                    assert.ok(!err, err);
+                    assert.ok(tile);
+
+                    done();
+                });
+            });
+
+            it('can use two columns with different types', function(done) {
+                this.testClient = new TestClient(histogramsMapConfig({
+                    updated_at: {
+                        type: 'histogram',
+                        options: {
+                            column: 'updated_at'
+                        }
+                    },
+                    pop_max: {
+                        type: 'histogram',
+                        options: {
+                            column: 'pop_max'
+                        }
+                    }
+                }));
+
+                var popMaxFilter = {
+                    pop_max: {
+                        max: 1e7
+                    }
+                };
+
+                var params = {
+                    own_filter: 1,
+                    filters: {
+                        layers: [popMaxFilter]
+                    }
+                };
+
+                this.testClient.getWidget('updated_at', params, function (err, res, histogram) {
+                    assert.ok(!err, err);
+                    assert.ok(histogram);
+                    assert.equal(histogram.type, 'histogram');
+
+                    assert.ok(histogram.bins.length);
+
+                    done();
+                });
+            });
+        });
+
+    });
+
+});

test/acceptance/widgets/ported/list.js

@@ -0,0 +1,106 @@
+require('../../../support/test_helper');
+
+var assert        = require('../../../support/assert');
+var TestClient = require('../../../support/test-client');
+var _ = require('underscore');
+
+describe('widgets', function() {
+
+    describe('lists', function() {
+
+        afterEach(function(done) {
+            if (this.testClient) {
+                this.testClient.drain(done);
+            } else {
+                done();
+            }
+        });
+
+        function listsMapConfig(columns) {
+            return {
+                version: '1.5.0',
+                layers: [
+                    {
+                        type: 'mapnik',
+                        options: {
+                            sql: 'select * from test_table',
+                            cartocss: '#layer0 { marker-fill: red; marker-width: 10; }',
+                            cartocss_version: '2.0.1',
+                            widgets: {
+                                places: {
+                                    type: 'list',
+                                    options: {
+                                        columns: columns || ['name', 'address']
+                                    }
+                                }
+                            }
+                        }
+                    }
+                ]
+            };
+        }
+
+        var EXPECTED_NAMES = ['Hawai', 'El Estocolmo', 'El Rey del Tallarín', 'El Lacón', 'El Pico'];
+
+        it('can be fetched from a valid list', function(done) {
+            var columns = ['name', 'address'];
+            this.testClient = new TestClient(listsMapConfig(columns));
+            this.testClient.getWidget('places', function (err, res, list) {
+                assert.ok(!err, err);
+                assert.ok(list);
+                assert.equal(list.type, 'list');
+                assert.equal(list.rows.length, 5);
+
+                assert.ok(onlyHasFields(list, columns));
+
+                var names = list.rows.map(function (item) {
+                    return item.name;
+                });
+                assert.deepEqual(names, EXPECTED_NAMES);
+
+                var expectedAddresses = [
+                    'Calle de Pérez Galdós 9, Madrid, Spain',
+                    'Calle de la Palma 72, Madrid, Spain',
+                    'Plaza Conde de Toreno 2, Madrid, Spain',
+                    'Manuel Fernández y González 8, Madrid, Spain',
+                    'Calle Divino Pastor 12, Madrid, Spain'
+                ];
+                var addresses = list.rows.map(function (item) {
+                    return item.address;
+                });
+                assert.deepEqual(addresses, expectedAddresses);
+
+                done();
+            });
+        });
+
+        it('should fetch just one column', function(done) {
+            var columns = ['name'];
+            this.testClient = new TestClient(listsMapConfig(columns));
+            this.testClient.getWidget('places', function (err, res, list) {
+                assert.ok(!err, err);
+                assert.ok(list);
+                assert.equal(list.type, 'list');
+                assert.equal(list.rows.length, 5);
+
+                assert.ok(onlyHasFields(list, columns));
+
+                var names = list.rows.map(function (item) {
+                    return item.name;
+                });
+                assert.deepEqual(names, EXPECTED_NAMES);
+
+                done();
+            });
+        });
+
+        function onlyHasFields(list, expectedFields) {
+            var fields = (!!list.rows[0]) ? Object.keys(list.rows[0]) : [];
+
+            return _.difference(fields, expectedFields).length === 0 &&
+                _.difference(expectedFields, fields).length === 0;
+        }
+
+    });
+
+});

test/acceptance/widgets/regressions.js

@@ -0,0 +1,223 @@
+require('../../support/test_helper');
+
+var assert = require('../../support/assert');
+var TestClient = require('../../support/test-client');
+
+describe('widgets-regressions', function() {
+
+    describe('aggregations', function() {
+
+        afterEach(function(done) {
+            if (this.testClient) {
+                this.testClient.drain(done);
+            } else {
+                done();
+            }
+        });
+
+        it('should work when there is a mix of layers with and without widgets', function(done) {
+            var layersWithNoWidgetsMapConfig = {
+                version: '1.5.0',
+                layers: [
+                    {
+                        type: 'mapnik',
+                        options: {
+                            sql: 'select * from populated_places_simple_reduced',
+                            cartocss: '#layer0 { marker-fill: red; marker-width: 10; }',
+                            cartocss_version: '2.0.1',
+                            widgets: {
+                                adm0name: {
+                                    type: 'aggregation',
+                                    options: {
+                                        column: 'adm0name',
+                                        aggregation: 'sum',
+                                        aggregationColumn: 'pop_max'
+                                    }
+                                }
+                            }
+                        }
+                    },
+                    {
+                        type: 'mapnik',
+                        options: {
+                            sql: 'select * from populated_places_simple_reduced limit 100',
+                            cartocss: '#layer0 { marker-fill: red; marker-width: 10; }',
+                            cartocss_version: '2.0.1'
+                        }
+                    }
+                ]
+            };
+
+            this.testClient = new TestClient(layersWithNoWidgetsMapConfig);
+            this.testClient.getWidget('adm0name', { own_filter: 0 }, function (err, res, aggregation) {
+                assert.ok(!err, err);
+                assert.ok(aggregation);
+                assert.equal(aggregation.type, 'aggregation');
+
+                assert.equal(aggregation.categories.length, 6);
+
+                assert.deepEqual(
+                    aggregation.categories[0],
+                    { category: 'China', value: 374537585, agg: false }
+                );
+
+                assert.deepEqual(
+                    aggregation.categories[aggregation.categories.length - 1],
+                    { category: 'Other', value: 1412626289, agg: true }
+                );
+
+                done();
+            });
+        });
+
+        it('should work when there is a mix of layers with and without widgets, source and sql', function(done) {
+            var mixOfLayersMapConfig = {
+                version: '1.5.0',
+                layers: [
+                    {
+                        type: 'mapnik',
+                        options: {
+                            sql: 'select * from populated_places_simple_reduced',
+                            cartocss: '#layer0 { marker-fill: red; marker-width: 10; }',
+                            cartocss_version: '2.0.1',
+                            widgets: {
+                                adm0name_categories: {
+                                    type: 'aggregation',
+                                    options: {
+                                        column: 'adm0name',
+                                        aggregation: 'sum',
+                                        aggregationColumn: 'pop_max'
+                                    }
+                                },
+                                adm1name_categories: {
+                                    type: 'aggregation',
+                                    options: {
+                                        column: 'adm1name',
+                                        aggregation: 'sum',
+                                        aggregationColumn: 'pop_max'
+                                    }
+                                }
+                            }
+                        }
+                    },
+                    {
+                        type: 'mapnik',
+                        options: {
+                            source: {id: 'head-limited'},
+                            cartocss: '#layer0 { marker-fill: red; marker-width: 10; }',
+                            cartocss_version: '2.0.1',
+                            widgets: {
+                                pop_max_histogram: {
+                                    type: 'histogram',
+                                    options: {
+                                        column: 'pop_max'
+                                    }
+                                }
+                            }
+                        }
+                    },
+                    {
+                        "type": "http",
+                        "options": {
+                            "urlTemplate": "http://{s}.basemaps.cartocdn.com/light_all/{z}/{x}/{y}.png",
+                            "subdomains": "abcd"
+                        }
+                    }
+                ],
+                analyses: [
+                    {
+                        id: 'head-limited',
+                        type: 'source',
+                        params: {
+                            query: 'select * from populated_places_simple_reduced limit 100'
+                        }
+                    }
+                ],
+                dataviews: {
+                    wadus: {
+                        type: 'histogram',
+                        source: {
+                            id: 'head-limited'
+                        },
+                        options: {
+                            column: 'population'
+                        }
+                    }
+                }
+            };
+
+            this.testClient = new TestClient(mixOfLayersMapConfig);
+            this.testClient.getLayergroup(function(err, layergroup) {
+                assert.ok(!err, err);
+                assert.ok(layergroup.metadata);
+                var metadata = layergroup.metadata;
+                assert.equal(metadata.layers.length, 3);
+                assert.equal(metadata.analyses.length, 2);
+                assert.equal(Object.keys(metadata.dataviews).length, 4);
+                assert.deepEqual(
+                    Object.keys(metadata.dataviews),
+                    ['wadus', 'adm0name_categories', 'adm1name_categories', 'pop_max_histogram']
+                );
+
+
+                done();
+            });
+        });
+
+        it('should work with layers not containing sql', function(done) {
+            var nonSqlLayersMapConfig = {
+                version: '1.5.0',
+                layers: [
+                    {
+                        type: 'mapnik',
+                        options: {
+                            sql: 'select * from populated_places_simple_reduced',
+                            cartocss: '#layer0 { marker-fill: red; marker-width: 10; }',
+                            cartocss_version: '2.0.1',
+                            widgets: {
+                                adm0name: {
+                                    type: 'aggregation',
+                                    options: {
+                                        column: 'adm0name',
+                                        aggregation: 'sum',
+                                        aggregationColumn: 'pop_max'
+                                    }
+                                }
+                            }
+                        }
+                    },
+                    {
+                        "type": "http",
+                        "options": {
+                            "urlTemplate": "http://{s}.basemaps.cartocdn.com/light_all/{z}/{x}/{y}.png",
+                            "subdomains": "abcd"
+                        }
+                    }
+                ]
+            };
+
+            this.testClient = new TestClient(nonSqlLayersMapConfig);
+            this.testClient.getWidget('adm0name', { own_filter: 0 }, function (err, res, aggregation) {
+                assert.ok(!err, err);
+                assert.ok(aggregation);
+                assert.equal(aggregation.type, 'aggregation');
+
+                assert.equal(aggregation.categories.length, 6);
+
+                assert.deepEqual(
+                    aggregation.categories[0],
+                    { category: 'China', value: 374537585, agg: false }
+                );
+
+                assert.deepEqual(
+                    aggregation.categories[aggregation.categories.length - 1],
+                    { category: 'Other', value: 1412626289, agg: true }
+                );
+
+                done();
+            });
+        });
+
+    });
+
+});

test/support/prepare_db.sh

@@ -12,6 +12,7 @@
 
 PREPARE_REDIS=yes
 PREPARE_PGSQL=yes
+DOWNLOAD_SQL_FILES=yes
 
 while [ -n "$1" ]; do
   if test "$1" = "--skip-pg"; then
@@ -20,6 +21,9 @@ while [ -n "$1" ]; do
   elif test "$1" = "--skip-redis"; then
     PREPARE_REDIS=no
     shift; continue
+  elif test "$1" = "--no-sql-download"; then
+    DOWNLOAD_SQL_FILES=no
+    shift; continue
   fi
 done
 
@@ -71,26 +75,31 @@ if test x"$PREPARE_PGSQL" = xyes; then
   dropdb "${TEST_DB}"
   createdb -Ttemplate_postgis -EUTF8 "${TEST_DB}" || die "Could not create test database"
 
-  curl -L -s https://raw.githubusercontent.com/CartoDB/camshaft/master/test/fixtures/cdb_analysis_catalog.sql -o sql/cdb_analysis_catalog.sql
-  cat sql/cdb_analysis_catalog.sql | psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
+  LOCAL_SQL_SCRIPTS='windshaft.test gadm4 ported/populated_places_simple_reduced'
+  REMOTE_SQL_SCRIPTS='CDB_QueryStatements CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_AnalysisCatalog CDB_ZoomFromScale CDB_OverviewsSupport CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins CDB_Hexagon CDB_XYZ'
 
-  cat sql/windshaft.test.sql sql/gadm4.sql |
-    sed "s/:PUBLICUSER/${PUBLICUSER}/" |
-    sed "s/:PUBLICPASS/${PUBLICPASS}/" |
-    sed "s/:TESTUSER/${TESTUSER}/" |
-    sed "s/:TESTPASS/${TESTPASS}/" |
-    psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
-
-  cat sql/_CDB_QueryStatements.sql | psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
-
-  SQL_SCRIPTS='CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins CDB_Hexagon CDB_XYZ'
-  for i in ${SQL_SCRIPTS}
+  CURL_ARGS=""
+  for i in ${REMOTE_SQL_SCRIPTS}
   do
-    curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/master/scripts-available/$i.sql -o sql/$i.sql
-    cat sql/$i.sql | sed -e 's/cartodb\./public./g' -e "s/''cartodb''/''public''/g" \
-        | psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
+    CURL_ARGS="${CURL_ARGS}\"https://github.com/CartoDB/cartodb-postgresql/raw/master/scripts-available/$i.sql\" -o sql/$i.sql "
   done
+  if test x"$DOWNLOAD_SQL_FILES" = xyes; then
+    echo "Downloading and updating: ${REMOTE_SQL_SCRIPTS}"
+    echo ${CURL_ARGS} | xargs curl -L -s
+  fi
 
+  psql -c "CREATE EXTENSION IF NOT EXISTS plpythonu;" ${TEST_DB}
+  ALL_SQL_SCRIPTS="${REMOTE_SQL_SCRIPTS} ${LOCAL_SQL_SCRIPTS}"
+  for i in ${ALL_SQL_SCRIPTS}
+  do
+    cat sql/${i}.sql |
+      sed -e 's/cartodb\./public./g' -e "s/''cartodb''/''public''/g" |
+      sed "s/:PUBLICUSER/${PUBLICUSER}/" |
+      sed "s/:PUBLICPASS/${PUBLICPASS}/" |
+      sed "s/:TESTUSER/${TESTUSER}/" |
+      sed "s/:TESTPASS/${TESTPASS}/" |
+      PGOPTIONS='--client-min-messages=WARNING' psql -q -v ON_ERROR_STOP=1 ${TEST_DB} > /dev/null || exit 1
+  done
 fi
 
 if test x"$PREPARE_REDIS" = xyes; then
@@ -122,17 +131,3 @@ EOF
 fi
 
 echo "Finished preparing data. Ready to run tests"
-
-
-############################ WINDSHAFT TESTS ############################
-
-echo "...Configuring Windshaft test database"
-
-cat sql/ported/populated_places_simple_reduced.sql |
-    sed "s/:PUBLICUSER/${PUBLICUSER}/" |
-    sed "s/:PUBLICPASS/${PUBLICPASS}/" |
-    sed "s/:TESTUSER/${TESTUSER}/" |
-    sed "s/:TESTPASS/${TESTPASS}/" |
-    psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
-
-echo "...Test database configuration complete"

test/support/sql/_CDB_QueryStatements.sql

@@ -1,16 +0,0 @@
--- DUMMY IMPLEMENTATION
--- Ref: https://github.com/CartoDB/cartodb-postgresql/blob/master/scripts-available/CDB_QueryStatements.sql
--- Originally implemented in plpython for performance reasons
-
--- Return an array of statements found in the given query text
---
--- Regexp curtesy of Hubert Lubaczewski (depesz)
---
-CREATE OR REPLACE FUNCTION CDB_QueryStatements(query text)
-RETURNS SETOF TEXT AS $$
-  with matches as (
-    select regexp_matches($1, $regexp$((?:[^'"$;]+|"[^"]*"|'[^']*'|(\$[^$]*\$).*?\2)+)$regexp$, 'g') as m
-  )
-  select btrim(m[1]) from matches
-$$
-LANGUAGE SQL IMMUTABLE STRICT;

test/support/test-client.js

@@ -24,6 +24,15 @@ function TestClient(mapConfig, apiKey) {
 
 module.exports = TestClient;
 
+module.exports.RESPONSE = {
+    ERROR: {
+        status: 400,
+        headers: {
+            'Content-Type': 'application/json; charset=utf-8'
+        }
+    }
+};
+
 TestClient.prototype.getWidget = function(widgetName, params, callback) {
     var self = this;
 
@@ -62,6 +71,7 @@ TestClient.prototype.getWidget = function(widgetName, params, callback) {
                         return next(err);
                     }
                     var parsedBody = JSON.parse(res.body);
+
                     var expectedWidgetURLS = {
                         http: "/api/v1/map/" + parsedBody.layergroupid + "/0/widget/" + widgetName
                     };
@@ -69,6 +79,15 @@ TestClient.prototype.getWidget = function(widgetName, params, callback) {
                     assert.ok(
                         parsedBody.metadata.layers[0].widgets[widgetName].url.http.match(expectedWidgetURLS.http)
                     );
+
+                    var expectedDataviewsURLS = {
+                        http: "/api/v1/map/" + parsedBody.layergroupid + "/dataview/" + widgetName
+                    };
+                    assert.ok(parsedBody.metadata.dataviews[widgetName]);
+                    assert.ok(
+                        parsedBody.metadata.dataviews[widgetName].url.http.match(expectedDataviewsURLS.http)
+                    );
+
                     return next(null, parsedBody.layergroupid);
                 }
             );
@@ -82,9 +101,12 @@ TestClient.prototype.getWidget = function(widgetName, params, callback) {
             var urlParams = {
                 own_filter: params.hasOwnProperty('own_filter') ? params.own_filter : 1
             };
-            if (params && params.bbox) {
-                urlParams.bbox = params.bbox;
-            }
+            ['bbox', 'bins', 'start', 'end'].forEach(function(extraParam) {
+                if (params.hasOwnProperty(extraParam)) {
+                    urlParams[extraParam] = params[extraParam];
+                }
+            });
+
             url = '/api/v1/map/' + layergroupId + '/0/widget/' + widgetName + '?' + qs.stringify(urlParams);
 
             assert.response(server,
@@ -113,7 +135,120 @@ TestClient.prototype.getWidget = function(widgetName, params, callback) {
         function finish(err, res) {
             self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
             self.keysToDelete['user:localhost:mapviews:global'] = 5;
-            return callback(err, res);
+            var widget;
+            if (!err && res.body) {
+                widget = JSON.parse(res.body);
+            }
+            return callback(err, res, widget);
+        }
+    );
+};
+
+TestClient.prototype.widgetSearch = function(widgetName, userQuery, params, callback) {
+    var self = this;
+
+    if (!callback) {
+        callback = params;
+        params = {};
+    }
+
+    var url = '/api/v1/map';
+    if (params && params.filters) {
+        url += '?' + qs.stringify({ filters: JSON.stringify(params.filters) });
+    }
+
+    var layergroupId;
+    step(
+        function createLayergroup() {
+            var next = this;
+            assert.response(server,
+                {
+                    url: url,
+                    method: 'POST',
+                    headers: {
+                        host: 'localhost',
+                        'Content-Type': 'application/json'
+                    },
+                    data: JSON.stringify(self.mapConfig)
+                },
+                {
+                    status: 200,
+                    headers: {
+                        'Content-Type': 'application/json; charset=utf-8'
+                    }
+                },
+                function(res, err) {
+                    if (err) {
+                        return next(err);
+                    }
+                    var parsedBody = JSON.parse(res.body);
+
+                    var expectedWidgetURLS = {
+                        http: "/api/v1/map/" + parsedBody.layergroupid + "/0/widget/" + widgetName
+                    };
+                    assert.ok(parsedBody.metadata.layers[0].widgets[widgetName]);
+                    assert.ok(
+                        parsedBody.metadata.layers[0].widgets[widgetName].url.http.match(expectedWidgetURLS.http)
+                    );
+
+                    var expectedDataviewsURLS = {
+                        http: "/api/v1/map/" + parsedBody.layergroupid + "/dataview/" + widgetName
+                    };
+                    assert.ok(parsedBody.metadata.dataviews[widgetName]);
+                    assert.ok(
+                        parsedBody.metadata.dataviews[widgetName].url.http.match(expectedDataviewsURLS.http)
+                    );
+
+                    return next(null, parsedBody.layergroupid);
+                }
+            );
+        },
+        function getWidgetSearchResult(err, _layergroupId) {
+            assert.ifError(err);
+
+            var next = this;
+            layergroupId = _layergroupId;
+
+            var urlParams = {
+                q: userQuery,
+                own_filter: params.hasOwnProperty('own_filter') ? params.own_filter : 1
+            };
+            if (params && params.bbox) {
+                urlParams.bbox = params.bbox;
+            }
+            url = '/api/v1/map/' + layergroupId + '/0/widget/' + widgetName + '/search?' + qs.stringify(urlParams);
+
+            assert.response(server,
+                {
+                    url: url,
+                    method: 'GET',
+                    headers: {
+                        host: 'localhost'
+                    }
+                },
+                {
+                    status: 200,
+                    headers: {
+                        'Content-Type': 'application/json; charset=utf-8'
+                    }
+                },
+                function(res, err) {
+                    if (err) {
+                        return next(err);
+                    }
+
+                    next(null, res);
+                }
+            );
+        },
+        function finish(err, res) {
+            self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
+            self.keysToDelete['user:localhost:mapviews:global'] = 5;
+            var searchResult;
+            if (!err && res.body) {
+                searchResult = JSON.parse(res.body);
+            }
+            return callback(err, res, searchResult);
         }
     );
 };
@@ -184,9 +319,13 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) {
             var urlParams = {
                 own_filter: params.hasOwnProperty('own_filter') ? params.own_filter : 1
             };
-            if (params && params.bbox) {
-                urlParams.bbox = params.bbox;
-            }
+
+            ['bbox', 'bins', 'start', 'end'].forEach(function(extraParam) {
+                if (params.hasOwnProperty(extraParam)) {
+                    urlParams[extraParam] = params[extraParam];
+                }
+            });
+
             if (self.apiKey) {
                 urlParams.api_key = self.apiKey;
             }
@@ -215,10 +354,10 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) {
                 }
             );
         },
-        function finish(err, res) {
+        function finish(err, dataview) {
             self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
             self.keysToDelete['user:localhost:mapviews:global'] = 5;
-            return callback(err, res);
+            return callback(err, dataview);
         }
     );
 };

test/unit/cartodb/mapconfig/dataviews-widgets-adapter.test.js

@@ -0,0 +1,331 @@
+//require('../../../support/test_helper');
+var assert = require('assert');
+
+var DataviewsMapConfigAdapter = require('../../../../lib/cartodb/models/mapconfig/adapter/dataviews-widgets-adapter');
+
+describe('dataviews-widgets-adapter', function() {
+
+    var widgetsMapConfigs = [
+        {
+            "input": {
+                "version": "1.4.0",
+                "layers": [
+                    {
+                        "type": "mapnik",
+                        "options": {
+                            "sql": "select * from populated_places_simple_reduced",
+                            "cartocss": "#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }",
+                            "cartocss_version": "2.3.0",
+                            "widgets": {
+                                "country_places_count": {
+                                    "type": "aggregation",
+                                    "options": {
+                                        "column": "adm0_a3",
+                                        "aggregation": "count"
+                                    }
+                                }
+                            }
+                        }
+                    }
+                ]
+            },
+            "expected": {
+                "version": "1.4.0",
+                "layers": [
+                    {
+                        "type": "mapnik",
+                        "options": {
+                            "source": {
+                                "id": "cdb-layer-source-0"
+                            },
+                            "cartocss": "#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }",
+                            "cartocss_version": "2.3.0",
+                            // keep them for now
+                            "widgets": {
+                                "country_places_count": {
+                                    "type": "aggregation",
+                                    "options": {
+                                        "column": "adm0_a3",
+                                        "aggregation": "count"
+                                    }
+                                }
+                            }
+                        }
+                    }
+                ],
+                "analyses": [
+                    {
+                        "id": "cdb-layer-source-0",
+                        "type": "source",
+                        "params": {
+                            "query": "select * from populated_places_simple_reduced"
+                        }
+                    }
+                ],
+                "dataviews": {
+                    "country_places_count": {
+                        "source": {
+                            "id": "cdb-layer-source-0"
+                        },
+                        "type": "aggregation",
+                        "options": {
+                            "column": "adm0_a3",
+                            "aggregation": "count"
+                        }
+                    }
+                }
+            }
+        },
+        {
+            "input": {
+                "version": "1.4.0",
+                "layers": [
+                    {
+                        "type": "mapnik",
+                        "options": {
+                            "sql": "select * from populated_places_simple_reduced",
+                            "cartocss": "#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }",
+                            "cartocss_version": "2.3.0",
+                            "widgets": {
+                                "pop_max": {
+                                    "type": "histogram",
+                                    "options": {
+                                        "column": "pop_max"
+                                    }
+                                }
+                            }
+                        }
+                    }
+                ]
+            },
+            "expected": {
+                "version": "1.4.0",
+                "layers": [
+                    {
+                        "type": "mapnik",
+                        "options": {
+                            "source": {
+                                "id": "cdb-layer-source-0"
+                            },
+                            "cartocss": "#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }",
+                            "cartocss_version": "2.3.0",
+                            // keep them for now
+                            "widgets": {
+                                "pop_max": {
+                                    "type": "histogram",
+                                    "options": {
+                                        "column": "pop_max"
+                                    }
+                                }
+                            }
+                        }
+                    }
+                ],
+                "analyses": [
+                    {
+                        "id": "cdb-layer-source-0",
+                        "type": "source",
+                        "params": {
+                            "query": "select * from populated_places_simple_reduced"
+                        }
+                    }
+                ],
+                "dataviews": {
+                    "pop_max": {
+                        "source": {
+                            "id": "cdb-layer-source-0"
+                        },
+                        "type": "histogram",
+                        "options": {
+                            "column": "pop_max"
+                        }
+                    }
+                }
+            }
+        },
+        {
+            "input": {
+                "version": "1.4.0",
+                "layers": [
+                    {
+                        "type": "mapnik",
+                        "options": {
+                            "sql": "select * from test_table",
+                            "cartocss": "#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }",
+                            "cartocss_version": "2.3.0",
+                            "widgets": {
+                                "names": {
+                                    "type": "list",
+                                    "options": {
+                                        "columns": [
+                                            "name"
+                                        ]
+                                    }
+                                }
+                            }
+                        }
+                    }
+                ]
+            },
+            "expected": {
+                "version": "1.4.0",
+                "layers": [
+                    {
+                        "type": "mapnik",
+                        "options": {
+                            "source": {
+                                "id": "cdb-layer-source-0"
+                            },
+                            "cartocss": "#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }",
+                            "cartocss_version": "2.3.0",
+                            // keep them for now
+                            "widgets": {
+                                "names": {
+                                    "type": "list",
+                                    "options": {
+                                        "columns": [
+                                            "name"
+                                        ]
+                                    }
+                                }
+                            }
+                        }
+                    }
+                ],
+                "analyses": [
+                    {
+                        "id": "cdb-layer-source-0",
+                        "type": "source",
+                        "params": {
+                            "query": "select * from test_table"
+                        }
+                    }
+                ],
+                "dataviews": {
+                    "names": {
+                        "source": {
+                            "id": "cdb-layer-source-0"
+                        },
+                        "type": "list",
+                        "options": {
+                            "columns": [
+                                "name"
+                            ]
+                        }
+                    }
+                }
+            }
+        },
+        {
+            "input": {
+                "version": "1.4.0",
+                "layers": [
+                    {
+                        "type": "mapnik",
+                        "options": {
+                            "sql": "select * from populated_places_simple_reduced",
+                            "cartocss": "#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }",
+                            "cartocss_version": "2.3.0",
+                            "widgets": {
+                                "country_places_count": {
+                                    "type": "aggregation",
+                                    "options": {
+                                        "column": "adm0_a3",
+                                        "aggregation": "count"
+                                    }
+                                },
+                                "country_places_histogram": {
+                                    "type": "histogram",
+                                    "options": {
+                                        "column": "pop_max"
+                                    }
+                                }
+                            }
+                        }
+                    }
+                ]
+            },
+            "expected": {
+                "version": "1.4.0",
+                "layers": [
+                    {
+                        "type": "mapnik",
+                        "options": {
+                            "source": {
+                                "id": "cdb-layer-source-0"
+                            },
+                            "cartocss": "#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }",
+                            "cartocss_version": "2.3.0",
+                            // keep them for now
+                            "widgets": {
+                                "country_places_count": {
+                                    "type": "aggregation",
+                                    "options": {
+                                        "column": "adm0_a3",
+                                        "aggregation": "count"
+                                    }
+                                },
+                                "country_places_histogram": {
+                                    "type": "histogram",
+                                    "options": {
+                                        "column": "pop_max"
+                                    }
+                                }
+                            }
+                        }
+                    }
+                ],
+                "analyses": [
+                    {
+                        "id": "cdb-layer-source-0",
+                        "type": "source",
+                        "params": {
+                            "query": "select * from populated_places_simple_reduced"
+                        }
+                    }
+                ],
+                "dataviews": {
+                    "country_places_count": {
+                        "source": {
+                            "id": "cdb-layer-source-0"
+                        },
+                        "type": "aggregation",
+                        "options": {
+                            "column": "adm0_a3",
+                            "aggregation": "count"
+                        }
+                    },
+                    "country_places_histogram": {
+                        "source": {
+                            "id": "cdb-layer-source-0"
+                        },
+                        "type": "histogram",
+                        "options": {
+                            "column": "pop_max"
+                        }
+                    }
+                }
+            }
+        }
+    ];
+
+    var user = 'wadus';
+    function params() {
+        return {};
+    }
+    function context() {
+        return {};
+    }
+
+    var dataviewsMapConfigAdapter = new DataviewsMapConfigAdapter();
+
+    widgetsMapConfigs.forEach(function(mapConfig, index) {
+        it('should adapt widgets ' + index, function(done) {
+            dataviewsMapConfigAdapter.getMapConfig(user, mapConfig.input, params(), context(), function(err, result) {
+                assert.deepEqual(result, mapConfig.expected);
+                done();
+            });
+        });
+    });
+
+});

test/unit/cartodb/model/filter/bbox-filters.test.js

@@ -0,0 +1,129 @@
+require('../../../../support/test_helper');
+var util = require('util');
+
+var assert = require('../../../../support/assert');
+var BboxFilter = require('../../../../../lib/cartodb/models/filter/bbox');
+
+var MAX_EXTENT_MERCATOR_REF = [
+    -BboxFilter.LONGITUDE_MAX_VALUE,
+    -BboxFilter.LATITUDE_MAX_VALUE,
+    BboxFilter.LONGITUDE_MAX_VALUE,
+    BboxFilter.LATITUDE_MAX_VALUE
+];
+
+describe('Bounding box filter', function() {
+
+    describe('wrap longitude', function() {
+        var longitudesScenarios = [
+            [[0, 90], [0, 90]],
+            [[-90, 0], [-90, 0]],
+            [[-90, 90], [-90, 90]],
+            [[-990, -720], [90, 360]],
+            [[810, 1080], [90, 360]],
+            [[-180, 180], [-180, 180]]
+        ];
+
+        longitudesScenarios.forEach(function(scenario) {
+            it(util.format('should adjust from %j to %j', scenario[0], scenario[1]), function() {
+                var we = BboxFilter.adjustLongitudeRange(scenario[0]);
+
+                assert.equal(
+                    we[0], scenario[1][0],
+                    util.format('west, got %d, expected %d, scenario: %s',
+                        we[1], scenario[1][1], JSON.stringify(scenario)
+                    )
+                );
+                assert.equal(
+                    we[1], scenario[1][1],
+                    util.format('east, got %d, expected %d, scenario: %s',
+                        we[1], scenario[1][1], JSON.stringify(scenario)
+                    )
+                );
+            });
+        });
+    });
+
+    function createFilter(bbox) {
+        return new BboxFilter({}, { bbox: bbox.join(',') });
+    }
+
+    function createRef(bbox) {
+        return bbox;
+//        return mercator.forward([bbox[0], bbox[1]]).concat(mercator.forward([bbox[2], bbox[3]]));
+    }
+
+
+    it('happy case', function() {
+        var bbox = [-90, -45, 90, 45];
+        var bboxFilter = createFilter(bbox);
+        assert.equal(bboxFilter.bboxes.length, 1);
+        assert.deepEqual(bboxFilter.bboxes[0], createRef(bbox));
+    });
+
+    describe('latitude', function() {
+        it('(hardcoded) clipping out of bounds', function() {
+            var bbox = [-180, -90, 180, 90];
+            var bboxFilter = createFilter(bbox);
+            assert.equal(bboxFilter.bboxes.length, 1);
+            assert.deepEqual(bboxFilter.bboxes[0], MAX_EXTENT_MERCATOR_REF);
+        });
+
+        it('clipping out of bounds', function() {
+            var bbox = [-180, -90, 180, 90];
+            var bboxFilter = createFilter(bbox);
+            assert.equal(bboxFilter.bboxes.length, 1);
+            assert.deepEqual(
+                bboxFilter.bboxes[0],
+                createRef([-180, -BboxFilter.LATITUDE_MAX_VALUE, 180, BboxFilter.LATITUDE_MAX_VALUE])
+            );
+        });
+    });
+
+    describe('longitude', function() {
+        it('generating multiple bbox for east out of bounds', function() {
+            var bbox = [90, -45, 360, 45];
+            var bboxFilter = createFilter(bbox);
+
+            assert.equal(bboxFilter.bboxes.length, 2, JSON.stringify([bboxFilter.bboxes, bbox]));
+
+            assert.deepEqual(
+                bboxFilter.bboxes[0],
+                createRef([90, -45, 180, 45])
+            );
+            assert.deepEqual(
+                bboxFilter.bboxes[1],
+                createRef([-180, -45, 0, 45])
+            );
+        });
+
+        it('generating multiple bbox for east out of bounds', function() {
+            var bbox = [-270, -45, 0, 45];
+            var bboxFilter = createFilter(bbox);
+
+            assert.equal(bboxFilter.bboxes.length, 2);
+
+            assert.deepEqual(
+                bboxFilter.bboxes[0],
+                createRef([90, -45, 180, 45])
+            );
+            assert.deepEqual(
+                bboxFilter.bboxes[1],
+                createRef([-180, -45, 0, 45])
+            );
+        });
+    });
+
+    describe('out of bounds', function() {
+        it('wraps longitude', function () {
+            var bbox = [-190, -45, 190, 45];
+            var bboxFilter = createFilter(bbox);
+
+            assert.equal(bboxFilter.bboxes.length, 1);
+
+            assert.deepEqual(
+                bboxFilter.bboxes[0],
+                createRef([-180, -45, 180, 45])
+            );
+        });
+    });
+});