Release version 2.43.0

Use overviews for widgets (dataviews, filtered queries)

NEWS.md

@@ -1,5 +1,52 @@
 # Changelog
 
+## 2.43.0
+
+Released 2016-05-18
+
+New features:
+ - Overviews now support dataviews and filtering #449
+
+
+## 2.42.2
+
+Released 2016-05-17
+
+New features:
+ - turbo-carto: mapnik substitution tokens support #455
+
+
+## 2.42.1
+
+Released 2016-05-17
+- Upgraded turbo-carto to fix reversed color scales
+
+
+## 2.42.0
+
+Released 2016-05-16
+
+Bug fixes:
+ - Fix named maps with analysis #453
+
+Enhancements:
+ - Use split strategy for head/tails turbo-carto quantification
+
+Announcements:
+ - Upgrades turbo-carto to [0.9.0](https://github.com/CartoDB/turbo-carto/releases/tag/0.9.0)
+
+
+## 2.41.1
+
+Released 2016-05-11
+
+Announcements:
+ - Upgrades camshaft to [0.8.0](https://github.com/CartoDB/camshaft/releases/tag/0.8.0)
+
+Bug fixes:
+ - Nicer error message when missing sql from layer options #446
+
+
 ## 2.41.0
 
 Released 2016-05-11

lib/cartodb/api/filter_stats_api.js

@@ -0,0 +1,59 @@
+var _ = require('underscore');
+var step = require('step');
+var CamshaftFilter = require('../models/filter/camshaft');
+
+function FilterStatsApi(pgQueryRunner) {
+    this.pgQueryRunner = pgQueryRunner;
+}
+
+module.exports = FilterStatsApi;
+
+function getEstimatedRows(pgQueryRunner, username, query, callback) {
+    pgQueryRunner.run(username, "EXPLAIN (FORMAT JSON)"+query, function(err, result_rows) {
+        if (err){
+            callback(err);
+            return;
+        }
+        var rows;
+        if ( result_rows[0] && result_rows[0]['QUERY PLAN'] &&
+             result_rows[0]['QUERY PLAN'][0] && result_rows[0]['QUERY PLAN'][0].Plan ) {
+            rows = result_rows[0]['QUERY PLAN'][0].Plan['Plan Rows'];
+        }
+        return callback(null, rows);
+    });
+}
+
+FilterStatsApi.prototype.getFilterStats = function (username, unfiltered_query, filters, callback) {
+  var stats = {};
+  var self = this;
+  step(
+      function getUnfilteredRows() {
+          getEstimatedRows(self.pgQueryRunner, username, unfiltered_query, this);
+      },
+      function receiveUnfilteredRows(err, rows) {
+          if (err){
+              callback(err);
+              return;
+          }
+          stats.unfiltered_rows = rows;
+          this(null, rows);
+      },
+      function getFilteredRows() {
+          if ( filters && !_.isEmpty(filters)) {
+              var camshaftFilter = new CamshaftFilter(filters);
+              var query = camshaftFilter.sql(unfiltered_query);
+              getEstimatedRows(self.pgQueryRunner, username, query, this);
+          } else {
+              this(null, null);
+          }
+      },
+      function receiveFilteredRows(err, rows) {
+          if (err){
+              callback(err);
+              return;
+          }
+          stats.filtered_rows = rows;
+          callback(null, stats);
+      }
+  );
+};

lib/cartodb/api/overviews_metadata_api.js

@@ -1,24 +1,18 @@
+var SubstitutionTokens = require('../utils/substitution-tokens');
+
 function OverviewsMetadataApi(pgQueryRunner) {
     this.pgQueryRunner = pgQueryRunner;
 }
 
 module.exports = OverviewsMetadataApi;
 
-// TODO: share this with QueryTablesApi? ... or maintain independence?
-var affectedTableRegexCache = {
-    bbox: /!bbox!/g,
-    scale_denominator: /!scale_denominator!/g,
-    pixel_width: /!pixel_width!/g,
-    pixel_height: /!pixel_height!/g
-};
-
 function prepareSql(sql) {
-    return sql
-        .replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)')
-        .replace(affectedTableRegexCache.scale_denominator, '0')
-        .replace(affectedTableRegexCache.pixel_width, '1')
-        .replace(affectedTableRegexCache.pixel_height, '1')
-    ;
+    return sql && SubstitutionTokens.replace(sql, {
+        bbox: 'ST_MakeEnvelope(0,0,0,0)',
+        scale_denominator: '0',
+        pixel_width: '1',
+        pixel_height: '1'
+    });
 }
 
 OverviewsMetadataApi.prototype.getOverviewsMetadata = function (username, sql, callback) {

lib/cartodb/backends/dataview.js

@@ -8,7 +8,13 @@ var step = require('step');
 var Timer = require('../stats/timer');
 
 var BBoxFilter = require('../models/filter/bbox');
+
 var DataviewFactory = require('../models/dataview/factory');
+var DataviewFactoryWithOverviews = require('../models/dataview/overviews/factory');
+var OverviewsQueryRewriter = require('../utils/overviews_query_rewriter');
+var overviewsQueryRewriter = new OverviewsQueryRewriter({
+    zoom_level: 'CDB_ZoomFromScale(!scale_denominator!)'
+});
 
 function DataviewBackend(analysisBackend) {
     this.analysisBackend = analysisBackend;
@@ -16,7 +22,6 @@ function DataviewBackend(analysisBackend) {
 
 module.exports = DataviewBackend;
 
-
 DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, params, callback) {
     var self = this;
 
@@ -101,6 +106,7 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param
             ownFilter = !!ownFilter;
 
             var query;
+
             if (ownFilter) {
                 query = node.getQuery();
             } else {
@@ -109,11 +115,46 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param
                 query = node.getQuery(applyFilters);
             }
 
+            var sourceId =  dataviewDefinition.source.id; // node.id
+            var layer = _.find(
+                mapConfig.obj().layers,
+                function(l){ return l.options.source && (l.options.source.id === sourceId); }
+            );
+            var queryRewriteData = layer && layer.options.query_rewrite_data;
+            if ( queryRewriteData ) {
+                if ( node.type === 'source' ) {
+                    var filters = node.filters; // TODO: node.getFilters() when available in camshaft
+                    var filters_disabler = Object.keys(filters).reduce(
+                        function(disabler, filter_id){ disabler[filter_id] = false; return disabler; },
+                        {}
+                    );
+                    var unfiltered_query = node.getQuery(filters_disabler);
+                    queryRewriteData = _.extend(
+                        {},
+                        queryRewriteData, { filters: filters, unfiltered_query: unfiltered_query }
+                    );
+                }
+            }
+
             if (params.bbox) {
                 var bboxFilter = new BBoxFilter({column: 'the_geom', srid: 4326}, {bbox: params.bbox});
                 query = bboxFilter.sql(query);
+                var bbox_filter_definition = {
+                    type: 'bbox',
+                    options: {
+                        column: 'the_geom',
+                        srid: 4326,
+                    },
+                    params: {
+                        bbox: params.bbox
+                    }
+                };
+                queryRewriteData = _.extend(queryRewriteData, { bbox_filter: bbox_filter_definition });
             }
 
+            var dataviewFactory = DataviewFactoryWithOverviews.getFactory(
+                overviewsQueryRewriter, queryRewriteData, { bbox: params.bbox }
+            );
 
             var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins'),
                 function castNumbers(overrides, val, k) {
@@ -123,7 +164,7 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param
                 {ownFilter: ownFilter}
             );
 
-            var dataview = DataviewFactory.getDataview(query, dataviewDefinition);
+            var dataview = dataviewFactory.getDataview(query, dataviewDefinition);
             dataview.getResult(pg, overrideParams, this);
         },
         function returnCallback(err, result) {
@@ -277,4 +318,4 @@ function dbParamsFromReqParams(params) {
         dbParams.dbname = params.dbname;
     }
     return dbParams;
-}
+}

lib/cartodb/cache/named_map_provider_cache.js

@@ -8,15 +8,15 @@ var queue = require('queue-async');
 
 var LruCache = require("lru-cache");
 
-function NamedMapProviderCache(templateMaps, pgConnection, metadataBackend, userLimitsApi, overviewsAdapter,
-                               turboCartoAdapter) {
+function NamedMapProviderCache(templateMaps, pgConnection, metadataBackend, analysisBackend, userLimitsApi,
+                               overviewsAdapter, turboCartoAdapter) {
     this.templateMaps = templateMaps;
     this.pgConnection = pgConnection;
     this.metadataBackend = metadataBackend;
     this.userLimitsApi = userLimitsApi;
 
     this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps);
-    this.analysisMapConfigAdapter = new AnalysisMapConfigAdapter();
+    this.analysisMapConfigAdapter = new AnalysisMapConfigAdapter(analysisBackend);
     this.overviewsAdapter = overviewsAdapter;
     this.turboCartoAdapter = turboCartoAdapter;
 

lib/cartodb/controllers/map.js

@@ -185,17 +185,20 @@ MapController.prototype.create = function(req, res, prepareConfigFn) {
         function addOverviewsInformation(err, requestMapConfig, datasource) {
             assert.ifError(err);
             var next = this;
-            self.overviewsAdapter.getLayers(req.context.user, requestMapConfig.layers, function(err, layers) {
-                if (err) {
-                    return next(err);
-                }
+            self.overviewsAdapter.getLayers(
+                req.context.user, requestMapConfig.layers, analysesResults,
+                function(err, layers) {
+                    if (err) {
+                        return next(err);
+                    }
 
-                if (layers) {
-                    requestMapConfig.layers = layers;
-                }
+                    if (layers) {
+                        requestMapConfig.layers = layers;
+                    }
 
-                return next(null, requestMapConfig, datasource);
-            });
+                    return next(null, requestMapConfig, datasource);
+                }
+            );
         },
         function parseTurboCarto(err, requestMapConfig, datasource) {
             assert.ifError(err);

lib/cartodb/models/dataview/overviews/factory.js

@@ -0,0 +1,32 @@
+var parentFactory = require('../factory');
+var dataviews = require('./');
+
+function OverviewsDataviewFactory(queryRewriter, queryRewriteData, options) {
+    this.queryRewriter = queryRewriter;
+    this.queryRewriteData = queryRewriteData;
+    this.options = options;
+}
+
+OverviewsDataviewFactory.prototype.getDataview = function(query, dataviewDefinition) {
+    var type = dataviewDefinition.type;
+    var dataviews = OverviewsDataviewMetaFactory.dataviews;
+    if ( !this.queryRewriter || !this.queryRewriteData || !dataviews[type] ) {
+        return parentFactory.getDataview(query, dataviewDefinition);
+    }
+    return new dataviews[type](
+        query, dataviewDefinition.options, this.queryRewriter, this.queryRewriteData, this.options
+    );
+};
+
+var OverviewsDataviewMetaFactory = {
+    dataviews: Object.keys(dataviews).reduce(function(allDataviews, dataviewClassName) {
+        allDataviews[dataviewClassName.toLowerCase()] = dataviews[dataviewClassName];
+        return allDataviews;
+    }, {}),
+
+    getFactory: function(queryRewriter, queryRewriteData, options) {
+        return new OverviewsDataviewFactory(queryRewriter, queryRewriteData, options);
+    },
+};
+
+module.exports = OverviewsDataviewMetaFactory;

lib/cartodb/models/dataview/overviews/formula.js

@@ -0,0 +1,104 @@
+var _ = require('underscore');
+var BaseWidget = require('../base');
+var BaseDataview = require('../formula');
+
+var debug = require('debug')('windshaft:widget:formula:overviews');
+
+var dot = require('dot');
+dot.templateSettings.strip = false;
+
+var formulaQueryTpls = {
+  'count': dot.template([
+      'SELECT',
+      'sum(_feature_count) AS result,',
+      '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
+      'FROM ({{=it._query}}) _cdb_formula'
+  ].join('\n')),
+  'sum': dot.template([
+      'SELECT',
+      'sum({{=it._column}}*_feature_count) AS result,',
+      '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
+      'FROM ({{=it._query}}) _cdb_formula'
+  ].join('\n')),
+  'avg': dot.template([
+      'SELECT',
+      'sum({{=it._column}}*_feature_count)/sum(_feature_count) AS result,',
+      '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
+      'FROM ({{=it._query}}) _cdb_formula'
+  ].join('\n')),
+};
+
+function Formula(query, options, queryRewriter, queryRewriteData, params) {
+    this.base_dataview = new BaseDataview(query, options);
+    this.query = query;
+    this.column = options.column || '1';
+    this.operation = options.operation;
+    this.queryRewriter = queryRewriter;
+    this.queryRewriteData = queryRewriteData;
+    this.options = params;
+}
+
+Formula.prototype = new BaseWidget();
+Formula.prototype.constructor = Formula;
+
+module.exports = Formula;
+
+var zoom_level_factor = 100.0;
+
+// Compute zoom level so that the the resolution grid size of the
+// selected overview is smaller (zoom_level_factor times smaller at least)
+// than the bounding box size.
+function zoom_level_for_bbox(bbox) {
+    var px_per_tile = 256.0;
+    var earth_width = 360.0;
+    // TODO: now we assume overviews are computed for 1-pixel tolerance;
+    // should use extended overviews metadata to compute this properly.
+    if ( bbox ) {
+      var bbox_values = _.map(bbox.split(','), function(v) { return +v; });
+      var w = Math.abs(bbox_values[2]-bbox_values[0]);
+      var h = Math.abs(bbox_values[3]-bbox_values[1]);
+      var max_dim = Math.min(w, h);
+
+      // Find minimum suitable z
+      // note that the QueryRewirter will use the minimum level overview
+      // of level >= z if it exists, and otherwise the base table
+      var z = Math.ceil(-Math.log(max_dim*px_per_tile/earth_width/zoom_level_factor)/Math.log(2.0));
+      return Math.max(z, 0);
+    }
+    return 0;
+}
+
+Formula.prototype.sql = function(psql, filters, override, callback) {
+    var _query = this.query;
+    var formulaQueryTpl = formulaQueryTpls[this.operation];
+
+    if ( formulaQueryTpl ) {
+        // supported formula for use with overviews
+        var zoom_level = zoom_level_for_bbox(this.options.bbox);
+        _query = this.queryRewriter.query(_query, this.queryRewriteData, { zoom_level: zoom_level });
+        var formulaSql = formulaQueryTpl({
+            _query: _query,
+            _operation: this.operation,
+            _column: this.column
+        });
+        debug(formulaSql);
+        callback = callback || override;
+
+        return callback(null, formulaSql);
+    }
+
+    // For non supported operations (min, max) we're not using overviews.
+    return this.base_dataview.sql(psql, filters, override, callback);
+};
+
+Formula.prototype.format = function(result) {
+    return this.base_dataview.format(result);
+};
+
+Formula.prototype.getType = function() {
+    return this.base_dataview.getType();
+};
+
+Formula.prototype.toString = function() {
+    return this.base_dataview.toString();
+};

lib/cartodb/models/dataview/overviews/index.js

@@ -0,0 +1,3 @@
+module.exports = {
+    Formula: require('./formula')
+};

lib/cartodb/models/filter/camshaft.js

@@ -0,0 +1,35 @@
+var filters = {
+    category: require('./camshaft/category'),
+    range: require('./camshaft/range')
+};
+
+function createFilter(filterDefinition) {
+    var filterType = filterDefinition.type.toLowerCase();
+    if (!filters.hasOwnProperty(filterType)) {
+        throw new Error('Unknown filter type: ' + filterType);
+    }
+    return new filters[filterType](filterDefinition.column, filterDefinition.params);
+}
+
+function CamshaftFilters(filters) {
+    this.filters = filters;
+}
+
+CamshaftFilters.prototype.sql = function(rawSql) {
+  var filters = this.filters || {};
+  var applyFilters = {};
+
+  return Object.keys(filters)
+      .filter(function(filterName) {
+          return applyFilters.hasOwnProperty(filterName) ? applyFilters[filterName] : true;
+      })
+      .map(function(filterName) {
+          var filterDefinition = filters[filterName];
+          return createFilter(filterDefinition);
+      })
+      .reduce(function(sql, filter) {
+          return filter.sql(sql);
+      }, rawSql);
+};
+
+module.exports = CamshaftFilters;

lib/cartodb/models/filter/camshaft/category.js

@@ -0,0 +1,79 @@
+'use strict';
+
+var debug = require('debug')('windshaft:filter:category');
+var dot = require('dot');
+dot.templateSettings.strip = false;
+
+var filterQueryTpl = dot.template([
+    'SELECT *',
+    'FROM ({{=it._sql}}) _camshaft_category_filter',
+    'WHERE {{=it._filters}}'
+].join('\n'));
+var escapeStringTpl = dot.template('$escape_{{=it._i}}${{=it._value}}$escape_{{=it._i}}$');
+var inConditionTpl = dot.template('{{=it._column}} IN ({{=it._values}})');
+var notInConditionTpl = dot.template('{{=it._column}} NOT IN ({{=it._values}})');
+
+function Category(column, filterParams) {
+    this.column = column;
+
+    if (!Array.isArray(filterParams.accept) && !Array.isArray(filterParams.reject)) {
+        throw new Error('Category filter expects at least one array in accept or reject params');
+    }
+
+    if (Array.isArray(filterParams.accept) && Array.isArray(filterParams.reject)) {
+        if (filterParams.accept.length === 0 && filterParams.reject.length === 0) {
+            throw new Error(
+                'Category filter expects one value either in accept or reject params when both are provided'
+            );
+        }
+    }
+
+    this.accept = filterParams.accept;
+    this.reject = filterParams.reject;
+}
+
+module.exports = Category;
+
+/*
+ - accept: [] => reject all
+ - reject: [] => accept all
+ */
+Category.prototype.sql = function(rawSql) {
+    var valueFilters = [];
+
+    if (Array.isArray(this.accept)) {
+        if (this.accept.length > 0) {
+            valueFilters.push(inConditionTpl({
+                _column: this.column,
+                _values: this.accept.map(function(value, i) {
+                    return Number.isFinite(value) ? value : escapeStringTpl({_i: i, _value: value});
+                }).join(',')
+            }));
+        } else {
+            valueFilters.push('0 = 1');
+        }
+    }
+
+    if (Array.isArray(this.reject)) {
+        if (this.reject.length > 0) {
+            valueFilters.push(notInConditionTpl({
+                _column: this.column,
+                _values: this.reject.map(function (value, i) {
+                    return Number.isFinite(value) ? value : escapeStringTpl({_i: i, _value: value});
+                }).join(',')
+            }));
+        } else {
+            valueFilters.push('1 = 1');
+        }
+    }
+
+    debug(filterQueryTpl({
+        _sql: rawSql,
+        _filters: valueFilters.join(' AND ')
+    }));
+
+    return filterQueryTpl({
+        _sql: rawSql,
+        _filters: valueFilters.join(' AND ')
+    });
+};

lib/cartodb/models/filter/camshaft/range.js

@@ -0,0 +1,43 @@
+'use strict';
+
+var dot = require('dot');
+dot.templateSettings.strip = false;
+
+var betweenFilterTpl = dot.template('{{=it._column}} BETWEEN {{=it._min}} AND {{=it._max}}');
+var minFilterTpl = dot.template('{{=it._column}} >= {{=it._min}}');
+var maxFilterTpl = dot.template('{{=it._column}} <= {{=it._max}}');
+var filterQueryTpl = dot.template('SELECT * FROM ({{=it._sql}}) _camshaft_range_filter WHERE {{=it._filter}}');
+
+function Range(column, filterParams) {
+    this.column = column;
+
+    if (!Number.isFinite(filterParams.min) && !Number.isFinite(filterParams.max)) {
+        throw new Error('Range filter expect to have at least one value in min or max numeric params');
+    }
+
+    this.min = filterParams.min;
+    this.max = filterParams.max;
+    this.columnType = filterParams.columnType;
+}
+
+module.exports = Range;
+
+Range.prototype.sql = function(rawSql) {
+    var minMaxFilter;
+    if (Number.isFinite(this.min) && Number.isFinite(this.max)) {
+        minMaxFilter = betweenFilterTpl({
+            _column: this.column,
+            _min: this.min,
+            _max: this.max
+        });
+    } else if (Number.isFinite(this.min)) {
+        minMaxFilter = minFilterTpl({ _column: this.column, _min: this.min });
+    } else {
+        minMaxFilter = maxFilterTpl({ _column: this.column, _max: this.max });
+    }
+
+    return filterQueryTpl({
+        _sql: rawSql,
+        _filter: minMaxFilter
+    });
+};

lib/cartodb/models/mapconfig/named_map_provider.js

@@ -142,7 +142,7 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) {
             assert.ifError(err);
             var next = this;
 
-            self.overviewsAdapter.getLayers(self.owner, _mapConfig.layers, function(err, layers) {
+            self.overviewsAdapter.getLayers(self.owner, _mapConfig.layers, self.analysesResults, function(err, layers) {
                 if (err) {
                     return next(err);
                 }

lib/cartodb/models/mapconfig_overviews_adapter.js

@@ -1,13 +1,15 @@
+var step = require('step');
 var queue = require('queue-async');
 var _ = require('underscore');
 
-function MapConfigOverviewsAdapter(overviewsMetadataApi) {
+function MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi) {
     this.overviewsMetadataApi = overviewsMetadataApi;
+    this.filterStatsApi = filterStatsApi;
 }
 
 module.exports = MapConfigOverviewsAdapter;
 
-MapConfigOverviewsAdapter.prototype.getLayers = function(username, layers, callback) {
+MapConfigOverviewsAdapter.prototype.getLayers = function(username, layers, analysesResults, callback) {
   var self = this;
 
   if (!layers || layers.length === 0) {
@@ -24,11 +26,52 @@ MapConfigOverviewsAdapter.prototype.getLayers = function(username, layers, callb
            if (err) {
                done(err, layer);
            } else {
-               if ( !_.isEmpty(metadata) ) {
-                   layer = _.extend({}, layer);
-                   layer.options = _.extend({}, layer.options, { query_rewrite_data: { overviews: metadata } });
-               }
-               done(null, layer);
+               var query_rewrite_data = { overviews: metadata };
+               step(
+                   function collectFiltersData() {
+                       var filters, unfiltered_query;
+                       if ( layer.options.source && analysesResults ) {
+                           var sourceId = layer.options.source.id;
+                           var node = _.find(analysesResults, function(a){ return a.rootNode.params.id === sourceId; });
+                           if ( node ) {
+                               node = node.rootNode;
+                               filters = node.filters; // TODO: node.getFilters() when available in camshaft
+                               var filters_disabler = Object.keys(filters).reduce(
+                                   function(disabler, filter_id){ disabler[filter_id] = false; return disabler; },
+                                   {}
+                               );
+                               unfiltered_query = node.getQuery(filters_disabler);
+                               query_rewrite_data.filters = filters;
+                               query_rewrite_data.unfiltered_query = unfiltered_query;
+                           }
+                       }
+                       this(null, filters, unfiltered_query);
+                   },
+                   function collectStatsData(err, filters, unfiltered_query) {
+                       var next_step = this;
+                       if ( filters ) {
+                           self.filterStatsApi.getFilterStats(
+                               username,
+                               unfiltered_query, filters,
+                               function(err, stats) {
+                                 if ( !err ) {
+                                     query_rewrite_data.filter_stats = stats;
+                                 }
+                                 return next_step(err);
+                               }
+                           );
+                       } else {
+                           return next_step(null);
+                       }
+                   },
+                   function addDataToLayer(err) {
+                       if ( !err && !_.isEmpty(metadata) ) {
+                           layer = _.extend({}, layer);
+                           layer.options = _.extend({}, layer.options, { query_rewrite_data: query_rewrite_data });
+                       }
+                       done(err, layer);
+                   }
+               );
            }
       });
   }

lib/cartodb/server.js

@@ -21,6 +21,7 @@ var mapnik = windshaft.mapnik;
 
 var TemplateMaps = require('./backends/template_maps.js');
 var OverviewsMetadataApi = require('./api/overviews_metadata_api');
+var FilterStatsApi = require('./api/filter_stats_api');
 var UserLimitsApi = require('./api/user_limits_api');
 var AuthApi = require('./api/auth_api');
 var LayergroupAffectedTablesCache = require('./cache/layergroup_affected_tables');
@@ -59,6 +60,7 @@ module.exports = function(serverOptions) {
     var pgConnection = new PgConnection(metadataBackend);
     var pgQueryRunner = new PgQueryRunner(pgConnection);
     var overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner);
+    var filterStatsApi = new FilterStatsApi(pgQueryRunner);
     var userLimitsApi = new UserLimitsApi(metadataBackend, {
         limits: {
             cacheOnTimeout: serverOptions.renderer.mapnik.limits.cacheOnTimeout || false,
@@ -148,7 +150,7 @@ module.exports = function(serverOptions) {
     var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache();
     app.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
 
-    var overviewsAdapter = new MapConfigOverviewsAdapter(overviewsMetadataApi);
+    var overviewsAdapter = new MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi);
 
     var turboCartoParser = new TurboCartoParser(pgQueryRunner);
     var turboCartoAdapter = new TurboCartoAdapter(turboCartoParser);
@@ -157,6 +159,7 @@ module.exports = function(serverOptions) {
         templateMaps,
         pgConnection,
         metadataBackend,
+        analysisBackend,
         userLimitsApi,
         overviewsAdapter,
         turboCartoAdapter

lib/cartodb/utils/overviews_query_rewriter.js

@@ -1,5 +1,26 @@
+var _ = require('underscore');
 var TableNameParser = require('./table_name_parser');
 
+var BBoxFilter = require('../models/filter/bbox');
+var CamshaftFilter = require('../models/filter/camshaft');
+
+// Minimim number of filtered rows to use overviews
+var FILTER_MIN_ROWS = 65536;
+// Maximum filtered fraction to not apply overviews
+var FILTER_MAX_FRACTION = 0.2;
+
+function apply_filters_to_query(query, filters, bbox_filter) {
+    if ( filters && !_.isEmpty(filters)) {
+        var camshaftFilter = new CamshaftFilter(filters);
+        query = camshaftFilter.sql(query);
+    }
+    if ( bbox_filter ) {
+        var bboxFilter = new BBoxFilter(bbox_filter.options, bbox_filter.params);
+        query = bboxFilter.sql(query);
+    }
+    return query;
+}
+
 function OverviewsQueryRewriter(options) {
 
     this.options = options;
@@ -119,26 +140,34 @@ function replace_table_in_query(sql, old_table_name, replacement) {
     return sql.replace(new RegExp(regexp, 'g'), replacement);
 }
 
-function overviews_query(query, overviews, zoom_level_expression) {
+
+function replace_table_in_query_with_schema(query, table, schema, replacement) {
+    if ( replacement ) {
+        query = replace_table_in_query(query, table, replacement);
+        var parsed_table = TableNameParser.parse(table);
+        if (!parsed_table.schema && schema) {
+            // replace also the qualified table name, if the table wasn't qualified
+            parsed_table.schema = schema;
+            table = TableNameParser.table_identifier(parsed_table);
+            query = replace_table_in_query(query, table, replacement);
+        }
+    }
+    return query;
+}
+
+// Build query to use overviews for a variant zoom level (given by a expression to
+// be evaluated by the database server)
+function overviews_query_with_zoom_expression(query, overviews, zoom_level_expression) {
     var replaced_query = query;
     var sql = "WITH\n  _vovw_scale AS ( SELECT " + zoom_level_expression + " AS _vovw_z )";
     var replacement;
-    for ( var table in overviews ) {
-        if (overviews.hasOwnProperty(table)) {
-            var table_overviews = overviews[table];
-            var table_view = overviews_view_name(table);
-            var schema = table_overviews.schema;
-            replacement = "(\n" + overviews_view_for_table(table, table_overviews) + "\n  ) AS " + table_view;
-            replaced_query = replace_table_in_query(replaced_query, table, replacement);
-            var parsed_table = TableNameParser.parse(table);
-            if (!parsed_table.schema && schema) {
-                // replace also the qualified table name, if the table wasn't qualified
-                parsed_table.schema = schema;
-                table = TableNameParser.table_identifier(parsed_table);
-                replaced_query = replace_table_in_query(replaced_query, table, replacement);
-            }
-        }
-    }
+    _.each(Object.keys(overviews), function(table) {
+        var table_overviews = overviews[table];
+        var table_view = overviews_view_name(table);
+        var schema = table_overviews.schema;
+        replacement = "(\n" + overviews_view_for_table(table, table_overviews) + "\n  ) AS " + table_view;
+        replaced_query = replace_table_in_query_with_schema(replaced_query, table, schema, replacement);
+    });
     if ( replaced_query !== query ) {
         sql += "\n";
         sql += replaced_query;
@@ -148,34 +177,128 @@ function overviews_query(query, overviews, zoom_level_expression) {
     return sql;
 }
 
+// Build query to use overviews for a specific zoom level value
+function overviews_query_with_definite_zoom(query, overviews, zoom_level) {
+    var replaced_query = query;
+    var replacement;
+    _.each(Object.keys(overviews), function(table) {
+        var table_overviews = overviews[table];
+        var schema = table_overviews.schema;
+        replacement = overview_table_for_zoom_level(table_overviews, zoom_level);
+        replaced_query = replace_table_in_query_with_schema(replaced_query, table, schema, replacement);
+    });
+    return replaced_query;
+}
+
+// Find a suitable overview table for a specific zoom_level
+function overview_table_for_zoom_level(table_overviews, zoom_level) {
+    var overview_table;
+    if ( table_overviews ) {
+        overview_table = table_overviews[zoom_level];
+        if ( !overview_table ) {
+            _.every(Object.keys(table_overviews).sort(function(x,y){ return x-y; }), function(overview_zoom) {
+              if ( +overview_zoom > +zoom_level ) {
+                  overview_table = table_overviews[overview_zoom];
+                  return false;
+              } else {
+                  return true;
+              }
+            });
+        }
+    }
+    if ( overview_table ) {
+        overview_table = overview_table.table;
+    }
+    return overview_table;
+}
+
 // Transform an SQL query so that it uses overviews.
-// overviews contains metadata about the overviews to be used:
-//     { 'table-name': {1: { table: 'overview-table-1' }, ... }, ... }
 //
 // For a given query `SELECT * FROM table`,  if any of tables in it
 // has overviews as defined by the provided metadat, the query will
 // be transform into something similar to this:
 //
 //     WITH _vovw_scale AS ( ... ), -- define scale level
-//     WITH _vovw_table AS ( ... ), -- define union of overviews and base table
-//     SELECT * FROM _vovw_table -- query with table replaced by _vovw_table
+//     SELECT * FROM                -- in the query the table is replaced by:
+//      ( ... ) AS _vovw_table      -- a union of overviews and base table
 //
-// This transformation can in principle be applied to arbitrary queries
-// (except for the case of queries that include the name of tables with
-// overviews inside text literals: at the current table name substitution
-// doesnn't prevent substitution inside literals).
-// But the transformation will currently only be applied to simple queries
-// of the form detected by the overviews_supported_query function.
-OverviewsQueryRewriter.prototype.query = function(query, data) {
-    var overviews = this.overviews_metadata(data);
-    if ( !overviews || !this.is_supported_query(query)) {
+// The data argument has the form:
+//    {
+//        overviews:             // overview tables metadata
+//             { 'table-name': {1: { table: 'overview-table-1' }, ... }, ... },
+//        zoom_level: ...,       // optional zoom level
+//        filters: ...,          // filters definition
+//        unfiltered_query: ..., // query without the filters
+//        bbox_filter: ...       // bounding-box filter
+//    }
+OverviewsQueryRewriter.prototype.query = function(query, data, options) {
+    options = options || {};
+    data    = data || {};
+
+    var overviews        = data.overviews;
+    var unfiltered_query = data.unfiltered_query;
+    var filters          = data.filters;
+    var bbox_filter      = data.bbox_filter;
+
+    if ( !unfiltered_query ) {
+        unfiltered_query = query;
+    }
+
+    if ( !should_use_overviews(unfiltered_query, data) ) {
         return query;
     }
-    var zoom_level_expression = this.options.zoom_level || '0';
-    return overviews_query(query, overviews, zoom_level_expression);
+
+    var rewritten_query;
+
+    var zoom_level_expression = this.options.zoom_level;
+    var zoom_level = zoom_level_for_query(unfiltered_query, zoom_level_expression, options);
+
+    rewritten_query = overviews_query(unfiltered_query, overviews, zoom_level, zoom_level_expression);
+
+    if ( rewritten_query === unfiltered_query ) {
+        // could not or didn't need to alter the query
+        rewritten_query = query;
+    } else {
+        rewritten_query = apply_filters_to_query(rewritten_query, filters, bbox_filter);
+    }
+
+    return rewritten_query;
 };
 
-OverviewsQueryRewriter.prototype.is_supported_query = function(sql) {
+function zoom_level_for_query(query, zoom_level_expression, options) {
+    var zoom_level = null;
+    if ( _.has(options, 'zoom_level') ) {
+        zoom_level = options.zoom_level || '0';
+    }
+    if ( zoom_level === null && !zoom_level_expression ) {
+        zoom_level = '0';
+    }
+    return zoom_level;
+}
+
+function overviews_query(query, overviews, zoom_level, zoom_level_expression) {
+    if ( zoom_level || zoom_level === '0' || zoom_level === 0 ) {
+        return overviews_query_with_definite_zoom(query, overviews, zoom_level);
+    } else {
+        return overviews_query_with_zoom_expression(query, overviews, zoom_level_expression);
+    }
+}
+
+function should_use_overviews(query, data) {
+    data = data || {};
+    var use_overviews = data.overviews && is_supported_query(query);
+    if ( use_overviews && data.filters && data.filter_stats ) {
+        var filtered_rows = data.filter_stats.filtered_rows;
+        var unfiltered_rows = data.filter_stats.unfiltered_rows;
+        if ( unfiltered_rows && (filtered_rows || filtered_rows === 0) ) {
+            use_overviews = filtered_rows >= FILTER_MIN_ROWS ||
+                            (filtered_rows/unfiltered_rows) > FILTER_MAX_FRACTION;
+        }
+    }
+    return use_overviews;
+}
+
+function is_supported_query(sql) {
     var basic_query =
         /\s*SELECT\s+[\*a-z0-9_,\s]+?\s+FROM\s+((\"[^"]+\"|[a-z0-9_]+)\.)?(\"[^"]+\"|[a-z0-9_]+)\s*;?\s*/i;
     var unwrapped_query = new RegExp("^"+basic_query.source+"$", 'i');
@@ -187,8 +310,4 @@ OverviewsQueryRewriter.prototype.is_supported_query = function(sql) {
         'i'
     );
     return !!(sql.match(unwrapped_query) || sql.match(wrapped_query));
-};
-
-OverviewsQueryRewriter.prototype.overviews_metadata = function(data) {
-    return data && data.overviews;
-};
+}

lib/cartodb/utils/style/postgres-datasource.js

@@ -23,6 +23,10 @@ var methodTemplates = Object.keys(methods).reduce(function(methodTemplates, meth
     return methodTemplates;
 }, {});
 
+var method2strategy = {
+    headtails: 'split'
+};
+
 function PostgresDatasource (pgQueryRunner, username, query) {
     this.pgQueryRunner = pgQueryRunner;
     this.username = username;
@@ -48,7 +52,7 @@ PostgresDatasource.prototype.getRamp = function (column, buckets, method, callba
             return a - b;
         });
 
-        return callback(null, ramp);
+        return callback(null, { ramp: ramp, strategy: method2strategy[methodName] });
     });
 };
 

lib/cartodb/utils/style/turbo-carto-adapter.js

@@ -1,6 +1,7 @@
 'use strict';
 
 var queue = require('queue-async');
+var SubstitutionTokens = require('../substitution-tokens');
 
 function TurboCartoAdapter(turboCartoParser) {
     this.turboCartoParser = turboCartoParser;
@@ -37,13 +38,26 @@ TurboCartoAdapter.prototype._parseCartoCss = function (username, layer, callback
         });
     }
 
-    this.turboCartoParser.process(username, layer.options.cartocss, layer.options.sql, function (err, cartocss) {
-        // Ignore turbo-carto errors and continue
-        if (!err && cartocss) {
-            layer.options.cartocss = cartocss;
+    var sql = SubstitutionTokens.replace(layer.options.sql, {
+        bbox: 'ST_MakeEnvelope(-20037508.34,-20037508.34,20037508.34,20037508.34,3857)',
+        scale_denominator: '500000001',
+        pixel_width: '156412',
+        pixel_height: '156412'
+    });
+
+    this.turboCartoParser.process(username, layer.options.cartocss, sql, function (err, cartocss) {
+        // Only return turbo-carto errors
+        if (err && err.name === 'TurboCartoError') {
+            err = new Error('turbo-carto: ' + err.message);
+            err.http_status = 400;
+            return callback(err);
         }
 
-        callback(null, layer);
+        // Try to continue in the rest of the cases
+        if (cartocss) {
+            layer.options.cartocss = cartocss;
+        }
+        return callback(null, layer);
     });
 };
 

lib/cartodb/utils/substitution-tokens.js

@@ -0,0 +1,19 @@
+var SUBSTITUTION_TOKENS = {
+    bbox: /!bbox!/g,
+    scale_denominator: /!scale_denominator!/g,
+    pixel_width: /!pixel_width!/g,
+    pixel_height: /!pixel_height!/g
+};
+
+var SubstitutionTokens = {
+    replace: function(sql, replaceValues) {
+        Object.keys(replaceValues).forEach(function(token) {
+            if (SUBSTITUTION_TOKENS[token]) {
+                sql = sql.replace(SUBSTITUTION_TOKENS[token], replaceValues[token]);
+            }
+        });
+        return sql;
+    }
+};
+
+module.exports = SubstitutionTokens;

package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "name": "windshaft-cartodb",
-  "version": "2.41.0",
+  "version": "2.43.0",
   "description": "A map tile server for CartoDB",
   "keywords": [
     "cartodb"
@@ -20,7 +20,7 @@
   ],
   "dependencies": {
     "body-parser": "~1.14.0",
-    "camshaft": "0.7.0",
+    "camshaft": "0.8.0",
     "cartodb-psql": "~0.6.1",
     "cartodb-query-tables": "~0.1.0",
     "cartodb-redis": "~0.13.0",
@@ -37,7 +37,7 @@
     "request": "~2.62.0",
     "step": "~0.0.6",
     "step-profiler": "~0.3.0",
-    "turbo-carto": "0.7.1",
+    "turbo-carto": "0.9.2",
     "underscore": "~1.6.0",
     "windshaft": "1.19.0"
   },

test/acceptance/analysis/named-maps.js

@@ -1,11 +1,11 @@
 var assert = require('../../support/assert');
-var step = require('step');
 
 var helper = require('../../support/test_helper');
 
 var CartodbWindshaft = require('../../../lib/cartodb/server');
 var serverOptions = require('../../../lib/cartodb/server_options');
 var server = new CartodbWindshaft(serverOptions);
+var TestClient = require('../../support/test-client');
 
 var LayergroupToken = require('../../../lib/cartodb/models/layergroup_token');
 
@@ -16,230 +16,259 @@ describe('named-maps analysis', function() {
     var username = 'localhost';
     var widgetsTemplateName = 'widgets-template';
 
-    var layergroupid;
-    var layergroup;
-    var keysToDelete;
-
-    beforeEach(function(done) {
-        keysToDelete = {};
-
-        var widgetsTemplate =  {
-            version: '0.0.1',
-            name: widgetsTemplateName,
-            layergroup:  {
-                version: '1.5.0',
-                layers: [
-                    {
-                        "type": "cartodb",
-                        "options": {
-                            "source": {
-                                "id": "HEAD"
-                            },
-                            "cartocss": '#buffer { polygon-fill: red; }',
-                            "cartocss_version": "2.3.0"
-                        }
-                    }
-                ],
-                dataviews: {
-                    pop_max_histogram: {
-                        source: {
-                            id: 'HEAD'
+    var widgetsTemplate =  {
+        version: '0.0.1',
+        name: widgetsTemplateName,
+        layergroup:  {
+            version: '1.5.0',
+            layers: [
+                {
+                    "type": "cartodb",
+                    "options": {
+                        "source": {
+                            "id": "HEAD"
                         },
-                        type: 'histogram',
-                        options: {
-                            column: 'pop_max'
-                        }
+                        "cartocss": '#buffer { polygon-fill: red; }',
+                        "cartocss_version": "2.3.0"
                     }
-                },
-                analyses: [
-                    {
-                        "id": "HEAD",
-                        "type": "buffer",
-                        "params": {
-                            "source": {
-                                "id": "2570e105-7b37-40d2-bdf4-1af889598745",
-                                "type": "source",
-                                "params": {
-                                    "query": "select * from populated_places_simple_reduced"
-                                }
-                            },
-                            "radius": 50000
-                        }
+                }
+            ],
+            dataviews: {
+                pop_max_histogram: {
+                    source: {
+                        id: 'HEAD'
+                    },
+                    type: 'histogram',
+                    options: {
+                        column: 'pop_max'
                     }
-                ]
-            }
-        };
-
-        var template_params = {};
-
-        step(
-            function createTemplate()
-            {
-                var next = this;
-                assert.response(
-                    server,
-                    {
-                        url: '/api/v1/map/named?api_key=1234',
-                        method: 'POST',
-                        headers: {
-                            host: username,
-                            'Content-Type': 'application/json'
+                }
+            },
+            analyses: [
+                {
+                    "id": "HEAD",
+                    "type": "buffer",
+                    "params": {
+                        "source": {
+                            "id": "2570e105-7b37-40d2-bdf4-1af889598745",
+                            "type": "source",
+                            "params": {
+                                "query": "select * from populated_places_simple_reduced"
+                            }
                         },
-                        data: JSON.stringify(widgetsTemplate)
-                    },
-                    {
-                        status: 200
-                    },
-                    function(res, err) {
-                        next(err, res);
+                        "radius": 50000
                     }
-                );
-            },
-            function instantiateTemplate(err, res) {
-                assert.ifError(err);
+                }
+            ]
+        }
+    };
 
-                assert.deepEqual(JSON.parse(res.body), { template_id: widgetsTemplateName });
-                var next = this;
-                assert.response(
-                    server,
-                    {
-                        url: '/api/v1/map/named/' + widgetsTemplateName,
-                        method: 'POST',
-                        headers: {
-                            host: username,
-                            'Content-Type': 'application/json'
-                        },
-                        data: JSON.stringify(template_params)
-                    },
-                    {
-                        status: 200
-                    },
-                    function(res) {
-                        next(null, res);
-                    }
-                );
-            },
-            function finish(err, res) {
-                assert.ifError(err);
-
-                layergroup = JSON.parse(res.body);
-                assert.ok(layergroup.hasOwnProperty('layergroupid'), "Missing 'layergroupid' from: " + res.body);
-                layergroupid = layergroup.layergroupid;
-
-                assert.ok(
-                    Array.isArray(layergroup.metadata.analyses),
-                    'Missing "analyses" array metadata from: ' + res.body
-                );
-                var analyses = layergroup.metadata.analyses;
-                assert.equal(analyses.length, 1, 'Invalid number of analyses in metadata');
-                var nodes = analyses[0].nodes;
-                var nodesIds = Object.keys(nodes);
-                assert.deepEqual(nodesIds, ['2570e105-7b37-40d2-bdf4-1af889598745', 'HEAD']);
-                nodesIds.forEach(function(nodeId) {
-                    var node = nodes[nodeId];
-                    assert.ok(node.hasOwnProperty('url'), 'Missing "url" attribute in node');
-                    assert.ok(node.hasOwnProperty('status'), 'Missing "status" attribute in node');
-                    assert.ok(!node.hasOwnProperty('query'), 'Unexpected "query" attribute in node');
-                });
-
-                keysToDelete['map_cfg|' + LayergroupToken.parse(layergroup.layergroupid).token] = 0;
-                keysToDelete['user:localhost:mapviews:global'] = 5;
-
-                return done();
-            }
-        );
-
-    });
-
-    afterEach(function(done) {
-        step(
-            function deleteTemplate(err) {
-                assert.ifError(err);
-                var next = this;
-                assert.response(
-                    server,
-                    {
-                        url: '/api/v1/map/named/' + widgetsTemplateName + '?api_key=1234',
-                        method: 'DELETE',
-                        headers: {
-                            host: username
-                        }
-                    },
-                    {
-                        status: 204
-                    },
-                    function(res, err) {
-                        next(err, res);
-                    }
-                );
-            },
-            function deleteRedisKeys(err) {
-                assert.ifError(err);
-                helper.deleteRedisKeys(keysToDelete, done);
-            }
-        );
-    });
-
-    it('should be able to retrieve images from analysis', function(done) {
+    beforeEach(function createTemplate(done) {
         assert.response(
             server,
             {
-                url: '/api/v1/map/' + layergroupid + '/6/31/24.png',
-                method: 'GET',
-                encoding: 'binary',
+                url: '/api/v1/map/named?api_key=1234',
+                method: 'POST',
+                headers: {
+                    host: username,
+                    'Content-Type': 'application/json'
+                },
+                data: JSON.stringify(widgetsTemplate)
+            },
+            {
+                status: 200
+            },
+            function(res, err) {
+                assert.deepEqual(JSON.parse(res.body), { template_id: widgetsTemplateName });
+                return done(err);
+            }
+        );
+    });
+
+    afterEach(function deleteTemplate(done) {
+        assert.response(
+            server,
+            {
+                url: '/api/v1/map/named/' + widgetsTemplateName + '?api_key=1234',
+                method: 'DELETE',
                 headers: {
                     host: username
                 }
             },
             {
-                status: 200,
-                headers: {
-                    'Content-Type': 'image/png'
-                }
+                status: 204
             },
             function(res, err) {
-                if (err) {
-                    return done(err);
-                }
+                return done(err);
+            }
+        );
+    });
 
-                var fixturePath = './test/fixtures/analysis/named-map-buffer.png';
-                assert.imageBufferIsSimilarToFile(res.body, fixturePath, IMAGE_TOLERANCE_PER_MIL, function(err) {
+    describe('layergroup', function() {
+        var layergroupid;
+        var layergroup;
+        var keysToDelete;
+
+        beforeEach(function(done) {
+            keysToDelete = {};
+
+            assert.response(
+                server,
+                {
+                    url: '/api/v1/map/named/' + widgetsTemplateName,
+                    method: 'POST',
+                    headers: {
+                        host: username,
+                        'Content-Type': 'application/json'
+                    },
+                    data: JSON.stringify({})
+                },
+                {
+                    status: 200
+                },
+                function(res, err) {
+                    assert.ifError(err);
+
+                    layergroup = JSON.parse(res.body);
+                    assert.ok(layergroup.hasOwnProperty('layergroupid'), "Missing 'layergroupid' from: " + res.body);
+                    layergroupid = layergroup.layergroupid;
+
+                    assert.ok(
+                        Array.isArray(layergroup.metadata.analyses),
+                            'Missing "analyses" array metadata from: ' + res.body
+                    );
+                    var analyses = layergroup.metadata.analyses;
+                    assert.equal(analyses.length, 1, 'Invalid number of analyses in metadata');
+                    var nodes = analyses[0].nodes;
+                    var nodesIds = Object.keys(nodes);
+                    assert.deepEqual(nodesIds, ['2570e105-7b37-40d2-bdf4-1af889598745', 'HEAD']);
+                    nodesIds.forEach(function(nodeId) {
+                        var node = nodes[nodeId];
+                        assert.ok(node.hasOwnProperty('url'), 'Missing "url" attribute in node');
+                        assert.ok(node.hasOwnProperty('status'), 'Missing "status" attribute in node');
+                        assert.ok(!node.hasOwnProperty('query'), 'Unexpected "query" attribute in node');
+                    });
+
+                    keysToDelete['map_cfg|' + LayergroupToken.parse(layergroup.layergroupid).token] = 0;
+                    keysToDelete['user:localhost:mapviews:global'] = 5;
+
+                    return done();
+                }
+            );
+
+        });
+
+        afterEach(function(done) {
+            helper.deleteRedisKeys(keysToDelete, done);
+        });
+
+        it('should be able to retrieve images from analysis', function(done) {
+            assert.response(
+                server,
+                {
+                    url: '/api/v1/map/' + layergroupid + '/6/31/24.png',
+                    method: 'GET',
+                    encoding: 'binary',
+                    headers: {
+                        host: username
+                    }
+                },
+                {
+                    status: 200,
+                    headers: {
+                        'Content-Type': 'image/png'
+                    }
+                },
+                function(res, err) {
+                    if (err) {
+                        return done(err);
+                    }
+
+                    var fixturePath = './test/fixtures/analysis/named-map-buffer.png';
+                    assert.imageBufferIsSimilarToFile(res.body, fixturePath, IMAGE_TOLERANCE_PER_MIL, function(err) {
+                        assert.ok(!err, err);
+                        done();
+                    });
+
+                }
+            );
+        });
+
+        it('should be able to retrieve dataviews from analysis', function(done) {
+            assert.response(
+                server,
+                {
+                    url: '/api/v1/map/' + layergroupid + '/dataview/pop_max_histogram',
+                    method: 'GET',
+                    headers: {
+                        host: username
+                    }
+                },
+                {
+                    status: 200,
+                    headers: {
+                        'Content-Type': 'application/json; charset=utf-8'
+                    }
+                },
+                function(res, err) {
+                    if (err) {
+                        return done(err);
+                    }
+
+                    var dataview = JSON.parse(res.body);
+                    assert.equal(dataview.type, 'histogram');
+                    assert.equal(dataview.bins_start, 0);
+
+                    done();
+                }
+            );
+        });
+
+        it('should be able to retrieve static map preview via layergroup', function(done) {
+            assert.response(
+                server,
+                {
+                    url: '/api/v1/map/static/center/' + layergroupid + '/4/42/-3/320/240.png',
+                    method: 'GET',
+                    encoding: 'binary',
+                    headers: {
+                        host: username
+                    }
+                },
+                {
+                    status: 200,
+                    headers: {
+                        'Content-Type': 'image/png'
+                    }
+                },
+                function(res, err) {
+                    if (err) {
+                        return done(err);
+                    }
+
+                    var fixturePath = './test/fixtures/analysis/named-map-buffer-layergroup-static-preview.png';
+                    assert.imageBufferIsSimilarToFile(res.body, fixturePath, IMAGE_TOLERANCE_PER_MIL, function(err) {
+                        assert.ok(!err, err);
+                        done();
+                    });
+
+                }
+            );
+        });
+
+    });
+
+    describe('auto-instantiation', function() {
+        it('should be able to retrieve static map preview via fixed url', function(done) {
+            TestClient.getStaticMap(widgetsTemplateName, function(err, image) {
+                assert.ok(!err, err);
+                var fixturePath = './test/fixtures/analysis/named-map-buffer-static-preview.png';
+                assert.imageIsSimilarToFile(image, fixturePath, IMAGE_TOLERANCE_PER_MIL, function(err) {
                     assert.ok(!err, err);
                     done();
                 });
-
-            }
-        );
-    });
-
-    it('should be able to retrieve dataviews from analysis', function(done) {
-        assert.response(
-            server,
-            {
-                url: '/api/v1/map/' + layergroupid + '/dataview/pop_max_histogram',
-                method: 'GET',
-                headers: {
-                    host: username
-                }
-            },
-            {
-                status: 200,
-                headers: {
-                    'Content-Type': 'application/json; charset=utf-8'
-                }
-            },
-            function(res, err) {
-                if (err) {
-                    return done(err);
-                }
-
-                var dataview = JSON.parse(res.body);
-                assert.equal(dataview.type, 'histogram');
-                assert.equal(dataview.bins_start, 0);
-
-                done();
-            }
-        );
+            });
+        });
     });
 
 });

test/acceptance/dataviews/overviews.js

@@ -0,0 +1,292 @@
+require('../../support/test_helper');
+
+var assert = require('../../support/assert');
+var TestClient = require('../../support/test-client');
+
+describe('dataviews using tables without overviews', function() {
+
+    var nonOverviewsMapConfig =  {
+        version: '1.5.0',
+        analyses: [
+            { id: 'data-source',
+                type: 'source',
+                params: {
+                  query: 'select * from populated_places_simple_reduced'
+                }
+            }
+        ],
+        dataviews:  {
+            country_places_count: {
+                type: 'formula',
+                source: {id: 'data-source'},
+                options: {
+                    column: 'adm0_a3',
+                    operation: 'count'
+                }
+            },
+            country_categories: {
+                type: 'aggregation',
+                source: {id: 'data-source'},
+                options: {
+                    column: 'adm0_a3',
+                    aggregation: 'count'
+                }
+            }
+        },
+        layers: [
+            {
+                type: 'mapnik',
+                options: {
+                    sql: 'select * from populated_places_simple_reduced',
+                    cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
+                    cartocss_version: '2.3.0',
+                    source: { id: 'data-source' }
+                }
+            }
+        ]
+    };
+
+    it("should expose a formula", function(done) {
+        var testClient = new TestClient(nonOverviewsMapConfig);
+        testClient.getDataview('country_places_count', { own_filter: 0 }, function(err, formula_result) {
+            if (err) {
+                return done(err);
+            }
+            assert.deepEqual(formula_result, { operation: 'count', result: 7313, nulls: 0, type: 'formula' });
+
+            testClient.drain(done);
+        });
+    });
+
+    describe('filters', function() {
+
+        describe('category', function () {
+
+            it("should expose a filtered formula", function (done) {
+                var params = {
+                    filters: {
+                        dataviews: {country_categories: {accept: ['CAN']}}
+                    }
+                };
+                var testClient = new TestClient(nonOverviewsMapConfig);
+                testClient.getDataview('country_places_count', params, function (err, formula_result) {
+                    if (err) {
+                        return done(err);
+                    }
+                    assert.deepEqual(formula_result, { operation: 'count', result: 256, nulls: 0, type: 'formula' });
+                    testClient.drain(done);
+                });
+            });
+        });
+
+    });
+});
+
+describe('dataviews using tables with overviews', function() {
+
+    var overviewsMapConfig =  {
+        version: '1.5.0',
+        analyses: [
+            { id: 'data-source',
+                type: 'source',
+                params: {
+                  query: 'select * from test_table_overviews'
+                }
+            }
+        ],
+        dataviews:  {
+            test_sum: {
+                type: 'formula',
+                source: {id: 'data-source'},
+                options: {
+                    column: 'value',
+                    operation: 'sum'
+                }
+            },
+            test_categories: {
+                type: 'aggregation',
+                source: {id: 'data-source'},
+                options: {
+                    column: 'name',
+                    aggregation: 'count',
+                    aggregationColumn: 'name',
+                }
+            },
+            test_avg: {
+                type: 'formula',
+                source: {id: 'data-source'},
+                options: {
+                    column: 'value',
+                    operation: 'avg'
+                }
+            },
+            test_count: {
+                type: 'formula',
+                source: {id: 'data-source'},
+                options: {
+                    column: 'value',
+                    operation: 'count'
+                }
+            },
+            test_min: {
+                type: 'formula',
+                source: {id: 'data-source'},
+                options: {
+                    column: 'value',
+                    operation: 'min'
+                }
+            },
+            test_max: {
+                type: 'formula',
+                source: {id: 'data-source'},
+                options: {
+                    column: 'value',
+                    operation: 'max'
+                }
+            }
+        },
+        layers: [
+            {
+                type: 'mapnik',
+                options: {
+                    sql: 'select * from test_table_overviews',
+                    cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
+                    cartocss_version: '2.3.0',
+                    source: { id: 'data-source' }
+                }
+            }
+        ]
+    };
+
+    it("should expose a sum formula", function(done) {
+        var testClient = new TestClient(overviewsMapConfig);
+        testClient.getDataview('test_sum', { own_filter: 0 }, function(err, formula_result) {
+            if (err) {
+                return done(err);
+            }
+            assert.deepEqual(formula_result, {"operation":"sum","result":15,"nulls":0,"type":"formula"});
+
+            testClient.drain(done);
+        });
+    });
+
+    it("should expose an avg formula", function(done) {
+        var testClient = new TestClient(overviewsMapConfig);
+        testClient.getDataview('test_avg', { own_filter: 0 }, function(err, formula_result) {
+            if (err) {
+                return done(err);
+            }
+            assert.deepEqual(formula_result, {"operation":"avg","result":3,"nulls":0,"type":"formula"});
+
+            testClient.drain(done);
+        });
+    });
+
+    it("should expose a count formula", function(done) {
+        var testClient = new TestClient(overviewsMapConfig);
+        testClient.getDataview('test_count', { own_filter: 0 }, function(err, formula_result) {
+            if (err) {
+                return done(err);
+            }
+            assert.deepEqual(formula_result, {"operation":"count","result":5,"nulls":0,"type":"formula"});
+
+            testClient.drain(done);
+        });
+    });
+
+    it("should expose a max formula", function(done) {
+        var testClient = new TestClient(overviewsMapConfig);
+        testClient.getDataview('test_max', { own_filter: 0 }, function(err, formula_result) {
+            if (err) {
+                return done(err);
+            }
+            assert.deepEqual(formula_result, {"operation":"max","result":5,"nulls":0,"type":"formula"});
+
+            testClient.drain(done);
+        });
+    });
+
+    it("should expose a min formula", function(done) {
+        var testClient = new TestClient(overviewsMapConfig);
+        testClient.getDataview('test_min', { own_filter: 0 }, function(err, formula_result) {
+            if (err) {
+                return done(err);
+            }
+            assert.deepEqual(formula_result, {"operation":"min","result":1,"nulls":0,"type":"formula"});
+
+            testClient.drain(done);
+        });
+    });
+
+    describe('filters', function() {
+
+        describe('category', function () {
+
+            it("should expose a filtered formula", function (done) {
+                var params = {
+                    filters: {
+                        dataviews: {test_categories: {accept: ['Hawai']}}
+                    }
+                };
+                var testClient = new TestClient(overviewsMapConfig);
+                testClient.getDataview('test_sum', params, function (err, formula_result) {
+                    if (err) {
+                        return done(err);
+                    }
+                    assert.deepEqual(formula_result, {"operation":"sum","result":1,"nulls":0,"type":"formula"});
+                    testClient.drain(done);
+                });
+
+                it("should expose an avg formula", function(done) {
+                    var testClient = new TestClient(overviewsMapConfig);
+                    testClient.getDataview('test_avg', { own_filter: 0 }, function(err, formula_result) {
+                        if (err) {
+                            return done(err);
+                        }
+                        assert.deepEqual(formula_result, {"operation":"avg","result":1,"nulls":0,"type":"formula"});
+
+                        testClient.drain(done);
+                    });
+                });
+
+                it("should expose a count formula", function(done) {
+                    var testClient = new TestClient(overviewsMapConfig);
+                    testClient.getDataview('test_count', { own_filter: 0 }, function(err, formula_result) {
+                        if (err) {
+                            return done(err);
+                        }
+                        assert.deepEqual(formula_result, {"operation":"count","result":1,"nulls":0,"type":"formula"});
+
+                        testClient.drain(done);
+                    });
+                });
+
+                it("should expose a max formula", function(done) {
+                    var testClient = new TestClient(overviewsMapConfig);
+                    testClient.getDataview('test_max', { own_filter: 0 }, function(err, formula_result) {
+                        if (err) {
+                            return done(err);
+                        }
+                        assert.deepEqual(formula_result, {"operation":"max","result":1,"nulls":0,"type":"formula"});
+
+                        testClient.drain(done);
+                    });
+                });
+
+                it("should expose a min formula", function(done) {
+                    var testClient = new TestClient(overviewsMapConfig);
+                    testClient.getDataview('test_min', { own_filter: 0 }, function(err, formula_result) {
+                        if (err) {
+                            return done(err);
+                        }
+                        assert.deepEqual(formula_result, {"operation":"min","result":1,"nulls":0,"type":"formula"});
+
+                        testClient.drain(done);
+                    });
+                });
+
+            });
+        });
+
+    });
+});

test/acceptance/overviews_metadata.js

@@ -109,3 +109,118 @@ describe('overviews metadata', function() {
         );
     });
 });
+
+describe('overviews metadata with filters', function() {
+    // configure redis pool instance to use in tests
+    var redisPool = new RedisPool(global.environment.redis);
+
+    var keysToDelete;
+
+    beforeEach(function() {
+        keysToDelete = {};
+    });
+
+    afterEach(function(done) {
+        test_helper.deleteRedisKeys(keysToDelete, done);
+    });
+
+    it("layers with overviews", function(done) {
+
+        var layergroup =  {
+            version: '1.5.0',
+            layers: [
+              {
+                  type: 'cartodb',
+                  options: {
+                      sql: 'SELECT * FROM test_table_overviews',
+                      source: { id: 'with_overviews' },
+                      cartocss: '#layer { marker-fill: black; }',
+                      cartocss_version: '2.3.0'
+                  }
+              }
+            ],
+            dataviews:  {
+                test_names: {
+                    type: 'aggregation',
+                    source: {id: 'with_overviews'},
+                    options: {
+                        column: 'name',
+                        aggregation: 'count'
+                    }
+                }
+            },
+            analyses: [
+                { id: 'with_overviews',
+                    type: 'source',
+                    params: {
+                      query: 'select * from test_table_overviews'
+                    }
+                }
+            ]
+        };
+
+        var filters = {
+            dataviews: {
+                test_names: { accept: ['Hawai'] }
+            }
+        };
+
+        var layergroup_url = '/api/v1/map';
+
+        var expected_token;
+        step(
+            function do_post()
+            {
+              var next = this;
+              assert.response(server, {
+                  url: layergroup_url + '?filters=' + JSON.stringify(filters),
+                  method: 'POST',
+                  headers: {host: 'localhost', 'Content-Type': 'application/json' },
+                  data: JSON.stringify(layergroup)
+              }, {}, function(res) {
+                  assert.equal(res.statusCode, 200, res.body);
+                  var parsedBody = JSON.parse(res.body);
+                  assert.equal(res.headers['x-layergroup-id'], parsedBody.layergroupid);
+                  expected_token = parsedBody.layergroupid;
+                  next(null, res);
+              });
+            },
+            function do_get_mapconfig(err)
+            {
+                assert.ifError(err);
+                var next = this;
+
+                var mapStore  = new windshaft.storage.MapStore({
+                    pool: redisPool,
+                    expire_time: 500000
+                });
+                mapStore.load(LayergroupToken.parse(expected_token).token, function(err, mapConfig) {
+                    assert.ifError(err);
+                    assert.equal(mapConfig._cfg.layers[0].type, 'cartodb');
+                    assert.ok(mapConfig._cfg.layers[0].options.query_rewrite_data);
+                    var expected_data = {
+                        overviews: {
+                            test_table_overviews: {
+                                schema: 'public',
+                                1: { table: '_vovw_1_test_table_overviews' },
+                                2: { table: '_vovw_2_test_table_overviews' }
+                            }
+                        },
+                        filters: { test_names: { type: 'category', column: 'name', params: { accept: [ 'Hawai' ] } } },
+                        unfiltered_query: 'select * from test_table_overviews',
+                        filter_stats: { unfiltered_rows: 5, filtered_rows: 1 }
+                      };
+                    assert.deepEqual(mapConfig._cfg.layers[0].options.query_rewrite_data, expected_data);
+
+                  });
+
+                  next(err);
+            },
+            function finish(err) {
+                keysToDelete['map_cfg|' + LayergroupToken.parse(expected_token).token] = 0;
+                keysToDelete['user:localhost:mapviews:global'] = 5;
+                done(err);
+            }
+        );
+    });
+});

test/acceptance/regressions.js

@@ -0,0 +1,40 @@
+require('../support/test_helper');
+
+var assert = require('../support/assert');
+var TestClient = require('../support/test-client');
+
+describe('regressions', function() {
+
+    var ERROR_RESPONSE = {
+        status: 400,
+        headers: {
+            'Content-Type': 'application/json; charset=utf-8'
+        }
+    };
+
+    it('should expose a nice error when missing sql option', function(done) {
+        var mapConfig = {
+            version: '1.5.0',
+            layers: [
+                {
+                    "type": "cartodb",
+                    "options": {
+                        "cartocss": '#polygons { polygon-fill: red; }',
+                        "cartocss_version": "2.3.0"
+                    }
+                }
+            ]
+        };
+
+        var testClient = new TestClient(mapConfig, 1234);
+
+        testClient.getLayergroup(ERROR_RESPONSE, function(err, layergroupResult) {
+            assert.ok(!err, err);
+
+            assert.equal(layergroupResult.errors.length, 1);
+            assert.equal(layergroupResult.errors[0], 'Missing sql for layer 0 options');
+
+            testClient.drain(done);
+        });
+    });
+});

test/acceptance/turbo-cartocss/error-cases.js

@@ -0,0 +1,94 @@
+require('../../support/test_helper');
+
+var assert = require('../../support/assert');
+var TestClient = require('../../support/test-client');
+
+function makeMapconfig(markerWidth, markerFill) {
+    return {
+        "version": "1.4.0",
+        "layers": [
+            {
+                "type": 'mapnik',
+                "options": {
+                    "cartocss_version": '2.3.0',
+                    "sql": 'SELECT * FROM populated_places_simple_reduced',
+                    "cartocss": createCartocss(markerWidth, markerFill)
+                }
+            }
+        ]
+    };
+}
+
+function createCartocss(markerWidth, markerFill) {
+    return [
+        "#populated_places_simple_reduced {",
+        "  marker-fill-opacity: 0.9;",
+        "  marker-line-color: #FFF;",
+        "  marker-line-width: 1;",
+        "  marker-line-opacity: 1;",
+        "  marker-placement: point;",
+        "  marker-type: ellipse;",
+        "  marker-allow-overlap: true;",
+        "  marker-width: " + (markerWidth || '10') + ";",
+        "  marker-fill: " + (markerFill || 'red') + ";",
+        "}"
+    ].join('\n');
+}
+
+var ERROR_RESPONSE = {
+    status: 400,
+    headers: {
+        'Content-Type': 'application/json; charset=utf-8'
+    }
+};
+
+describe('turbo-carto error cases', function() {
+    afterEach(function (done) {
+        if (this.testClient) {
+            this.testClient.drain(done);
+        }
+    });
+
+    it('should return invalid number of ramp error', function(done) {
+        this.testClient = new TestClient(makeMapconfig('ramp([pop_max], (8,24,96), (8,24,96,128))'));
+        this.testClient.getLayergroup(ERROR_RESPONSE, function(err, layergroup) {
+            assert.ok(!err, err);
+
+            assert.ok(layergroup.hasOwnProperty('errors'));
+            assert.equal(layergroup.errors.length, 1);
+            assert.ok(layergroup.errors[0].match(/^turbo-carto/));
+            assert.ok(layergroup.errors[0].match(/invalid\sramp\slength/i));
+
+            done();
+        });
+    });
+
+    it('should return invalid column from datasource', function(done) {
+        this.testClient = new TestClient(makeMapconfig(null, 'ramp([wadus_column], (red, green, blue))'));
+        this.testClient.getLayergroup(ERROR_RESPONSE, function(err, layergroup) {
+            assert.ok(!err, err);
+
+            assert.ok(layergroup.hasOwnProperty('errors'));
+            assert.equal(layergroup.errors.length, 1);
+            assert.ok(layergroup.errors[0].match(/^turbo-carto/));
+            assert.ok(layergroup.errors[0].match(/unable\sto\scompute\sramp/i));
+            assert.ok(layergroup.errors[0].match(/wadus_column/));
+
+            done();
+        });
+    });
+
+    it('should fail by falling back to normal carto parser', function(done) {
+        this.testClient = new TestClient(makeMapconfig('ramp([price], (8,24,96), (8,24,96));//(red, green, blue))'));
+        this.testClient.getLayergroup(ERROR_RESPONSE, function(err, layergroup) {
+            assert.ok(!err, err);
+
+            assert.ok(layergroup.hasOwnProperty('errors'));
+            assert.equal(layergroup.errors.length, 1);
+            assert.ok(!layergroup.errors[0].match(/^turbo-carto/));
+            assert.ok(layergroup.errors[0].match(/invalid\scode/i));
+
+            done();
+        });
+    });
+});

test/acceptance/turbo-cartocss/regressions.js

@@ -3,7 +3,7 @@ require('../../support/test_helper');
 var assert = require('../../support/assert');
 var TestClient = require('../../support/test-client');
 
-function makeMapconfig(cartocss) {
+function makeMapconfig(sql, cartocss) {
     return {
         "version": "1.4.0",
         "layers": [
@@ -11,19 +11,7 @@ function makeMapconfig(cartocss) {
                 "type": 'mapnik',
                 "options": {
                     "cartocss_version": '2.3.0',
-                    "sql": [
-                        'SELECT test_table.*, _prices.price FROM test_table JOIN (' +
-                        '  SELECT 1 AS cartodb_id, 10.00 AS price',
-                        '  UNION',
-                        '  SELECT 2, 10.50',
-                        '  UNION',
-                        '  SELECT 3, 11.00',
-                        '  UNION',
-                        '  SELECT 4, 12.00',
-                        '  UNION',
-                        '  SELECT 5, 21.00',
-                        ') _prices ON _prices.cartodb_id = test_table.cartodb_id'
-                    ].join('\n'),
+                    "sql": sql,
                     "cartocss": cartocss
                 }
             }
@@ -33,42 +21,84 @@ function makeMapconfig(cartocss) {
 
 describe('turbo-carto regressions', function() {
 
-    var cartocss = [
-        "/** simple visualization */",
-        "",
-        "Map {",
-        "  buffer-size: 256;",
-        "}",
-        "",
-        "#county_points_with_population{",
-        "  marker-fill-opacity: 0.1;",
-        "  marker-line-color:#FFFFFF;//#CF1C90;",
-        "  marker-line-width: 0;",
-        "  marker-line-opacity: 0.3;",
-        "  marker-placement: point;",
-        "  marker-type: ellipse;",
-        "  //marker-comp-op: overlay;",
-        "  marker-width: [price];",
-        "  [zoom=5]{marker-width: [price]*2;}",
-        "  [zoom=6]{marker-width: [price]*4;}",
-        "  marker-fill: #000000;",
-        "  marker-allow-overlap: true;",
-        "  ",
-        "",
-        "}"
-    ].join('\n');
-
-    beforeEach(function () {
-        this.testClient = new TestClient(makeMapconfig(cartocss));
-    });
-
     afterEach(function (done) {
-        this.testClient.drain(done);
+        if (this.testClient) {
+            this.testClient.drain(done);
+        }
     });
 
     it('should accept // comments', function(done) {
-        this.testClient.getTile(0, 0, 0, function(err) {
+        var cartocss = [
+            "/** simple visualization */",
+            "",
+            "Map {",
+            "  buffer-size: 256;",
+            "}",
+            "",
+            "#county_points_with_population{",
+            "  marker-fill-opacity: 0.1;",
+            "  marker-line-color:#FFFFFF;//#CF1C90;",
+            "  marker-line-width: 0;",
+            "  marker-line-opacity: 0.3;",
+            "  marker-placement: point;",
+            "  marker-type: ellipse;",
+            "  //marker-comp-op: overlay;",
+            "  marker-width: [cartodb_id];",
+            "  [zoom=5]{marker-width: [cartodb_id]*2;}",
+            "  [zoom=6]{marker-width: [cartodb_id]*4;}",
+            "  marker-fill: #000000;",
+            "  marker-allow-overlap: true;",
+            "  ",
+            "",
+            "}"
+        ].join('\n');
+
+        this.testClient = new TestClient(makeMapconfig('SELECT * FROM populated_places_simple_reduced', cartocss));
+        this.testClient.getLayergroup(function(err, layergroup) {
             assert.ok(!err, err);
+
+            assert.ok(layergroup.hasOwnProperty('layergroupid'));
+            assert.ok(!layergroup.hasOwnProperty('errors'));
+
+            done();
+        });
+    });
+
+    it('should work with mapnik substitution tokens', function(done) {
+        var cartocss = [
+            "#layer {",
+            "  line-width: 2;",
+            "  line-color: #3B3B58;",
+            "  line-opacity: 1;",
+            "  polygon-opacity: 0.7;",
+            "  polygon-fill: ramp([points_count], (#E5F5F9,#99D8C9,#2CA25F))",
+            "}"
+        ].join('\n');
+
+        var sql = [
+            'WITH hgrid AS (',
+            '  SELECT CDB_HexagonGrid(',
+            '    ST_Expand(!bbox!, greatest(!pixel_width!,!pixel_height!) * 100),',
+            '    greatest(!pixel_width!,!pixel_height!) * 100',
+            '  ) as cell',
+            ')',
+            'SELECT',
+            '  hgrid.cell as the_geom_webmercator,',
+            '  count(1) as points_count,',
+            '  count(1)/power(100 * CDB_XYZ_Resolution(CDB_ZoomFromScale(!scale_denominator!)), 2) as points_density,',
+            '  1 as cartodb_id',
+            'FROM hgrid, (SELECT * FROM populated_places_simple_reduced) i',
+            'where ST_Intersects(i.the_geom_webmercator, hgrid.cell)',
+            'GROUP BY hgrid.cell'
+        ].join('\n');
+
+        this.testClient = new TestClient(makeMapconfig(sql, cartocss));
+        this.testClient.getLayergroup(function(err, layergroup) {
+            assert.ok(!err, err);
+
+            assert.ok(layergroup.hasOwnProperty('layergroupid'));
+            assert.ok(!layergroup.hasOwnProperty('errors'));
+
             done();
         });
     });

test/integration/mapconfig_overviews_adapter.js

@@ -6,6 +6,7 @@ var cartodbRedis = require('cartodb-redis');
 var PgConnection = require(__dirname + '/../../lib/cartodb/backends/pg_connection');
 var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner');
 var OverviewsMetadataApi = require('../../lib/cartodb/api/overviews_metadata_api');
+var FilterStatsApi = require('../../lib/cartodb/api/filter_stats_api');
 var MapConfigOverviewsAdapter = require('../../lib/cartodb/models/mapconfig_overviews_adapter');
 
 // configure redis pool instance to use in tests
@@ -17,9 +18,9 @@ var metadataBackend = cartodbRedis({pool: redisPool});
 var pgConnection = new PgConnection(metadataBackend);
 var pgQueryRunner = new PgQueryRunner(pgConnection);
 var overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner);
+var filterStatsApi = new FilterStatsApi(pgQueryRunner);
 
-
-var mapConfigOverviewsAdapter = new MapConfigOverviewsAdapter(overviewsMetadataApi);
+var mapConfigOverviewsAdapter = new MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi);
 
 describe('MapConfigOverviewsAdapter', function() {
 
@@ -36,7 +37,7 @@ describe('MapConfigOverviewsAdapter', function() {
             }
         };
 
-        mapConfigOverviewsAdapter.getLayers('localhost', [layer_without_overviews], function(err, layers) {
+        mapConfigOverviewsAdapter.getLayers('localhost', [layer_without_overviews], [], function(err, layers) {
             assert.ok(!err);
             assert.equal(layers.length, 1);
             assert.equal(layers[0].type, 'cartodb');
@@ -64,7 +65,7 @@ describe('MapConfigOverviewsAdapter', function() {
             }
         };
 
-        mapConfigOverviewsAdapter.getLayers('localhost', [layer_without_overviews], function(err, layers) {
+        mapConfigOverviewsAdapter.getLayers('localhost', [layer_without_overviews], [], function(err, layers) {
             assert.ok(!err);
             assert.equal(layers.length, 1);
             assert.equal(layers[0].type, 'cartodb');

test/support/prepare_db.sh

@@ -83,7 +83,7 @@ if test x"$PREPARE_PGSQL" = xyes; then
 
   cat sql/_CDB_QueryStatements.sql | psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
 
-  SQL_SCRIPTS='CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins'
+  SQL_SCRIPTS='CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins CDB_Hexagon CDB_XYZ'
   for i in ${SQL_SCRIPTS}
   do
     curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/master/scripts-available/$i.sql -o sql/$i.sql

test/support/sql/windshaft.test.sql

@@ -247,8 +247,10 @@ CREATE TABLE test_table_overviews (
     cartodb_id integer NOT NULL,
     name character varying,
     address character varying,
+    value float8,
     the_geom geometry,
     the_geom_webmercator geometry,
+    _feature_count integer,
     CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)),
     CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)),
     CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))),
@@ -274,11 +276,11 @@ SELECT pg_catalog.setval('test_table_overviews_cartodb_id_seq', 60, true);
 ALTER TABLE test_table_overviews ALTER COLUMN cartodb_id SET DEFAULT nextval('test_table_overviews_cartodb_id_seq'::regclass);
 
 INSERT INTO test_table_overviews VALUES
-('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241'),
-('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241'),
-('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.324', 3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241'),
-('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.329509', 4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241'),
-('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.334931', 5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241');
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', 1.0, '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241', 1),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', 2.0, '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241', 1),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.324', 3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', 3.0, '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241', 1),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.329509', 4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', 4.0, '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241', 1),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.334931', 5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', 5.0, '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241', 1);
 
 ALTER TABLE ONLY test_table_overviews ADD CONSTRAINT test_table_overviews_pkey PRIMARY KEY (cartodb_id);
 
@@ -294,8 +296,10 @@ CREATE TABLE _vovw_1_test_table_overviews (
     cartodb_id integer NOT NULL,
     name character varying,
     address character varying,
+    value float8,
     the_geom geometry,
     the_geom_webmercator geometry,
+    _feature_count integer,
     CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)),
     CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)),
     CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))),
@@ -313,8 +317,10 @@ CREATE TABLE _vovw_2_test_table_overviews (
     cartodb_id integer NOT NULL,
     name character varying,
     address character varying,
+    value float8,
     the_geom geometry,
     the_geom_webmercator geometry,
+    _feature_count integer,
     CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)),
     CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)),
     CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))),
@@ -327,11 +333,11 @@ GRANT ALL ON TABLE _vovw_2_test_table_overviews TO :TESTUSER;
 GRANT SELECT ON TABLE _vovw_2_test_table_overviews TO :PUBLICUSER;
 
 INSERT INTO _vovw_2_test_table_overviews VALUES
-('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241'),
-('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E610000000000000009431C026043C75E7224340', '0101000020110F0000C4356B29423319C15DD1092DADCC5241');
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', 8.0/3.0, '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241', 3),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', 7.0/2.0, '0101000020E610000000000000009431C026043C75E7224340', '0101000020110F0000C4356B29423319C15DD1092DADCC5241', 2);
 
 INSERT INTO _vovw_1_test_table_overviews VALUES
-('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241');
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', 3.0, '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241', 5);
 
 
 -- analysis tables -----------------------------------------------

test/support/test-client.js

@@ -471,3 +471,41 @@ TestClient.prototype.getNodeStatus = function(nodeName, callback) {
 TestClient.prototype.drain = function(callback) {
     helper.deleteRedisKeys(this.keysToDelete, callback);
 };
+
+module.exports.getStaticMap = function getStaticMap(templateName, params, callback) {
+    if (!callback) {
+        callback = params;
+        params = null;
+    }
+
+    var url = '/api/v1/map/static/named/' + templateName + '/640/480.png';
+
+    if (params !== null) {
+        url += '?' + qs.stringify(params);
+    }
+
+    var requestOptions = {
+        url: url,
+        method: 'GET',
+        headers: {
+            host: 'localhost'
+        },
+        encoding: 'binary'
+    };
+
+    var expectedResponse = {
+        status: 200,
+        headers: {
+            'Content-Type': 'image/png'
+        }
+    };
+
+    // this could be removed once named maps are invalidated, otherwise you hits the cache
+    var server = new CartodbWindshaft(serverOptions);
+
+    assert.response(server, requestOptions, expectedResponse, function (res, err) {
+        helper.deleteRedisKeys({'user:localhost:mapviews:global': 5}, function() {
+            return callback(err, mapnik.Image.fromBytes(new Buffer(res.body, 'binary')));
+        });
+    });
+};

test/unit/cartodb/overviews_query_rewriter.js

@@ -475,4 +475,137 @@ describe('Overviews query rewriter', function() {
         ";
         assertSameSql(overviews_sql, expected_sql);
     });
+
+    it('generates query for specific Z level', function(){
+        var sql = "SELECT * FROM table1";
+        var data = {
+            overviews: {
+                table1: {
+                    0: { table: 'table1_ov0' },
+                    2: { table: 'table1_ov2' },
+                    3: { table: 'table1_ov3' }
+                }
+            }
+        };
+        var overviews_sql = overviewsQueryRewriter.query(sql, data, { zoom_level: 3 });
+        var expected_sql = "SELECT * FROM table1_ov3";
+        assertSameSql(overviews_sql, expected_sql);
+    });
+
+    it('generates query for specific nonpresent Z level', function(){
+        var sql = "SELECT * FROM table1";
+        var data = {
+            overviews: {
+                table1: {
+                    0: { table: 'table1_ov0' },
+                    2: { table: 'table1_ov2' },
+                    3: { table: 'table1_ov3' }
+                }
+            }
+        };
+        var overviews_sql = overviewsQueryRewriter.query(sql, data, { zoom_level: 1 });
+        var expected_sql = "SELECT * FROM table1_ov2";
+        assertSameSql(overviews_sql, expected_sql);
+    });
+
+    it('does not use overviews for specific out-of-range Z level', function(){
+        var sql = "SELECT * FROM table1";
+        var data = {
+            overviews: {
+                table1: {
+                    0: { table: 'table1_ov0' },
+                    2: { table: 'table1_ov2' },
+                    3: { table: 'table1_ov3' }
+                }
+            }
+        };
+        var overviews_sql = overviewsQueryRewriter.query(sql, data, { zoom_level: 4 });
+        var expected_sql = "SELECT * FROM table1";
+        assertSameSql(overviews_sql, expected_sql);
+    });
+
+    it('generates query with filters', function(){
+        var sql = "SELECT ST_Transform(the_geom, 3857) the_geom_webmercator, cartodb_id, name\
+                   FROM (SELECT *\
+                   FROM (select * from table1) _camshaft_category_filter\
+                   WHERE name IN ($escape_0$X$escape_0$)) _cdb_analysis_query";
+        var data = {
+            overviews: {
+                table1: {
+                    0: { table: 'table1_ov0' },
+                    1: { table: 'table1_ov1' },
+                    2: { table: 'table1_ov2' },
+                    3: { table: 'table1_ov3' }
+                }
+            },
+            filters: { name_filter: { type: 'category', column: 'name', params: { accept: [ 'X' ] } } },
+            unfiltered_query: 'SELECT * FROM table1'
+        };
+        var overviews_sql = overviewsQueryRewriter.query(sql, data);
+        var expected_sql = "\
+            SELECT * FROM (WITH\
+                _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+                SELECT * FROM  (\
+                  SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\
+                  UNION ALL\
+                  SELECT * FROM table1_ov1, _vovw_scale WHERE _vovw_z = 1\
+                  UNION ALL\
+                  SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z = 2\
+                  UNION ALL\
+                  SELECT * FROM table1_ov3, _vovw_scale WHERE _vovw_z = 3\
+                  UNION ALL\
+                  SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 3\
+                ) AS _vovw_table1) _camshaft_category_filter\
+             WHERE name IN ($escape_0$X$escape_0$)\
+        ";
+        assertSameSql(overviews_sql, expected_sql);
+    });
+
+    it('generates query with filters for specific zoom level', function(){
+        var sql = "SELECT ST_Transform(the_geom, 3857) the_geom_webmercator, cartodb_id, name\
+                   FROM (SELECT *\
+                   FROM (select * from table1) _camshaft_category_filter\
+                   WHERE name IN ($escape_0$X$escape_0$)) _cdb_analysis_query";
+        var data = {
+            overviews: {
+                table1: {
+                    0: { table: 'table1_ov0' },
+                    1: { table: 'table1_ov1' },
+                    2: { table: 'table1_ov2' },
+                    3: { table: 'table1_ov3' }
+                }
+            },
+            filters: { name_filter: { type: 'category', column: 'name', params: { accept: [ 'X' ] } } },
+            unfiltered_query: 'SELECT * FROM table1',
+            filter_stats: { unfiltered_rows: 1000, filtered_rows: 900 }
+        };
+        var overviews_sql = overviewsQueryRewriter.query(sql, data, { zoom_level: 2 });
+        var expected_sql = "\
+            SELECT * FROM (SELECT * FROM table1_ov2) _camshaft_category_filter\
+            WHERE name IN ($escape_0$X$escape_0$)\
+        ";
+        assertSameSql(overviews_sql, expected_sql);
+    });
+
+    it('does not generates query with aggressive filtering', function(){
+        var sql = "SELECT ST_Transform(the_geom, 3857) the_geom_webmercator, cartodb_id, name\
+                   FROM (SELECT *\
+                   FROM (select * from table1) _camshaft_category_filter\
+                   WHERE name IN ($escape_0$X$escape_0$)) _cdb_analysis_query";
+        var data = {
+            overviews: {
+                table1: {
+                    0: { table: 'table1_ov0' },
+                    1: { table: 'table1_ov1' },
+                    2: { table: 'table1_ov2' },
+                    3: { table: 'table1_ov3' }
+                }
+            },
+            filters: { name_filter: { type: 'category', column: 'name', params: { accept: [ 'X' ] } } },
+            unfiltered_query: 'SELECT * FROM table1',
+            filter_stats: { unfiltered_rows: 1000, filtered_rows: 10 }
+        };
+        var overviews_sql = overviewsQueryRewriter.query(sql, data);
+        assert.equal(overviews_sql, sql);
+    });
 });