diff --git a/NEWS.md b/NEWS.md index 19ad5960..9eeb06e3 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,9 +1,27 @@ # Changelog -## 2.42.3 +## 2.44.0 Released 2016-mm-dd +Adds support for sql wrap in all layers + + +## 2.43.1 + +Released 2016-05-19 + +Bug fixes: + - Dataview error when bbox present without query rewrite data #458 + + +## 2.43.0 + +Released 2016-05-18 + +New features: + - Overviews now support dataviews and filtering #449 + ## 2.42.2 diff --git a/lib/cartodb/api/filter_stats_api.js b/lib/cartodb/api/filter_stats_api.js new file mode 100644 index 00000000..d872ad8c --- /dev/null +++ b/lib/cartodb/api/filter_stats_api.js @@ -0,0 +1,59 @@ +var _ = require('underscore'); +var step = require('step'); +var CamshaftFilter = require('../models/filter/camshaft'); + +function FilterStatsApi(pgQueryRunner) { + this.pgQueryRunner = pgQueryRunner; +} + +module.exports = FilterStatsApi; + +function getEstimatedRows(pgQueryRunner, username, query, callback) { + pgQueryRunner.run(username, "EXPLAIN (FORMAT JSON)"+query, function(err, result_rows) { + if (err){ + callback(err); + return; + } + var rows; + if ( result_rows[0] && result_rows[0]['QUERY PLAN'] && + result_rows[0]['QUERY PLAN'][0] && result_rows[0]['QUERY PLAN'][0].Plan ) { + rows = result_rows[0]['QUERY PLAN'][0].Plan['Plan Rows']; + } + return callback(null, rows); + }); +} + +FilterStatsApi.prototype.getFilterStats = function (username, unfiltered_query, filters, callback) { + var stats = {}; + var self = this; + step( + function getUnfilteredRows() { + getEstimatedRows(self.pgQueryRunner, username, unfiltered_query, this); + }, + function receiveUnfilteredRows(err, rows) { + if (err){ + callback(err); + return; + } + stats.unfiltered_rows = rows; + this(null, rows); + }, + function getFilteredRows() { + if ( filters && !_.isEmpty(filters)) { + var camshaftFilter = new CamshaftFilter(filters); + var query = camshaftFilter.sql(unfiltered_query); + getEstimatedRows(self.pgQueryRunner, username, query, this); + } else { + this(null, null); + } + }, + function receiveFilteredRows(err, rows) { + if (err){ + callback(err); + return; + } + stats.filtered_rows = rows; + callback(null, stats); + } + ); +}; diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index 510709c4..b7bb6d0b 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -8,7 +8,13 @@ var step = require('step'); var Timer = require('../stats/timer'); var BBoxFilter = require('../models/filter/bbox'); + var DataviewFactory = require('../models/dataview/factory'); +var DataviewFactoryWithOverviews = require('../models/dataview/overviews/factory'); +var OverviewsQueryRewriter = require('../utils/overviews_query_rewriter'); +var overviewsQueryRewriter = new OverviewsQueryRewriter({ + zoom_level: 'CDB_ZoomFromScale(!scale_denominator!)' +}); function DataviewBackend(analysisBackend) { this.analysisBackend = analysisBackend; @@ -16,7 +22,6 @@ function DataviewBackend(analysisBackend) { module.exports = DataviewBackend; - DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, params, callback) { var self = this; @@ -101,6 +106,7 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param ownFilter = !!ownFilter; var query; + if (ownFilter) { query = node.getQuery(); } else { @@ -109,11 +115,48 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param query = node.getQuery(applyFilters); } + var sourceId = dataviewDefinition.source.id; // node.id + var layer = _.find( + mapConfig.obj().layers, + function(l){ return l.options.source && (l.options.source.id === sourceId); } + ); + var queryRewriteData = layer && layer.options.query_rewrite_data; + if ( queryRewriteData ) { + if ( node.type === 'source' ) { + var filters = node.filters; // TODO: node.getFilters() when available in camshaft + var filters_disabler = Object.keys(filters).reduce( + function(disabler, filter_id){ disabler[filter_id] = false; return disabler; }, + {} + ); + var unfiltered_query = node.getQuery(filters_disabler); + queryRewriteData = _.extend( + {}, + queryRewriteData, { filters: filters, unfiltered_query: unfiltered_query } + ); + } + } + if (params.bbox) { var bboxFilter = new BBoxFilter({column: 'the_geom', srid: 4326}, {bbox: params.bbox}); query = bboxFilter.sql(query); + if ( queryRewriteData ) { + var bbox_filter_definition = { + type: 'bbox', + options: { + column: 'the_geom', + srid: 4326, + }, + params: { + bbox: params.bbox + } + }; + queryRewriteData = _.extend(queryRewriteData, { bbox_filter: bbox_filter_definition }); + } } + var dataviewFactory = DataviewFactoryWithOverviews.getFactory( + overviewsQueryRewriter, queryRewriteData, { bbox: params.bbox } + ); var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins'), function castNumbers(overrides, val, k) { @@ -123,7 +166,7 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param {ownFilter: ownFilter} ); - var dataview = DataviewFactory.getDataview(query, dataviewDefinition); + var dataview = dataviewFactory.getDataview(query, dataviewDefinition); dataview.getResult(pg, overrideParams, this); }, function returnCallback(err, result) { @@ -277,4 +320,4 @@ function dbParamsFromReqParams(params) { dbParams.dbname = params.dbname; } return dbParams; -} \ No newline at end of file +} diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 442079e0..1f8f25bb 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -19,6 +19,7 @@ var MapConfigNamedLayersAdapter = require('../models/mapconfig_named_layers_adap var AnalysisMapConfigAdapter = require('../models/analysis-mapconfig-adapter'); var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider'); var CreateLayergroupMapConfigProvider = require('../models/mapconfig/create_layergroup_provider'); +var SqlWrapMapConfigAdapter = require('../models/mapconfig/adapter/sql-wrap-mapconfig-adapter'); /** * @param {AuthApi} authApi @@ -52,6 +53,7 @@ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadata this.analysisMapConfigAdapter = new AnalysisMapConfigAdapter(analysisBackend); this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps); this.overviewsAdapter = overviewsAdapter; + this.sqlWrapMapConfigAdapter = new SqlWrapMapConfigAdapter(); } util.inherits(MapController, BaseController); @@ -139,6 +141,10 @@ MapController.prototype.create = function(req, res, prepareConfigFn) { self.req2params(req, this); }, prepareConfigFn, + function prepareSqlWrap(err, requestMapConfig) { + assert.ifError(err); + self.sqlWrapMapConfigAdapter.getMapConfig(requestMapConfig, this); + }, function prepareAnalysisLayers(err, requestMapConfig) { assert.ifError(err); var analysisConfiguration = { @@ -185,17 +191,20 @@ MapController.prototype.create = function(req, res, prepareConfigFn) { function addOverviewsInformation(err, requestMapConfig, datasource) { assert.ifError(err); var next = this; - self.overviewsAdapter.getLayers(req.context.user, requestMapConfig.layers, function(err, layers) { - if (err) { - return next(err); - } + self.overviewsAdapter.getLayers( + req.context.user, requestMapConfig.layers, analysesResults, + function(err, layers) { + if (err) { + return next(err); + } - if (layers) { - requestMapConfig.layers = layers; - } + if (layers) { + requestMapConfig.layers = layers; + } - return next(null, requestMapConfig, datasource); - }); + return next(null, requestMapConfig, datasource); + } + ); }, function parseTurboCarto(err, requestMapConfig, datasource) { assert.ifError(err); diff --git a/lib/cartodb/models/dataview/overviews/factory.js b/lib/cartodb/models/dataview/overviews/factory.js new file mode 100644 index 00000000..b63b004a --- /dev/null +++ b/lib/cartodb/models/dataview/overviews/factory.js @@ -0,0 +1,32 @@ +var parentFactory = require('../factory'); +var dataviews = require('./'); + +function OverviewsDataviewFactory(queryRewriter, queryRewriteData, options) { + this.queryRewriter = queryRewriter; + this.queryRewriteData = queryRewriteData; + this.options = options; +} + +OverviewsDataviewFactory.prototype.getDataview = function(query, dataviewDefinition) { + var type = dataviewDefinition.type; + var dataviews = OverviewsDataviewMetaFactory.dataviews; + if ( !this.queryRewriter || !this.queryRewriteData || !dataviews[type] ) { + return parentFactory.getDataview(query, dataviewDefinition); + } + return new dataviews[type]( + query, dataviewDefinition.options, this.queryRewriter, this.queryRewriteData, this.options + ); +}; + +var OverviewsDataviewMetaFactory = { + dataviews: Object.keys(dataviews).reduce(function(allDataviews, dataviewClassName) { + allDataviews[dataviewClassName.toLowerCase()] = dataviews[dataviewClassName]; + return allDataviews; + }, {}), + + getFactory: function(queryRewriter, queryRewriteData, options) { + return new OverviewsDataviewFactory(queryRewriter, queryRewriteData, options); + }, +}; + +module.exports = OverviewsDataviewMetaFactory; diff --git a/lib/cartodb/models/dataview/overviews/formula.js b/lib/cartodb/models/dataview/overviews/formula.js new file mode 100644 index 00000000..aa57792e --- /dev/null +++ b/lib/cartodb/models/dataview/overviews/formula.js @@ -0,0 +1,104 @@ +var _ = require('underscore'); +var BaseWidget = require('../base'); +var BaseDataview = require('../formula'); + +var debug = require('debug')('windshaft:widget:formula:overviews'); + +var dot = require('dot'); +dot.templateSettings.strip = false; + +var formulaQueryTpls = { + 'count': dot.template([ + 'SELECT', + 'sum(_feature_count) AS result,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', + 'FROM ({{=it._query}}) _cdb_formula' + ].join('\n')), + 'sum': dot.template([ + 'SELECT', + 'sum({{=it._column}}*_feature_count) AS result,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', + 'FROM ({{=it._query}}) _cdb_formula' + ].join('\n')), + 'avg': dot.template([ + 'SELECT', + 'sum({{=it._column}}*_feature_count)/sum(_feature_count) AS result,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', + 'FROM ({{=it._query}}) _cdb_formula' + ].join('\n')), +}; + +function Formula(query, options, queryRewriter, queryRewriteData, params) { + this.base_dataview = new BaseDataview(query, options); + this.query = query; + this.column = options.column || '1'; + this.operation = options.operation; + this.queryRewriter = queryRewriter; + this.queryRewriteData = queryRewriteData; + this.options = params; +} + +Formula.prototype = new BaseWidget(); +Formula.prototype.constructor = Formula; + +module.exports = Formula; + +var zoom_level_factor = 100.0; + +// Compute zoom level so that the the resolution grid size of the +// selected overview is smaller (zoom_level_factor times smaller at least) +// than the bounding box size. +function zoom_level_for_bbox(bbox) { + var px_per_tile = 256.0; + var earth_width = 360.0; + // TODO: now we assume overviews are computed for 1-pixel tolerance; + // should use extended overviews metadata to compute this properly. + if ( bbox ) { + var bbox_values = _.map(bbox.split(','), function(v) { return +v; }); + var w = Math.abs(bbox_values[2]-bbox_values[0]); + var h = Math.abs(bbox_values[3]-bbox_values[1]); + var max_dim = Math.min(w, h); + + // Find minimum suitable z + // note that the QueryRewirter will use the minimum level overview + // of level >= z if it exists, and otherwise the base table + var z = Math.ceil(-Math.log(max_dim*px_per_tile/earth_width/zoom_level_factor)/Math.log(2.0)); + return Math.max(z, 0); + } + return 0; +} + +Formula.prototype.sql = function(psql, filters, override, callback) { + var _query = this.query; + var formulaQueryTpl = formulaQueryTpls[this.operation]; + + if ( formulaQueryTpl ) { + // supported formula for use with overviews + var zoom_level = zoom_level_for_bbox(this.options.bbox); + _query = this.queryRewriter.query(_query, this.queryRewriteData, { zoom_level: zoom_level }); + var formulaSql = formulaQueryTpl({ + _query: _query, + _operation: this.operation, + _column: this.column + }); + debug(formulaSql); + callback = callback || override; + + return callback(null, formulaSql); + } + + // For non supported operations (min, max) we're not using overviews. + return this.base_dataview.sql(psql, filters, override, callback); +}; + +Formula.prototype.format = function(result) { + return this.base_dataview.format(result); +}; + +Formula.prototype.getType = function() { + return this.base_dataview.getType(); +}; + +Formula.prototype.toString = function() { + return this.base_dataview.toString(); +}; diff --git a/lib/cartodb/models/dataview/overviews/index.js b/lib/cartodb/models/dataview/overviews/index.js new file mode 100644 index 00000000..775ceca1 --- /dev/null +++ b/lib/cartodb/models/dataview/overviews/index.js @@ -0,0 +1,3 @@ +module.exports = { + Formula: require('./formula') +}; diff --git a/lib/cartodb/models/filter/camshaft.js b/lib/cartodb/models/filter/camshaft.js new file mode 100644 index 00000000..97cb25b6 --- /dev/null +++ b/lib/cartodb/models/filter/camshaft.js @@ -0,0 +1,35 @@ +var filters = { + category: require('./camshaft/category'), + range: require('./camshaft/range') +}; + +function createFilter(filterDefinition) { + var filterType = filterDefinition.type.toLowerCase(); + if (!filters.hasOwnProperty(filterType)) { + throw new Error('Unknown filter type: ' + filterType); + } + return new filters[filterType](filterDefinition.column, filterDefinition.params); +} + +function CamshaftFilters(filters) { + this.filters = filters; +} + +CamshaftFilters.prototype.sql = function(rawSql) { + var filters = this.filters || {}; + var applyFilters = {}; + + return Object.keys(filters) + .filter(function(filterName) { + return applyFilters.hasOwnProperty(filterName) ? applyFilters[filterName] : true; + }) + .map(function(filterName) { + var filterDefinition = filters[filterName]; + return createFilter(filterDefinition); + }) + .reduce(function(sql, filter) { + return filter.sql(sql); + }, rawSql); +}; + +module.exports = CamshaftFilters; diff --git a/lib/cartodb/models/filter/camshaft/category.js b/lib/cartodb/models/filter/camshaft/category.js new file mode 100644 index 00000000..6181de7f --- /dev/null +++ b/lib/cartodb/models/filter/camshaft/category.js @@ -0,0 +1,79 @@ +'use strict'; + +var debug = require('debug')('windshaft:filter:category'); +var dot = require('dot'); +dot.templateSettings.strip = false; + +var filterQueryTpl = dot.template([ + 'SELECT *', + 'FROM ({{=it._sql}}) _camshaft_category_filter', + 'WHERE {{=it._filters}}' +].join('\n')); +var escapeStringTpl = dot.template('$escape_{{=it._i}}${{=it._value}}$escape_{{=it._i}}$'); +var inConditionTpl = dot.template('{{=it._column}} IN ({{=it._values}})'); +var notInConditionTpl = dot.template('{{=it._column}} NOT IN ({{=it._values}})'); + +function Category(column, filterParams) { + this.column = column; + + if (!Array.isArray(filterParams.accept) && !Array.isArray(filterParams.reject)) { + throw new Error('Category filter expects at least one array in accept or reject params'); + } + + if (Array.isArray(filterParams.accept) && Array.isArray(filterParams.reject)) { + if (filterParams.accept.length === 0 && filterParams.reject.length === 0) { + throw new Error( + 'Category filter expects one value either in accept or reject params when both are provided' + ); + } + } + + this.accept = filterParams.accept; + this.reject = filterParams.reject; +} + +module.exports = Category; + +/* + - accept: [] => reject all + - reject: [] => accept all + */ +Category.prototype.sql = function(rawSql) { + var valueFilters = []; + + if (Array.isArray(this.accept)) { + if (this.accept.length > 0) { + valueFilters.push(inConditionTpl({ + _column: this.column, + _values: this.accept.map(function(value, i) { + return Number.isFinite(value) ? value : escapeStringTpl({_i: i, _value: value}); + }).join(',') + })); + } else { + valueFilters.push('0 = 1'); + } + } + + if (Array.isArray(this.reject)) { + if (this.reject.length > 0) { + valueFilters.push(notInConditionTpl({ + _column: this.column, + _values: this.reject.map(function (value, i) { + return Number.isFinite(value) ? value : escapeStringTpl({_i: i, _value: value}); + }).join(',') + })); + } else { + valueFilters.push('1 = 1'); + } + } + + debug(filterQueryTpl({ + _sql: rawSql, + _filters: valueFilters.join(' AND ') + })); + + return filterQueryTpl({ + _sql: rawSql, + _filters: valueFilters.join(' AND ') + }); +}; diff --git a/lib/cartodb/models/filter/camshaft/range.js b/lib/cartodb/models/filter/camshaft/range.js new file mode 100644 index 00000000..f894ca32 --- /dev/null +++ b/lib/cartodb/models/filter/camshaft/range.js @@ -0,0 +1,43 @@ +'use strict'; + +var dot = require('dot'); +dot.templateSettings.strip = false; + +var betweenFilterTpl = dot.template('{{=it._column}} BETWEEN {{=it._min}} AND {{=it._max}}'); +var minFilterTpl = dot.template('{{=it._column}} >= {{=it._min}}'); +var maxFilterTpl = dot.template('{{=it._column}} <= {{=it._max}}'); +var filterQueryTpl = dot.template('SELECT * FROM ({{=it._sql}}) _camshaft_range_filter WHERE {{=it._filter}}'); + +function Range(column, filterParams) { + this.column = column; + + if (!Number.isFinite(filterParams.min) && !Number.isFinite(filterParams.max)) { + throw new Error('Range filter expect to have at least one value in min or max numeric params'); + } + + this.min = filterParams.min; + this.max = filterParams.max; + this.columnType = filterParams.columnType; +} + +module.exports = Range; + +Range.prototype.sql = function(rawSql) { + var minMaxFilter; + if (Number.isFinite(this.min) && Number.isFinite(this.max)) { + minMaxFilter = betweenFilterTpl({ + _column: this.column, + _min: this.min, + _max: this.max + }); + } else if (Number.isFinite(this.min)) { + minMaxFilter = minFilterTpl({ _column: this.column, _min: this.min }); + } else { + minMaxFilter = maxFilterTpl({ _column: this.column, _max: this.max }); + } + + return filterQueryTpl({ + _sql: rawSql, + _filter: minMaxFilter + }); +}; diff --git a/lib/cartodb/models/mapconfig/adapter/sql-wrap-mapconfig-adapter.js b/lib/cartodb/models/mapconfig/adapter/sql-wrap-mapconfig-adapter.js new file mode 100644 index 00000000..6b9a5130 --- /dev/null +++ b/lib/cartodb/models/mapconfig/adapter/sql-wrap-mapconfig-adapter.js @@ -0,0 +1,25 @@ +function SqlWrapMapConfigAdapter() { +} + +module.exports = SqlWrapMapConfigAdapter; + + +SqlWrapMapConfigAdapter.prototype.getMapConfig = function(requestMapConfig, callback) { + if (requestMapConfig && Array.isArray(requestMapConfig.layers)) { + requestMapConfig.layers = requestMapConfig.layers.map(function(layer) { + if (layer.options) { + var sqlQueryWrap = layer.options.sql_wrap; + if (sqlQueryWrap) { + var layerSql = layer.options.sql; + if (layerSql) { + layer.options.sql_raw = layerSql; + layer.options.sql = sqlQueryWrap.replace(/<%=\s*sql\s*%>/g, layerSql); + } + } + } + return layer; + }); + } + + return callback(null, requestMapConfig); +}; diff --git a/lib/cartodb/models/mapconfig/named_map_provider.js b/lib/cartodb/models/mapconfig/named_map_provider.js index 9c8c218d..e1ad6fea 100644 --- a/lib/cartodb/models/mapconfig/named_map_provider.js +++ b/lib/cartodb/models/mapconfig/named_map_provider.js @@ -6,6 +6,7 @@ var step = require('step'); var MapConfig = require('windshaft').model.MapConfig; var templateName = require('../../backends/template_maps').templateName; var QueryTables = require('cartodb-query-tables'); +var SqlWrapMapConfigAdapter = require('./adapter/sql-wrap-mapconfig-adapter'); /** * @constructor @@ -22,6 +23,7 @@ function NamedMapMapConfigProvider(templateMaps, pgConnection, metadataBackend, this.turboCartoAdapter = turboCartoAdapter; this.analysisMapConfigAdapter = analysisMapConfigAdapter; this.overviewsAdapter = overviewsAdapter; + this.sqlWrapMapConfigAdapter = new SqlWrapMapConfigAdapter(); this.owner = owner; this.templateName = templateName(templateId); @@ -95,6 +97,10 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) { assert.ifError(err); return self.templateMaps.instance(self.template, templateParams); }, + function prepareSqlWrap(err, requestMapConfig) { + assert.ifError(err); + self.sqlWrapMapConfigAdapter.getMapConfig(requestMapConfig, this); + }, function prepareAnalysisLayers(err, requestMapConfig) { assert.ifError(err); var analysisConfiguration = { @@ -142,7 +148,7 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) { assert.ifError(err); var next = this; - self.overviewsAdapter.getLayers(self.owner, _mapConfig.layers, function(err, layers) { + self.overviewsAdapter.getLayers(self.owner, _mapConfig.layers, self.analysesResults, function(err, layers) { if (err) { return next(err); } diff --git a/lib/cartodb/models/mapconfig_overviews_adapter.js b/lib/cartodb/models/mapconfig_overviews_adapter.js index 70a02dc5..f5cfde9e 100644 --- a/lib/cartodb/models/mapconfig_overviews_adapter.js +++ b/lib/cartodb/models/mapconfig_overviews_adapter.js @@ -1,13 +1,15 @@ +var step = require('step'); var queue = require('queue-async'); var _ = require('underscore'); -function MapConfigOverviewsAdapter(overviewsMetadataApi) { +function MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi) { this.overviewsMetadataApi = overviewsMetadataApi; + this.filterStatsApi = filterStatsApi; } module.exports = MapConfigOverviewsAdapter; -MapConfigOverviewsAdapter.prototype.getLayers = function(username, layers, callback) { +MapConfigOverviewsAdapter.prototype.getLayers = function(username, layers, analysesResults, callback) { var self = this; if (!layers || layers.length === 0) { @@ -24,11 +26,52 @@ MapConfigOverviewsAdapter.prototype.getLayers = function(username, layers, callb if (err) { done(err, layer); } else { - if ( !_.isEmpty(metadata) ) { - layer = _.extend({}, layer); - layer.options = _.extend({}, layer.options, { query_rewrite_data: { overviews: metadata } }); - } - done(null, layer); + var query_rewrite_data = { overviews: metadata }; + step( + function collectFiltersData() { + var filters, unfiltered_query; + if ( layer.options.source && analysesResults ) { + var sourceId = layer.options.source.id; + var node = _.find(analysesResults, function(a){ return a.rootNode.params.id === sourceId; }); + if ( node ) { + node = node.rootNode; + filters = node.filters; // TODO: node.getFilters() when available in camshaft + var filters_disabler = Object.keys(filters).reduce( + function(disabler, filter_id){ disabler[filter_id] = false; return disabler; }, + {} + ); + unfiltered_query = node.getQuery(filters_disabler); + query_rewrite_data.filters = filters; + query_rewrite_data.unfiltered_query = unfiltered_query; + } + } + this(null, filters, unfiltered_query); + }, + function collectStatsData(err, filters, unfiltered_query) { + var next_step = this; + if ( filters ) { + self.filterStatsApi.getFilterStats( + username, + unfiltered_query, filters, + function(err, stats) { + if ( !err ) { + query_rewrite_data.filter_stats = stats; + } + return next_step(err); + } + ); + } else { + return next_step(null); + } + }, + function addDataToLayer(err) { + if ( !err && !_.isEmpty(metadata) ) { + layer = _.extend({}, layer); + layer.options = _.extend({}, layer.options, { query_rewrite_data: query_rewrite_data }); + } + done(err, layer); + } + ); } }); } diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 0f15e240..8117d208 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -21,6 +21,7 @@ var mapnik = windshaft.mapnik; var TemplateMaps = require('./backends/template_maps.js'); var OverviewsMetadataApi = require('./api/overviews_metadata_api'); +var FilterStatsApi = require('./api/filter_stats_api'); var UserLimitsApi = require('./api/user_limits_api'); var AuthApi = require('./api/auth_api'); var LayergroupAffectedTablesCache = require('./cache/layergroup_affected_tables'); @@ -59,6 +60,7 @@ module.exports = function(serverOptions) { var pgConnection = new PgConnection(metadataBackend); var pgQueryRunner = new PgQueryRunner(pgConnection); var overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner); + var filterStatsApi = new FilterStatsApi(pgQueryRunner); var userLimitsApi = new UserLimitsApi(metadataBackend, { limits: { cacheOnTimeout: serverOptions.renderer.mapnik.limits.cacheOnTimeout || false, @@ -148,7 +150,7 @@ module.exports = function(serverOptions) { var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache(); app.layergroupAffectedTablesCache = layergroupAffectedTablesCache; - var overviewsAdapter = new MapConfigOverviewsAdapter(overviewsMetadataApi); + var overviewsAdapter = new MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi); var turboCartoParser = new TurboCartoParser(pgQueryRunner); var turboCartoAdapter = new TurboCartoAdapter(turboCartoParser); diff --git a/lib/cartodb/utils/overviews_query_rewriter.js b/lib/cartodb/utils/overviews_query_rewriter.js index b5b35503..ebb91fb0 100644 --- a/lib/cartodb/utils/overviews_query_rewriter.js +++ b/lib/cartodb/utils/overviews_query_rewriter.js @@ -1,5 +1,26 @@ +var _ = require('underscore'); var TableNameParser = require('./table_name_parser'); +var BBoxFilter = require('../models/filter/bbox'); +var CamshaftFilter = require('../models/filter/camshaft'); + +// Minimim number of filtered rows to use overviews +var FILTER_MIN_ROWS = 65536; +// Maximum filtered fraction to not apply overviews +var FILTER_MAX_FRACTION = 0.2; + +function apply_filters_to_query(query, filters, bbox_filter) { + if ( filters && !_.isEmpty(filters)) { + var camshaftFilter = new CamshaftFilter(filters); + query = camshaftFilter.sql(query); + } + if ( bbox_filter ) { + var bboxFilter = new BBoxFilter(bbox_filter.options, bbox_filter.params); + query = bboxFilter.sql(query); + } + return query; +} + function OverviewsQueryRewriter(options) { this.options = options; @@ -119,26 +140,34 @@ function replace_table_in_query(sql, old_table_name, replacement) { return sql.replace(new RegExp(regexp, 'g'), replacement); } -function overviews_query(query, overviews, zoom_level_expression) { + +function replace_table_in_query_with_schema(query, table, schema, replacement) { + if ( replacement ) { + query = replace_table_in_query(query, table, replacement); + var parsed_table = TableNameParser.parse(table); + if (!parsed_table.schema && schema) { + // replace also the qualified table name, if the table wasn't qualified + parsed_table.schema = schema; + table = TableNameParser.table_identifier(parsed_table); + query = replace_table_in_query(query, table, replacement); + } + } + return query; +} + +// Build query to use overviews for a variant zoom level (given by a expression to +// be evaluated by the database server) +function overviews_query_with_zoom_expression(query, overviews, zoom_level_expression) { var replaced_query = query; var sql = "WITH\n _vovw_scale AS ( SELECT " + zoom_level_expression + " AS _vovw_z )"; var replacement; - for ( var table in overviews ) { - if (overviews.hasOwnProperty(table)) { - var table_overviews = overviews[table]; - var table_view = overviews_view_name(table); - var schema = table_overviews.schema; - replacement = "(\n" + overviews_view_for_table(table, table_overviews) + "\n ) AS " + table_view; - replaced_query = replace_table_in_query(replaced_query, table, replacement); - var parsed_table = TableNameParser.parse(table); - if (!parsed_table.schema && schema) { - // replace also the qualified table name, if the table wasn't qualified - parsed_table.schema = schema; - table = TableNameParser.table_identifier(parsed_table); - replaced_query = replace_table_in_query(replaced_query, table, replacement); - } - } - } + _.each(Object.keys(overviews), function(table) { + var table_overviews = overviews[table]; + var table_view = overviews_view_name(table); + var schema = table_overviews.schema; + replacement = "(\n" + overviews_view_for_table(table, table_overviews) + "\n ) AS " + table_view; + replaced_query = replace_table_in_query_with_schema(replaced_query, table, schema, replacement); + }); if ( replaced_query !== query ) { sql += "\n"; sql += replaced_query; @@ -148,34 +177,128 @@ function overviews_query(query, overviews, zoom_level_expression) { return sql; } +// Build query to use overviews for a specific zoom level value +function overviews_query_with_definite_zoom(query, overviews, zoom_level) { + var replaced_query = query; + var replacement; + _.each(Object.keys(overviews), function(table) { + var table_overviews = overviews[table]; + var schema = table_overviews.schema; + replacement = overview_table_for_zoom_level(table_overviews, zoom_level); + replaced_query = replace_table_in_query_with_schema(replaced_query, table, schema, replacement); + }); + return replaced_query; +} + +// Find a suitable overview table for a specific zoom_level +function overview_table_for_zoom_level(table_overviews, zoom_level) { + var overview_table; + if ( table_overviews ) { + overview_table = table_overviews[zoom_level]; + if ( !overview_table ) { + _.every(Object.keys(table_overviews).sort(function(x,y){ return x-y; }), function(overview_zoom) { + if ( +overview_zoom > +zoom_level ) { + overview_table = table_overviews[overview_zoom]; + return false; + } else { + return true; + } + }); + } + } + if ( overview_table ) { + overview_table = overview_table.table; + } + return overview_table; +} + // Transform an SQL query so that it uses overviews. -// overviews contains metadata about the overviews to be used: -// { 'table-name': {1: { table: 'overview-table-1' }, ... }, ... } // // For a given query `SELECT * FROM table`, if any of tables in it // has overviews as defined by the provided metadat, the query will // be transform into something similar to this: // // WITH _vovw_scale AS ( ... ), -- define scale level -// WITH _vovw_table AS ( ... ), -- define union of overviews and base table -// SELECT * FROM _vovw_table -- query with table replaced by _vovw_table +// SELECT * FROM -- in the query the table is replaced by: +// ( ... ) AS _vovw_table -- a union of overviews and base table // -// This transformation can in principle be applied to arbitrary queries -// (except for the case of queries that include the name of tables with -// overviews inside text literals: at the current table name substitution -// doesnn't prevent substitution inside literals). -// But the transformation will currently only be applied to simple queries -// of the form detected by the overviews_supported_query function. -OverviewsQueryRewriter.prototype.query = function(query, data) { - var overviews = this.overviews_metadata(data); - if ( !overviews || !this.is_supported_query(query)) { +// The data argument has the form: +// { +// overviews: // overview tables metadata +// { 'table-name': {1: { table: 'overview-table-1' }, ... }, ... }, +// zoom_level: ..., // optional zoom level +// filters: ..., // filters definition +// unfiltered_query: ..., // query without the filters +// bbox_filter: ... // bounding-box filter +// } +OverviewsQueryRewriter.prototype.query = function(query, data, options) { + options = options || {}; + data = data || {}; + + var overviews = data.overviews; + var unfiltered_query = data.unfiltered_query; + var filters = data.filters; + var bbox_filter = data.bbox_filter; + + if ( !unfiltered_query ) { + unfiltered_query = query; + } + + if ( !should_use_overviews(unfiltered_query, data) ) { return query; } - var zoom_level_expression = this.options.zoom_level || '0'; - return overviews_query(query, overviews, zoom_level_expression); + + var rewritten_query; + + var zoom_level_expression = this.options.zoom_level; + var zoom_level = zoom_level_for_query(unfiltered_query, zoom_level_expression, options); + + rewritten_query = overviews_query(unfiltered_query, overviews, zoom_level, zoom_level_expression); + + if ( rewritten_query === unfiltered_query ) { + // could not or didn't need to alter the query + rewritten_query = query; + } else { + rewritten_query = apply_filters_to_query(rewritten_query, filters, bbox_filter); + } + + return rewritten_query; }; -OverviewsQueryRewriter.prototype.is_supported_query = function(sql) { +function zoom_level_for_query(query, zoom_level_expression, options) { + var zoom_level = null; + if ( _.has(options, 'zoom_level') ) { + zoom_level = options.zoom_level || '0'; + } + if ( zoom_level === null && !zoom_level_expression ) { + zoom_level = '0'; + } + return zoom_level; +} + +function overviews_query(query, overviews, zoom_level, zoom_level_expression) { + if ( zoom_level || zoom_level === '0' || zoom_level === 0 ) { + return overviews_query_with_definite_zoom(query, overviews, zoom_level); + } else { + return overviews_query_with_zoom_expression(query, overviews, zoom_level_expression); + } +} + +function should_use_overviews(query, data) { + data = data || {}; + var use_overviews = data.overviews && is_supported_query(query); + if ( use_overviews && data.filters && data.filter_stats ) { + var filtered_rows = data.filter_stats.filtered_rows; + var unfiltered_rows = data.filter_stats.unfiltered_rows; + if ( unfiltered_rows && (filtered_rows || filtered_rows === 0) ) { + use_overviews = filtered_rows >= FILTER_MIN_ROWS || + (filtered_rows/unfiltered_rows) > FILTER_MAX_FRACTION; + } + } + return use_overviews; +} + +function is_supported_query(sql) { var basic_query = /\s*SELECT\s+[\*a-z0-9_,\s]+?\s+FROM\s+((\"[^"]+\"|[a-z0-9_]+)\.)?(\"[^"]+\"|[a-z0-9_]+)\s*;?\s*/i; var unwrapped_query = new RegExp("^"+basic_query.source+"$", 'i'); @@ -187,8 +310,4 @@ OverviewsQueryRewriter.prototype.is_supported_query = function(sql) { 'i' ); return !!(sql.match(unwrapped_query) || sql.match(wrapped_query)); -}; - -OverviewsQueryRewriter.prototype.overviews_metadata = function(data) { - return data && data.overviews; -}; +} diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index c3b9529d..04f931c5 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -1,6 +1,6 @@ { "name": "windshaft-cartodb", - "version": "2.42.3", + "version": "2.44.0", "dependencies": { "body-parser": { "version": "1.14.2", diff --git a/package.json b/package.json index 5ce84b37..c426011b 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "2.42.3", + "version": "2.44.0", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" diff --git a/test/acceptance/dataviews/overviews.js b/test/acceptance/dataviews/overviews.js new file mode 100644 index 00000000..601124a7 --- /dev/null +++ b/test/acceptance/dataviews/overviews.js @@ -0,0 +1,358 @@ +require('../../support/test_helper'); + +var assert = require('../../support/assert'); +var TestClient = require('../../support/test-client'); + +describe('dataviews using tables without overviews', function() { + + var nonOverviewsMapConfig = { + version: '1.5.0', + analyses: [ + { id: 'data-source', + type: 'source', + params: { + query: 'select * from populated_places_simple_reduced' + } + } + ], + dataviews: { + country_places_count: { + type: 'formula', + source: {id: 'data-source'}, + options: { + column: 'adm0_a3', + operation: 'count' + } + }, + country_categories: { + type: 'aggregation', + source: {id: 'data-source'}, + options: { + column: 'adm0_a3', + aggregation: 'count' + } + } + }, + layers: [ + { + type: 'mapnik', + options: { + sql: 'select * from populated_places_simple_reduced', + cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }', + cartocss_version: '2.3.0', + source: { id: 'data-source' } + } + } + ] + }; + + it("should expose a formula", function(done) { + var testClient = new TestClient(nonOverviewsMapConfig); + testClient.getDataview('country_places_count', { own_filter: 0 }, function(err, formula_result) { + if (err) { + return done(err); + } + assert.deepEqual(formula_result, { operation: 'count', result: 7313, nulls: 0, type: 'formula' }); + + testClient.drain(done); + }); + }); + + it("should admit a bbox", function(done) { + var params = { + bbox: "-170,-80,170,80" + }; + var testClient = new TestClient(nonOverviewsMapConfig); + testClient.getDataview('country_places_count', params, function(err, formula_result) { + if (err) { + return done(err); + } + assert.deepEqual(formula_result, { operation: 'count', result: 7253, nulls: 0, type: 'formula' }); + + testClient.drain(done); + }); + }); + + describe('filters', function() { + + describe('category', function () { + + it("should expose a filtered formula", function (done) { + var params = { + filters: { + dataviews: {country_categories: {accept: ['CAN']}} + } + }; + var testClient = new TestClient(nonOverviewsMapConfig); + testClient.getDataview('country_places_count', params, function (err, formula_result) { + if (err) { + return done(err); + } + assert.deepEqual(formula_result, { operation: 'count', result: 256, nulls: 0, type: 'formula' }); + testClient.drain(done); + }); + }); + + it("should expose a filtered formula and admit a bbox", function (done) { + var params = { + filters: { + dataviews: {country_categories: {accept: ['CAN']}} + }, + bbox: "-170,-80,170,80" + }; + var testClient = new TestClient(nonOverviewsMapConfig); + testClient.getDataview('country_places_count', params, function (err, formula_result) { + if (err) { + return done(err); + } + assert.deepEqual(formula_result, { operation: 'count', result: 254, nulls: 0, type: 'formula' }); + testClient.drain(done); + }); + }); + }); + + }); +}); + +describe('dataviews using tables with overviews', function() { + + var overviewsMapConfig = { + version: '1.5.0', + analyses: [ + { id: 'data-source', + type: 'source', + params: { + query: 'select * from test_table_overviews' + } + } + ], + dataviews: { + test_sum: { + type: 'formula', + source: {id: 'data-source'}, + options: { + column: 'value', + operation: 'sum' + } + }, + test_categories: { + type: 'aggregation', + source: {id: 'data-source'}, + options: { + column: 'name', + aggregation: 'count', + aggregationColumn: 'name', + } + }, + test_avg: { + type: 'formula', + source: {id: 'data-source'}, + options: { + column: 'value', + operation: 'avg' + } + }, + test_count: { + type: 'formula', + source: {id: 'data-source'}, + options: { + column: 'value', + operation: 'count' + } + }, + test_min: { + type: 'formula', + source: {id: 'data-source'}, + options: { + column: 'value', + operation: 'min' + } + }, + test_max: { + type: 'formula', + source: {id: 'data-source'}, + options: { + column: 'value', + operation: 'max' + } + } + }, + layers: [ + { + type: 'mapnik', + options: { + sql: 'select * from test_table_overviews', + cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }', + cartocss_version: '2.3.0', + source: { id: 'data-source' } + } + } + ] + }; + + it("should expose a sum formula", function(done) { + var testClient = new TestClient(overviewsMapConfig); + testClient.getDataview('test_sum', { own_filter: 0 }, function(err, formula_result) { + if (err) { + return done(err); + } + assert.deepEqual(formula_result, {"operation":"sum","result":15,"nulls":0,"type":"formula"}); + + testClient.drain(done); + }); + }); + + it("should expose an avg formula", function(done) { + var testClient = new TestClient(overviewsMapConfig); + testClient.getDataview('test_avg', { own_filter: 0 }, function(err, formula_result) { + if (err) { + return done(err); + } + assert.deepEqual(formula_result, {"operation":"avg","result":3,"nulls":0,"type":"formula"}); + + testClient.drain(done); + }); + }); + + it("should expose a count formula", function(done) { + var testClient = new TestClient(overviewsMapConfig); + testClient.getDataview('test_count', { own_filter: 0 }, function(err, formula_result) { + if (err) { + return done(err); + } + assert.deepEqual(formula_result, {"operation":"count","result":5,"nulls":0,"type":"formula"}); + + testClient.drain(done); + }); + }); + + it("should expose a max formula", function(done) { + var testClient = new TestClient(overviewsMapConfig); + testClient.getDataview('test_max', { own_filter: 0 }, function(err, formula_result) { + if (err) { + return done(err); + } + assert.deepEqual(formula_result, {"operation":"max","result":5,"nulls":0,"type":"formula"}); + + testClient.drain(done); + }); + }); + + it("should expose a min formula", function(done) { + var testClient = new TestClient(overviewsMapConfig); + testClient.getDataview('test_min', { own_filter: 0 }, function(err, formula_result) { + if (err) { + return done(err); + } + assert.deepEqual(formula_result, {"operation":"min","result":1,"nulls":0,"type":"formula"}); + + testClient.drain(done); + }); + }); + + it("should admit a bbox", function(done) { + var params = { + bbox: "-170,-80,170,80" + }; + var testClient = new TestClient(overviewsMapConfig); + testClient.getDataview('test_sum', params, function(err, formula_result) { + if (err) { + return done(err); + } + assert.deepEqual(formula_result, {"operation":"sum","result":15,"nulls":0,"type":"formula"}); + + testClient.drain(done); + }); + }); + + describe('filters', function() { + + describe('category', function () { + + var params = { + filters: { + dataviews: {test_categories: {accept: ['Hawai']}} + } + }; + + it("should expose a filtered sum formula", function (done) { + var testClient = new TestClient(overviewsMapConfig); + testClient.getDataview('test_sum', params, function (err, formula_result) { + if (err) { + return done(err); + } + assert.deepEqual(formula_result, {"operation":"sum","result":1,"nulls":0,"type":"formula"}); + testClient.drain(done); + }); + }); + + it("should expose a filtered avg formula", function(done) { + var testClient = new TestClient(overviewsMapConfig); + testClient.getDataview('test_avg', params, function(err, formula_result) { + if (err) { + return done(err); + } + assert.deepEqual(formula_result, {"operation":"avg","result":1,"nulls":0,"type":"formula"}); + + testClient.drain(done); + }); + }); + + it("should expose a filtered count formula", function(done) { + var testClient = new TestClient(overviewsMapConfig); + testClient.getDataview('test_count', params, function(err, formula_result) { + if (err) { + return done(err); + } + assert.deepEqual(formula_result, {"operation":"count","result":1,"nulls":0,"type":"formula"}); + + testClient.drain(done); + }); + }); + + it("should expose a filterd max formula", function(done) { + var testClient = new TestClient(overviewsMapConfig); + testClient.getDataview('test_max', params, function(err, formula_result) { + if (err) { + return done(err); + } + assert.deepEqual(formula_result, {"operation":"max","result":1,"nulls":0,"type":"formula"}); + + testClient.drain(done); + }); + }); + + it("should expose a filterd min formula", function(done) { + var testClient = new TestClient(overviewsMapConfig); + testClient.getDataview('test_min', params, function(err, formula_result) { + if (err) { + return done(err); + } + assert.deepEqual(formula_result, {"operation":"min","result":1,"nulls":0,"type":"formula"}); + + testClient.drain(done); + }); + }); + + it("should expose a filtered sum formula with bbox", function (done) { + var bboxparams = { + filters: { + dataviews: {test_categories: {accept: ['Hawai']}} + }, + bbox: "-170,-80,170,80" + }; + var testClient = new TestClient(overviewsMapConfig); + testClient.getDataview('test_sum', bboxparams, function (err, formula_result) { + if (err) { + return done(err); + } + assert.deepEqual(formula_result, {"operation":"sum","result":1,"nulls":0,"type":"formula"}); + testClient.drain(done); + }); + }); + + + }); + + }); +}); diff --git a/test/acceptance/overviews_metadata.js b/test/acceptance/overviews_metadata.js index 08b9cb59..0822cfe8 100644 --- a/test/acceptance/overviews_metadata.js +++ b/test/acceptance/overviews_metadata.js @@ -109,3 +109,118 @@ describe('overviews metadata', function() { ); }); }); + +describe('overviews metadata with filters', function() { + // configure redis pool instance to use in tests + var redisPool = new RedisPool(global.environment.redis); + + var keysToDelete; + + beforeEach(function() { + keysToDelete = {}; + }); + + afterEach(function(done) { + test_helper.deleteRedisKeys(keysToDelete, done); + }); + + it("layers with overviews", function(done) { + + var layergroup = { + version: '1.5.0', + layers: [ + { + type: 'cartodb', + options: { + sql: 'SELECT * FROM test_table_overviews', + source: { id: 'with_overviews' }, + cartocss: '#layer { marker-fill: black; }', + cartocss_version: '2.3.0' + } + } + ], + dataviews: { + test_names: { + type: 'aggregation', + source: {id: 'with_overviews'}, + options: { + column: 'name', + aggregation: 'count' + } + } + }, + analyses: [ + { id: 'with_overviews', + type: 'source', + params: { + query: 'select * from test_table_overviews' + } + } + ] + }; + + var filters = { + dataviews: { + test_names: { accept: ['Hawai'] } + } + }; + + var layergroup_url = '/api/v1/map'; + + var expected_token; + step( + function do_post() + { + var next = this; + assert.response(server, { + url: layergroup_url + '?filters=' + JSON.stringify(filters), + method: 'POST', + headers: {host: 'localhost', 'Content-Type': 'application/json' }, + data: JSON.stringify(layergroup) + }, {}, function(res) { + assert.equal(res.statusCode, 200, res.body); + var parsedBody = JSON.parse(res.body); + assert.equal(res.headers['x-layergroup-id'], parsedBody.layergroupid); + expected_token = parsedBody.layergroupid; + next(null, res); + }); + }, + function do_get_mapconfig(err) + { + assert.ifError(err); + var next = this; + + var mapStore = new windshaft.storage.MapStore({ + pool: redisPool, + expire_time: 500000 + }); + mapStore.load(LayergroupToken.parse(expected_token).token, function(err, mapConfig) { + assert.ifError(err); + assert.equal(mapConfig._cfg.layers[0].type, 'cartodb'); + assert.ok(mapConfig._cfg.layers[0].options.query_rewrite_data); + var expected_data = { + overviews: { + test_table_overviews: { + schema: 'public', + 1: { table: '_vovw_1_test_table_overviews' }, + 2: { table: '_vovw_2_test_table_overviews' } + } + }, + filters: { test_names: { type: 'category', column: 'name', params: { accept: [ 'Hawai' ] } } }, + unfiltered_query: 'select * from test_table_overviews', + filter_stats: { unfiltered_rows: 5, filtered_rows: 1 } + }; + assert.deepEqual(mapConfig._cfg.layers[0].options.query_rewrite_data, expected_data); + + }); + + next(err); + }, + function finish(err) { + keysToDelete['map_cfg|' + LayergroupToken.parse(expected_token).token] = 0; + keysToDelete['user:localhost:mapviews:global'] = 5; + done(err); + } + ); + }); +}); diff --git a/test/acceptance/sql-wrap.js b/test/acceptance/sql-wrap.js new file mode 100644 index 00000000..7b3c25d8 --- /dev/null +++ b/test/acceptance/sql-wrap.js @@ -0,0 +1,52 @@ +require('../support/test_helper'); + +var assert = require('../support/assert'); +var TestClient = require('../support/test-client'); + +describe('sql-wrap', function() { + + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + return done(); + } + }); + + it('should use sql_wrap from layer options', function(done) { + var mapConfig = { + version: '1.5.0', + layers: [ + { + "type": "cartodb", + "options": { + "sql": "SELECT * FROM populated_places_simple_reduced", + "sql_wrap": "SELECT * FROM (<%= sql %>) _w WHERE adm0_a3 = 'USA'", + "cartocss": [ + "#points {", + " marker-fill-opacity: 1;", + " marker-line-color: #FFF;", + " marker-line-width: 0.5;", + " marker-line-opacity: 1;", + " marker-placement: point;", + " marker-type: ellipse;", + " marker-width: 8;", + " marker-fill: red;", + " marker-allow-overlap: true;", + "}" + ].join('\n'), + "cartocss_version": "2.3.0" + } + } + ] + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getTile(0, 0, 0, function(err, tile, img) { + assert.ok(!err, err); + var fixtureImg = './test/fixtures/sql-wrap-usa-filter.png'; + assert.imageIsSimilarToFile(img, fixtureImg, 20, done); + }); + }); + +}); diff --git a/test/fixtures/sql-wrap-usa-filter.png b/test/fixtures/sql-wrap-usa-filter.png new file mode 100644 index 00000000..30bd4eb3 Binary files /dev/null and b/test/fixtures/sql-wrap-usa-filter.png differ diff --git a/test/integration/mapconfig_overviews_adapter.js b/test/integration/mapconfig_overviews_adapter.js index ebc70635..d168c02f 100644 --- a/test/integration/mapconfig_overviews_adapter.js +++ b/test/integration/mapconfig_overviews_adapter.js @@ -6,6 +6,7 @@ var cartodbRedis = require('cartodb-redis'); var PgConnection = require(__dirname + '/../../lib/cartodb/backends/pg_connection'); var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner'); var OverviewsMetadataApi = require('../../lib/cartodb/api/overviews_metadata_api'); +var FilterStatsApi = require('../../lib/cartodb/api/filter_stats_api'); var MapConfigOverviewsAdapter = require('../../lib/cartodb/models/mapconfig_overviews_adapter'); // configure redis pool instance to use in tests @@ -17,9 +18,9 @@ var metadataBackend = cartodbRedis({pool: redisPool}); var pgConnection = new PgConnection(metadataBackend); var pgQueryRunner = new PgQueryRunner(pgConnection); var overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner); +var filterStatsApi = new FilterStatsApi(pgQueryRunner); - -var mapConfigOverviewsAdapter = new MapConfigOverviewsAdapter(overviewsMetadataApi); +var mapConfigOverviewsAdapter = new MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi); describe('MapConfigOverviewsAdapter', function() { @@ -36,7 +37,7 @@ describe('MapConfigOverviewsAdapter', function() { } }; - mapConfigOverviewsAdapter.getLayers('localhost', [layer_without_overviews], function(err, layers) { + mapConfigOverviewsAdapter.getLayers('localhost', [layer_without_overviews], [], function(err, layers) { assert.ok(!err); assert.equal(layers.length, 1); assert.equal(layers[0].type, 'cartodb'); @@ -64,7 +65,7 @@ describe('MapConfigOverviewsAdapter', function() { } }; - mapConfigOverviewsAdapter.getLayers('localhost', [layer_without_overviews], function(err, layers) { + mapConfigOverviewsAdapter.getLayers('localhost', [layer_without_overviews], [], function(err, layers) { assert.ok(!err); assert.equal(layers.length, 1); assert.equal(layers[0].type, 'cartodb'); diff --git a/test/support/sql/windshaft.test.sql b/test/support/sql/windshaft.test.sql index 4690f081..7920a88d 100644 --- a/test/support/sql/windshaft.test.sql +++ b/test/support/sql/windshaft.test.sql @@ -247,8 +247,10 @@ CREATE TABLE test_table_overviews ( cartodb_id integer NOT NULL, name character varying, address character varying, + value float8, the_geom geometry, the_geom_webmercator geometry, + _feature_count integer, CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)), CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)), CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))), @@ -274,11 +276,11 @@ SELECT pg_catalog.setval('test_table_overviews_cartodb_id_seq', 60, true); ALTER TABLE test_table_overviews ALTER COLUMN cartodb_id SET DEFAULT nextval('test_table_overviews_cartodb_id_seq'::regclass); INSERT INTO test_table_overviews VALUES -('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241'), -('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241'), -('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.324', 3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241'), -('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.329509', 4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241'), -('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.334931', 5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241'); +('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', 1.0, '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241', 1), +('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', 2.0, '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241', 1), +('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.324', 3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', 3.0, '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241', 1), +('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.329509', 4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', 4.0, '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241', 1), +('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.334931', 5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', 5.0, '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241', 1); ALTER TABLE ONLY test_table_overviews ADD CONSTRAINT test_table_overviews_pkey PRIMARY KEY (cartodb_id); @@ -294,8 +296,10 @@ CREATE TABLE _vovw_1_test_table_overviews ( cartodb_id integer NOT NULL, name character varying, address character varying, + value float8, the_geom geometry, the_geom_webmercator geometry, + _feature_count integer, CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)), CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)), CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))), @@ -313,8 +317,10 @@ CREATE TABLE _vovw_2_test_table_overviews ( cartodb_id integer NOT NULL, name character varying, address character varying, + value float8, the_geom geometry, the_geom_webmercator geometry, + _feature_count integer, CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)), CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)), CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))), @@ -327,11 +333,11 @@ GRANT ALL ON TABLE _vovw_2_test_table_overviews TO :TESTUSER; GRANT SELECT ON TABLE _vovw_2_test_table_overviews TO :PUBLICUSER; INSERT INTO _vovw_2_test_table_overviews VALUES -('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241'), -('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E610000000000000009431C026043C75E7224340', '0101000020110F0000C4356B29423319C15DD1092DADCC5241'); +('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', 8.0/3.0, '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241', 3), +('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', 7.0/2.0, '0101000020E610000000000000009431C026043C75E7224340', '0101000020110F0000C4356B29423319C15DD1092DADCC5241', 2); INSERT INTO _vovw_1_test_table_overviews VALUES -('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241'); +('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', 3.0, '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241', 5); -- analysis tables ----------------------------------------------- diff --git a/test/unit/cartodb/overviews_query_rewriter.js b/test/unit/cartodb/overviews_query_rewriter.js index 0e40ef92..dd6b0798 100644 --- a/test/unit/cartodb/overviews_query_rewriter.js +++ b/test/unit/cartodb/overviews_query_rewriter.js @@ -475,4 +475,137 @@ describe('Overviews query rewriter', function() { "; assertSameSql(overviews_sql, expected_sql); }); + + it('generates query for specific Z level', function(){ + var sql = "SELECT * FROM table1"; + var data = { + overviews: { + table1: { + 0: { table: 'table1_ov0' }, + 2: { table: 'table1_ov2' }, + 3: { table: 'table1_ov3' } + } + } + }; + var overviews_sql = overviewsQueryRewriter.query(sql, data, { zoom_level: 3 }); + var expected_sql = "SELECT * FROM table1_ov3"; + assertSameSql(overviews_sql, expected_sql); + }); + + it('generates query for specific nonpresent Z level', function(){ + var sql = "SELECT * FROM table1"; + var data = { + overviews: { + table1: { + 0: { table: 'table1_ov0' }, + 2: { table: 'table1_ov2' }, + 3: { table: 'table1_ov3' } + } + } + }; + var overviews_sql = overviewsQueryRewriter.query(sql, data, { zoom_level: 1 }); + var expected_sql = "SELECT * FROM table1_ov2"; + assertSameSql(overviews_sql, expected_sql); + }); + + it('does not use overviews for specific out-of-range Z level', function(){ + var sql = "SELECT * FROM table1"; + var data = { + overviews: { + table1: { + 0: { table: 'table1_ov0' }, + 2: { table: 'table1_ov2' }, + 3: { table: 'table1_ov3' } + } + } + }; + var overviews_sql = overviewsQueryRewriter.query(sql, data, { zoom_level: 4 }); + var expected_sql = "SELECT * FROM table1"; + assertSameSql(overviews_sql, expected_sql); + }); + + it('generates query with filters', function(){ + var sql = "SELECT ST_Transform(the_geom, 3857) the_geom_webmercator, cartodb_id, name\ + FROM (SELECT *\ + FROM (select * from table1) _camshaft_category_filter\ + WHERE name IN ($escape_0$X$escape_0$)) _cdb_analysis_query"; + var data = { + overviews: { + table1: { + 0: { table: 'table1_ov0' }, + 1: { table: 'table1_ov1' }, + 2: { table: 'table1_ov2' }, + 3: { table: 'table1_ov3' } + } + }, + filters: { name_filter: { type: 'category', column: 'name', params: { accept: [ 'X' ] } } }, + unfiltered_query: 'SELECT * FROM table1' + }; + var overviews_sql = overviewsQueryRewriter.query(sql, data); + var expected_sql = "\ + SELECT * FROM (WITH\ + _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\ + SELECT * FROM (\ + SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\ + UNION ALL\ + SELECT * FROM table1_ov1, _vovw_scale WHERE _vovw_z = 1\ + UNION ALL\ + SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z = 2\ + UNION ALL\ + SELECT * FROM table1_ov3, _vovw_scale WHERE _vovw_z = 3\ + UNION ALL\ + SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 3\ + ) AS _vovw_table1) _camshaft_category_filter\ + WHERE name IN ($escape_0$X$escape_0$)\ + "; + assertSameSql(overviews_sql, expected_sql); + }); + + it('generates query with filters for specific zoom level', function(){ + var sql = "SELECT ST_Transform(the_geom, 3857) the_geom_webmercator, cartodb_id, name\ + FROM (SELECT *\ + FROM (select * from table1) _camshaft_category_filter\ + WHERE name IN ($escape_0$X$escape_0$)) _cdb_analysis_query"; + var data = { + overviews: { + table1: { + 0: { table: 'table1_ov0' }, + 1: { table: 'table1_ov1' }, + 2: { table: 'table1_ov2' }, + 3: { table: 'table1_ov3' } + } + }, + filters: { name_filter: { type: 'category', column: 'name', params: { accept: [ 'X' ] } } }, + unfiltered_query: 'SELECT * FROM table1', + filter_stats: { unfiltered_rows: 1000, filtered_rows: 900 } + }; + var overviews_sql = overviewsQueryRewriter.query(sql, data, { zoom_level: 2 }); + var expected_sql = "\ + SELECT * FROM (SELECT * FROM table1_ov2) _camshaft_category_filter\ + WHERE name IN ($escape_0$X$escape_0$)\ + "; + assertSameSql(overviews_sql, expected_sql); + }); + + it('does not generates query with aggressive filtering', function(){ + var sql = "SELECT ST_Transform(the_geom, 3857) the_geom_webmercator, cartodb_id, name\ + FROM (SELECT *\ + FROM (select * from table1) _camshaft_category_filter\ + WHERE name IN ($escape_0$X$escape_0$)) _cdb_analysis_query"; + var data = { + overviews: { + table1: { + 0: { table: 'table1_ov0' }, + 1: { table: 'table1_ov1' }, + 2: { table: 'table1_ov2' }, + 3: { table: 'table1_ov3' } + } + }, + filters: { name_filter: { type: 'category', column: 'name', params: { accept: [ 'X' ] } } }, + unfiltered_query: 'SELECT * FROM table1', + filter_stats: { unfiltered_rows: 1000, filtered_rows: 10 } + }; + var overviews_sql = overviewsQueryRewriter.query(sql, data); + assert.equal(overviews_sql, sql); + }); });