diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index a6b5f549..84a47cf5 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -8,7 +8,9 @@ var step = require('step'); var Timer = require('../stats/timer'); var BBoxFilter = require('../models/filter/bbox'); + var DataviewFactory = require('../models/dataview/factory'); +var DataviewFactoryWithOverviews = require('../models/dataview/overviews/factory'); var OverviewsQueryRewriter = require('../utils/overviews_query_rewriter'); var overviewsQueryRewriter = new OverviewsQueryRewriter({ zoom_level: 'CDB_ZoomFromScale(!scale_denominator!)' @@ -132,16 +134,15 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param {}, queryRewriteData, { filters: filters, unfiltered_query: unfiltered_query } ); - query = overviewsQueryRewriter.query(query, queryRewriteData, { bounding_box: params.bbox }); } } + var dataviewFactory = DataviewFactoryWithOverviews.getFactory(overviewsQueryRewriter, queryRewriteData); if (params.bbox) { var bboxFilter = new BBoxFilter({column: 'the_geom', srid: 4326}, {bbox: params.bbox}); query = bboxFilter.sql(query); } - var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins'), function castNumbers(overrides, val, k) { overrides[k] = Number.isFinite(+val) ? +val : val; @@ -150,7 +151,7 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param {ownFilter: ownFilter} ); - var dataview = DataviewFactory.getDataview(query, dataviewDefinition); + var dataview = dataviewFactory.getDataview(query, dataviewDefinition); dataview.getResult(pg, overrideParams, this); }, function returnCallback(err, result) { @@ -253,6 +254,8 @@ DataviewBackend.prototype.search = function (mapConfigProvider, user, params, ca query = node.getQuery(applyFilters); } + // TODO: should handle overviews as getDataview ? + if (params.bbox) { var bboxFilter = new BBoxFilter({column: 'the_geom', srid: 4326}, {bbox: params.bbox}); query = bboxFilter.sql(query); diff --git a/lib/cartodb/models/dataview/overviews/factory.js b/lib/cartodb/models/dataview/overviews/factory.js new file mode 100644 index 00000000..b5ce8dd3 --- /dev/null +++ b/lib/cartodb/models/dataview/overviews/factory.js @@ -0,0 +1,29 @@ +var parentFactory = require('../factory'); +var dataviews = require('./'); + +function OverviewsDataviewFactory(queryRewriter, queryRewriteData) { + this.queryRewriter = queryRewriter; + this.queryRewriteData = queryRewriteData; +} + +OverviewsDataviewFactory.prototype.getDataview = function(query, dataviewDefinition) { + var type = dataviewDefinition.type; + var dataviews = OverviewsDataviewMetaFactory.dataviews; + if ( !this.queryRewriter || !this.queryRewriteData || !dataviews[type] ) { + return parentFactory.getDataview(query, dataviewDefinition); + } + return new dataviews[type](query, dataviewDefinition.options, this.queryRewriter, this.queryRewriteData); +}; + +var OverviewsDataviewMetaFactory = { + dataviews: Object.keys(dataviews).reduce(function(allDataviews, dataviewClassName) { + allDataviews[dataviewClassName.toLowerCase()] = dataviews[dataviewClassName]; + return allDataviews; + }, {}), + + getFactory: function(queryRewriter, queryRewriteData) { + return new OverviewsDataviewFactory(queryRewriter, queryRewriteData); + }, +}; + +module.exports = OverviewsDataviewMetaFactory; diff --git a/lib/cartodb/models/dataview/overviews/formula.js b/lib/cartodb/models/dataview/overviews/formula.js new file mode 100644 index 00000000..1144f5dd --- /dev/null +++ b/lib/cartodb/models/dataview/overviews/formula.js @@ -0,0 +1,84 @@ +var BaseWidget = require('../base'); +var BaseDataview = require('../formula'); + +var debug = require('debug')('windshaft:widget:formula:overviews'); + +var dot = require('dot'); +dot.templateSettings.strip = false; + +var formulaQueryTpls = { + 'count': dot.template([ + 'SELECT', + 'sum(_feature_count) AS result,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', + 'FROM ({{=it._query}}) _cdb_formula' + ].join('\n')), + 'sum': dot.template([ + 'SELECT', + 'sum({{=it._column}}*_feature_count) AS result,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', + 'FROM ({{=it._query}}) _cdb_formula' + ].join('\n')), + 'avg': dot.template([ + 'SELECT', + 'sum({{=it._column}}*_feature_count)/sum(_feature_count) AS result,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', + 'FROM ({{=it._query}}) _cdb_formula' + ].join('\n')), +}; + +function Formula(query, options, queryRewriter, queryRewriteData) { + this.base_dataview = new BaseDataview(query, options); + this.query = query; + this.column = options.column || '1'; + this.operation = options.operation; + this.queryRewriter = queryRewriter; + this.queryRewriteData = queryRewriteData; +} + +Formula.prototype = new BaseWidget(); +Formula.prototype.constructor = Formula; + +module.exports = Formula; + +Formula.prototype.sql = function(psql, filters, override, callback) { + var _query = this.query; + var formulaQueryTpl = formulaQueryTpls[this.operation]; + + if ( formulaQueryTpl ) { + // supported formula for use with overviews + + // TODO: determine zoom level using bounding box so that the resolution + // (grid size) of the overview much smaller thatn the bounding box. + // This could be left to be computed by the queryRewriter passing the + // bounding box to it. + // The bounding box could be passed to the dataview constructor, or + // it could b extracted from the query. + var zoom_level = 0; + + _query = this.queryRewriter.query(_query, this.queryRewriteData, { zoom_level: zoom_level }); + var formulaSql = formulaQueryTpl({ + _query: _query, + _operation: this.operation, + _column: this.column + }); + debug(formulaSql); + callback = callback || override; + return callback(null, formulaSql); + } + + // For non supported operations (min, max) we're not using overviews. + return this.base_dataview.sql(psql, filters, override, callback); +}; + +Formula.prototype.format = function(result) { + return this.base_dataview.format(result); +}; + +Formula.prototype.getType = function() { + return this.base_dataview.getType(); +}; + +Formula.prototype.toString = function() { + return this.base_dataview.toString(); +}; diff --git a/lib/cartodb/models/dataview/overviews/index.js b/lib/cartodb/models/dataview/overviews/index.js new file mode 100644 index 00000000..775ceca1 --- /dev/null +++ b/lib/cartodb/models/dataview/overviews/index.js @@ -0,0 +1,3 @@ +module.exports = { + Formula: require('./formula') +}; diff --git a/lib/cartodb/utils/overviews_query_rewriter.js b/lib/cartodb/utils/overviews_query_rewriter.js index bf20d6c9..4999073b 100644 --- a/lib/cartodb/utils/overviews_query_rewriter.js +++ b/lib/cartodb/utils/overviews_query_rewriter.js @@ -1,9 +1,15 @@ var _ = require('underscore'); var TableNameParser = require('./table_name_parser'); -// this is meant as a hack for development (otherwise we'd be coupling -// this to Camshaft impolementation) +// this is meant as a hack for development var queryBuilder = require('camshaft/lib/filter/query-builder'); +function apply_filters_to_query(query, filters) { + // TODO: implement filter application here + if ( filters && !_.isEmpty(filters)) { + query = queryBuilder.getSql(query, filters); + } + return query; +} function OverviewsQueryRewriter(options) { @@ -196,20 +202,6 @@ function overview_table_for_zoom_level(table_overviews, zoom_level) { return overview_table; } -function apply_filters_to_query(query, filters) { - // TODO: it'd be nice to have a public API in camshaft to apply filters! - if ( filters && !_.isEmpty(filters)) { - query = queryBuilder.getSql(query, filters); - } - return query; -} - -function zoom_level_for_bounding_box(query, bounding_box) { - // TODO: implement - // jshint unused:false - return '100'; // force use of base table -} - // Transform an SQL query so that it uses overviews. // overviews contains metadata about the overviews to be used: // { 'table-name': {1: { table: 'overview-table-1' }, ... }, ... } @@ -228,7 +220,7 @@ function zoom_level_for_bounding_box(query, bounding_box) { // doesnn't prevent substitution inside literals). // But the transformation will currently only be applied to simple queries // of the form detected by the overviews_supported_query function. -// TODO: document recent changes (options bounding_box, zoom_level, filters...) +// TODO: document recent changes (zoom_level option, filters...) OverviewsQueryRewriter.prototype.query = function(query, data, options) { options = options || {}; @@ -255,6 +247,11 @@ OverviewsQueryRewriter.prototype.query = function(query, data, options) { // could not or didn't need to alter the query rewritten_query = query; } else { + // TODO: determine if overviews can be used for the filters. + // Otherwise use the base table. + // Applicability of the filters should depend both on the expected + // row count of the result and on the columns to be filter and the + // way they're agregated in overviews. rewritten_query = apply_filters_to_query(rewritten_query, filters); } @@ -263,12 +260,8 @@ OverviewsQueryRewriter.prototype.query = function(query, data, options) { function zoom_level_for_query(query, zoom_level_expression, options) { var zoom_level = null; - if ( _.has(options, 'bounding_box') ) { - // The presence of this key, even if it has a null value indicates the - // query cannot handle dynamic zoom expressinos (is not for the tiler) - zoom_level = zoom_level_for_bounding_box(query, options.bounding_box); - } else if ( _.has(options, 'zoom_level') ) { - zoom_level = options.zoom_level; + if ( _.has(options, 'zoom_level') ) { + zoom_level = options.zoom_level || '0'; } if ( zoom_level === null && !zoom_level_expression ) { zoom_level = '0';