diff --git a/lib/cartodb/models/dataview/overviews/aggregation.js b/lib/cartodb/models/dataview/overviews/aggregation.js new file mode 100644 index 00000000..41685ff5 --- /dev/null +++ b/lib/cartodb/models/dataview/overviews/aggregation.js @@ -0,0 +1,141 @@ +var BaseOverviewsDataview = require('./base'); +var BaseDataview = require('../aggregation'); + +var dot = require('dot'); +dot.templateSettings.strip = false; + +var summaryQueryTpl = dot.template([ + 'summary AS (', + ' SELECT', + ' sum(_feature_count) AS count,', + ' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count', + ' FROM ({{=it._query}}) _cdb_aggregation_nulls', + ')' +].join('\n')); + +var rankedCategoriesQueryTpl = dot.template([ + 'categories AS(', + ' SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,', + ' row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank', + ' FROM ({{=it._query}}) _cdb_aggregation_all', + ' GROUP BY {{=it._column}}', + ' ORDER BY 2 DESC', + ')' +].join('\n')); + +var categoriesSummaryQueryTpl = dot.template([ + 'categories_summary AS(', + ' SELECT count(1) categories_count, max(value) max_val, min(value) min_val', + ' FROM categories', + ')' +].join('\n')); + +var rankedAggregationQueryTpl = dot.template([ + 'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count', + ' FROM categories, summary, categories_summary', + ' WHERE rank < {{=it._limit}}', + 'UNION ALL', + 'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val, count, categories_count', + ' FROM categories, summary, categories_summary', + ' WHERE rank >= {{=it._limit}}', + 'GROUP BY nulls_count, min_val, max_val, count, categories_count' +].join('\n')); + +var aggregationQueryTpl = dot.template([ + 'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,', + ' nulls_count, min_val, max_val, count, categories_count', + 'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary', + 'GROUP BY category, nulls_count, min_val, max_val, count, categories_count', + 'ORDER BY value DESC' +].join('\n')); + +var CATEGORIES_LIMIT = 6; + + function Aggregation(query, options, queryRewriter, queryRewriteData, params) { + BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); + + this.query = query; + this.column = options.column; + this.aggregation = options.aggregation; + this.aggregationColumn = options.aggregationColumn; +} + +Aggregation.prototype = Object.create(BaseOverviewsDataview.prototype); +Aggregation.prototype.constructor = Aggregation; + +module.exports = Aggregation; + +Aggregation.prototype.sql = function(psql, filters, override, callback) { + if (!callback) { + callback = override; + override = {}; + } + + var _query = this.rewrittenQuery(this.query); + + var aggregationSql; + if (!!override.ownFilter) { + aggregationSql = [ + "WITH", + [ + summaryQueryTpl({ + _query: _query, + _column: this.column + }), + rankedCategoriesQueryTpl({ + _query: _query, + _column: this.column, + _aggregation: this.getAggregationSql() + }), + categoriesSummaryQueryTpl({ + _query: _query, + _column: this.column + }) + ].join(',\n'), + aggregationQueryTpl({ + _query: _query, + _column: this.column, + _aggregation: this.getAggregationSql(), + _limit: CATEGORIES_LIMIT + }) + ].join('\n'); + } else { + aggregationSql = [ + "WITH", + [ + summaryQueryTpl({ + _query: _query, + _column: this.column + }), + rankedCategoriesQueryTpl({ + _query: _query, + _column: this.column, + _aggregation: this.getAggregationSql() + }), + categoriesSummaryQueryTpl({ + _query: _query, + _column: this.column + }) + ].join(',\n'), + rankedAggregationQueryTpl({ + _query: _query, + _column: this.column, + _limit: CATEGORIES_LIMIT + }) + ].join('\n'); + } + + return callback(null, aggregationSql); +}; + +var aggregationFnQueryTpl = { + count: dot.template('sum(_feature_count)'), + sum: dot.template('sum({{=it._aggregationColumn}}*_feature_count)') +}; + +Aggregation.prototype.getAggregationSql = function() { + return aggregationFnQueryTpl[this.aggregation]({ + _aggregationFn: this.aggregation, + _aggregationColumn: this.aggregationColumn || 1 + }); +}; diff --git a/lib/cartodb/models/dataview/overviews/base.js b/lib/cartodb/models/dataview/overviews/base.js new file mode 100644 index 00000000..bbf70742 --- /dev/null +++ b/lib/cartodb/models/dataview/overviews/base.js @@ -0,0 +1,88 @@ +var _ = require('underscore'); +var BaseDataview = require('../base'); + +function BaseOverviewsDataview(query, queryOptions, BaseDataview, queryRewriter, queryRewriteData, options) { + this.BaseDataview = BaseDataview; + this.query = query; + this.queryOptions = queryOptions; + this.queryRewriter = queryRewriter; + this.queryRewriteData = queryRewriteData; + this.options = options; + this.baseDataview = new this.BaseDataview(this.query, this.queryOptions); +} + +module.exports = BaseOverviewsDataview; + +BaseOverviewsDataview.prototype = new BaseDataview(); +BaseOverviewsDataview.prototype.constructor = BaseOverviewsDataview; + +// TODO: parameterized these settings +var SETTINGS = { + // use overviews as a default fallback strategy + defaultOverviews: false, + + // minimum ratio of bounding box size to grid size + // (this would ideally be based on the viewport size in pixels) + zoomLevelFactor: 1024.0 +}; + +// Compute zoom level so that the the resolution grid size of the +// selected overview is smaller (zoomLevelFactor times smaller at least) +// than the bounding box size. +BaseOverviewsDataview.prototype.zoomLevelForBbox = function(bbox) { + var pxPerTile = 256.0; + var earthWidth = 360.0; + // TODO: now we assume overviews are computed for 1-pixel tolerance; + // should use extended overviews metadata to compute this properly. + if ( bbox ) { + var bboxValues = _.map(bbox.split(','), function(v) { return +v; }); + var w = Math.abs(bboxValues[2]-bboxValues[0]); + var h = Math.abs(bboxValues[3]-bboxValues[1]); + var maxDim = Math.min(w, h); + + // Find minimum suitable z + // note that the QueryRewirter will use the minimum level overview + // of level >= z if it exists, and otherwise the base table + var z = Math.ceil(-Math.log(maxDim*pxPerTile/earthWidth/SETTINGS.zoomLevelFactor)/Math.log(2.0)); + return Math.max(z, 0); + } + return 0; +}; + +BaseOverviewsDataview.prototype.rewrittenQuery = function(query) { + var zoom_level = this.zoomLevelForBbox(this.options.bbox); + return this.queryRewriter.query(query, this.queryRewriteData, { zoom_level: zoom_level }); +}; + +// Default behaviour +BaseOverviewsDataview.prototype.defaultSql = function(psql, filters, override, callback) { + var query = this.query; + var dataview = this.baseDataview; + if ( SETTINGS.defaultOverviews ) { + query = this.rewrittenQuery(query); + dataview = new this.BaseDataview(query, this.queryOptions); + } + return dataview.sql(psql, filters, override, callback); +}; + +// default implementation that can be override in derived classes: + +BaseOverviewsDataview.prototype.sql = function(psql, filters, override, callback) { + return this.defaultSql(psql, filters, override, callback); +}; + +BaseOverviewsDataview.prototype.search = function(psql, userQuery, callback) { + return this.baseDataview.search(psql, userQuery, callback); +}; + +BaseOverviewsDataview.prototype.format = function(result) { + return this.baseDataview.format(result); +}; + +BaseOverviewsDataview.prototype.getType = function() { + return this.baseDataview.getType(); +}; + +BaseOverviewsDataview.prototype.toString = function() { + return this.baseDataview.toString(); +}; diff --git a/lib/cartodb/models/dataview/overviews/formula.js b/lib/cartodb/models/dataview/overviews/formula.js index aa57792e..adad5f58 100644 --- a/lib/cartodb/models/dataview/overviews/formula.js +++ b/lib/cartodb/models/dataview/overviews/formula.js @@ -1,9 +1,6 @@ -var _ = require('underscore'); -var BaseWidget = require('../base'); +var BaseOverviewsDataview = require('./base'); var BaseDataview = require('../formula'); -var debug = require('debug')('windshaft:widget:formula:overviews'); - var dot = require('dot'); dot.templateSettings.strip = false; @@ -29,76 +26,31 @@ var formulaQueryTpls = { }; function Formula(query, options, queryRewriter, queryRewriteData, params) { - this.base_dataview = new BaseDataview(query, options); - this.query = query; + BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); this.column = options.column || '1'; this.operation = options.operation; - this.queryRewriter = queryRewriter; - this.queryRewriteData = queryRewriteData; - this.options = params; } -Formula.prototype = new BaseWidget(); +Formula.prototype = Object.create(BaseOverviewsDataview.prototype); Formula.prototype.constructor = Formula; module.exports = Formula; -var zoom_level_factor = 100.0; - -// Compute zoom level so that the the resolution grid size of the -// selected overview is smaller (zoom_level_factor times smaller at least) -// than the bounding box size. -function zoom_level_for_bbox(bbox) { - var px_per_tile = 256.0; - var earth_width = 360.0; - // TODO: now we assume overviews are computed for 1-pixel tolerance; - // should use extended overviews metadata to compute this properly. - if ( bbox ) { - var bbox_values = _.map(bbox.split(','), function(v) { return +v; }); - var w = Math.abs(bbox_values[2]-bbox_values[0]); - var h = Math.abs(bbox_values[3]-bbox_values[1]); - var max_dim = Math.min(w, h); - - // Find minimum suitable z - // note that the QueryRewirter will use the minimum level overview - // of level >= z if it exists, and otherwise the base table - var z = Math.ceil(-Math.log(max_dim*px_per_tile/earth_width/zoom_level_factor)/Math.log(2.0)); - return Math.max(z, 0); - } - return 0; -} - Formula.prototype.sql = function(psql, filters, override, callback) { - var _query = this.query; var formulaQueryTpl = formulaQueryTpls[this.operation]; if ( formulaQueryTpl ) { // supported formula for use with overviews - var zoom_level = zoom_level_for_bbox(this.options.bbox); - _query = this.queryRewriter.query(_query, this.queryRewriteData, { zoom_level: zoom_level }); var formulaSql = formulaQueryTpl({ - _query: _query, + _query: this.rewrittenQuery(this.query), _operation: this.operation, - _column: this.column + _column: this.column }); - debug(formulaSql); callback = callback || override; return callback(null, formulaSql); } - // For non supported operations (min, max) we're not using overviews. - return this.base_dataview.sql(psql, filters, override, callback); -}; - -Formula.prototype.format = function(result) { - return this.base_dataview.format(result); -}; - -Formula.prototype.getType = function() { - return this.base_dataview.getType(); -}; - -Formula.prototype.toString = function() { - return this.base_dataview.toString(); + // default behaviour + return this.defaultSql(psql, filters, override, callback); }; diff --git a/lib/cartodb/models/dataview/overviews/histogram.js b/lib/cartodb/models/dataview/overviews/histogram.js new file mode 100644 index 00000000..20dd7712 --- /dev/null +++ b/lib/cartodb/models/dataview/overviews/histogram.js @@ -0,0 +1,217 @@ +var _ = require('underscore'); +var BaseOverviewsDataview = require('./base'); +var BaseDataview = require('../histogram'); + +var dot = require('dot'); +dot.templateSettings.strip = false; + +var columnTypeQueryTpl = dot.template( + 'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1' +); +var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})"); + +var BIN_MIN_NUMBER = 6; +var BIN_MAX_NUMBER = 48; + +var basicsQueryTpl = dot.template([ + 'basics AS (', + ' SELECT', + ' max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,', + ' sum({{=it._column}}*_feature_count)/sum(_feature_count) AS avg_val, sum(_feature_count) AS total_rows', + ' FROM ({{=it._query}}) _cdb_basics', + ')' +].join(' \n')); + +var overrideBasicsQueryTpl = dot.template([ + 'basics AS (', + ' SELECT', + ' max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,', + ' sum({{=it._column}}*_feature_count)/sum(_feature_count) AS avg_val, sum(_feature_count) AS total_rows', + ' FROM ({{=it._query}}) _cdb_basics', + ')' +].join('\n')); + +var iqrQueryTpl = dot.template([ + 'iqrange AS (', + ' SELECT max(quartile_max) - min(quartile_max) AS iqr', + ' FROM (', + ' SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (', + ' SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}', + ' ) AS quartile', + ' FROM ({{=it._query}}) _cdb_rank) _cdb_quartiles', + ' WHERE quartile = 1 or quartile = 3', + ' GROUP BY quartile', + ' ) _cdb_iqr', + ')' +].join('\n')); + +var binsQueryTpl = dot.template([ + 'bins AS (', + ' SELECT CASE WHEN total_rows = 0 OR iqr = 0', + ' THEN 1', + ' ELSE GREATEST(', + ' LEAST({{=it._minBins}}, CAST(total_rows AS INT)),', + ' LEAST(', + ' CAST(((max_val - min_val) / (2 * iqr * power(total_rows, 1/3))) AS INT),', + ' {{=it._maxBins}}', + ' )', + ' )', + ' END AS bins_number', + ' FROM basics, iqrange, ({{=it._query}}) _cdb_bins', + ' LIMIT 1', + ')' +].join('\n')); + +var overrideBinsQueryTpl = dot.template([ + 'bins AS (', + ' SELECT {{=it._bins}} AS bins_number', + ')' +].join('\n')); + +var nullsQueryTpl = dot.template([ + 'nulls AS (', + ' SELECT', + ' count(*) AS nulls_count', + ' FROM ({{=it._query}}) _cdb_histogram_nulls', + ' WHERE {{=it._column}} IS NULL', + ')' +].join('\n')); + +var histogramQueryTpl = dot.template([ + 'SELECT', + ' (max_val - min_val) / cast(bins_number as float) AS bin_width,', + ' bins_number,', + ' nulls_count,', + ' avg_val,', + ' CASE WHEN min_val = max_val', + ' THEN 0', + ' ELSE GREATEST(1, LEAST(WIDTH_BUCKET({{=it._column}}, min_val, max_val, bins_number), bins_number)) - 1', + ' END AS bin,', + ' min({{=it._column}})::numeric AS min,', + ' max({{=it._column}})::numeric AS max,', + ' sum({{=it._column}}*_feature_count)/sum(_feature_count)::numeric AS avg,', + ' sum(_feature_count) AS freq', + 'FROM ({{=it._query}}) _cdb_histogram, basics, nulls, bins', + 'WHERE {{=it._column}} IS NOT NULL', + 'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val', + 'ORDER BY bin' +].join('\n')); + +function Histogram(query, options, queryRewriter, queryRewriteData, params) { + BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); + + this.query = query; + this.column = options.column; + this.bins = options.bins; + + this._columnType = null; +} + +Histogram.prototype = Object.create(BaseOverviewsDataview.prototype); +Histogram.prototype.constructor = Histogram; + +module.exports = Histogram; + + +var DATE_OIDS = { + 1082: true, + 1114: true, + 1184: true +}; + +Histogram.prototype.sql = function(psql, override, callback) { + if (!callback) { + callback = override; + override = {}; + } + + var self = this; + + var _column = this.column; + + var columnTypeQuery = columnTypeQueryTpl({ + column: _column, query: this.rewrittenQuery(this.query) + }); + + if (this._columnType === null) { + psql.query(columnTypeQuery, function(err, result) { + // assume numeric, will fail later + self._columnType = 'numeric'; + if (!err && !!result.rows[0]) { + var pgType = result.rows[0].pg_typeof; + if (DATE_OIDS.hasOwnProperty(pgType)) { + self._columnType = 'date'; + } + } + self.sql(psql, override, callback); + }, true); // use read-only transaction + return null; + } + + if (this._columnType === 'date') { + _column = columnCastTpl({column: _column}); + } + + var _query = this.rewrittenQuery(this.query); + + var basicsQuery, binsQuery; + + if (override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins')) { + basicsQuery = overrideBasicsQueryTpl({ + _query: _query, + _column: _column, + _start: override.start, + _end: override.end + }); + + binsQuery = [ + overrideBinsQueryTpl({ + _bins: override.bins + }) + ].join(',\n'); + } else { + basicsQuery = basicsQueryTpl({ + _query: _query, + _column: _column + }); + + if (override && _.has(override, 'bins')) { + binsQuery = [ + overrideBinsQueryTpl({ + _bins: override.bins + }) + ].join(',\n'); + } else { + binsQuery = [ + iqrQueryTpl({ + _query: _query, + _column: _column + }), + binsQueryTpl({ + _query: _query, + _minBins: BIN_MIN_NUMBER, + _maxBins: BIN_MAX_NUMBER + }) + ].join(',\n'); + } + } + + + var histogramSql = [ + "WITH", + [ + basicsQuery, + binsQuery, + nullsQueryTpl({ + _query: _query, + _column: _column + }) + ].join(',\n'), + histogramQueryTpl({ + _query: _query, + _column: _column + }) + ].join('\n'); + + return callback(null, histogramSql); +}; diff --git a/lib/cartodb/models/dataview/overviews/index.js b/lib/cartodb/models/dataview/overviews/index.js index 775ceca1..c541973e 100644 --- a/lib/cartodb/models/dataview/overviews/index.js +++ b/lib/cartodb/models/dataview/overviews/index.js @@ -1,3 +1,6 @@ module.exports = { - Formula: require('./formula') + Aggregation: require('./aggregation'), + Formula: require('./formula'), + Histogram: require('./histogram'), + List: require('./list') }; diff --git a/lib/cartodb/models/dataview/overviews/list.js b/lib/cartodb/models/dataview/overviews/list.js new file mode 100644 index 00000000..7e3b3161 --- /dev/null +++ b/lib/cartodb/models/dataview/overviews/list.js @@ -0,0 +1,11 @@ +var BaseOverviewsDataview = require('./base'); +var BaseDataview = require('../list'); + +function List(query, options, queryRewriter, queryRewriteData, params) { + BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); +} + +List.prototype = Object.create(BaseOverviewsDataview.prototype); +List.prototype.constructor = List; + +module.exports = List;