From 5ba72b4894e84ad63f36cf9bb5da29f819c2e63f Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Mon, 23 May 2016 14:18:45 +0200 Subject: [PATCH 01/10] Create base class for overviews dataviews --- lib/cartodb/models/dataview/overviews/base.js | 34 +++++++++++++++++++ .../models/dataview/overviews/formula.js | 32 ++--------------- 2 files changed, 37 insertions(+), 29 deletions(-) create mode 100644 lib/cartodb/models/dataview/overviews/base.js diff --git a/lib/cartodb/models/dataview/overviews/base.js b/lib/cartodb/models/dataview/overviews/base.js new file mode 100644 index 00000000..f7ee7e5c --- /dev/null +++ b/lib/cartodb/models/dataview/overviews/base.js @@ -0,0 +1,34 @@ +var _ = require('underscore'); +var BaseWidget = require('../base'); + +function BaseOverviewsDataview() {} + +module.exports = BaseOverviewsDataview; + +BaseOverviewsDataview.prototype = new BaseWidget(); +BaseOverviewsDataview.prototype.constructor = BaseOverviewsDataview; + +var zoomLevelFactor = 100.0; + +// Compute zoom level so that the the resolution grid size of the +// selected overview is smaller (zoomLevelFactor times smaller at least) +// than the bounding box size. +BaseOverviewsDataview.prototype.zoomLevelForBbox = function(bbox) { + var pxPerTile = 256.0; + var earthWidth = 360.0; + // TODO: now we assume overviews are computed for 1-pixel tolerance; + // should use extended overviews metadata to compute this properly. + if ( bbox ) { + var bboxValues = _.map(bbox.split(','), function(v) { return +v; }); + var w = Math.abs(bboxValues[2]-bboxValues[0]); + var h = Math.abs(bboxValues[3]-bboxValues[1]); + var maxDim = Math.min(w, h); + + // Find minimum suitable z + // note that the QueryRewirter will use the minimum level overview + // of level >= z if it exists, and otherwise the base table + var z = Math.ceil(-Math.log(maxDim*pxPerTile/earthWidth/zoomLevelFactor)/Math.log(2.0)); + return Math.max(z, 0); + } + return 0; +}; diff --git a/lib/cartodb/models/dataview/overviews/formula.js b/lib/cartodb/models/dataview/overviews/formula.js index aa57792e..54b4e0eb 100644 --- a/lib/cartodb/models/dataview/overviews/formula.js +++ b/lib/cartodb/models/dataview/overviews/formula.js @@ -1,5 +1,4 @@ -var _ = require('underscore'); -var BaseWidget = require('../base'); +var BaseOverviewsWidget = require('./base'); var BaseDataview = require('../formula'); var debug = require('debug')('windshaft:widget:formula:overviews'); @@ -38,43 +37,18 @@ function Formula(query, options, queryRewriter, queryRewriteData, params) { this.options = params; } -Formula.prototype = new BaseWidget(); +Formula.prototype = new BaseOverviewsWidget(); Formula.prototype.constructor = Formula; module.exports = Formula; -var zoom_level_factor = 100.0; - -// Compute zoom level so that the the resolution grid size of the -// selected overview is smaller (zoom_level_factor times smaller at least) -// than the bounding box size. -function zoom_level_for_bbox(bbox) { - var px_per_tile = 256.0; - var earth_width = 360.0; - // TODO: now we assume overviews are computed for 1-pixel tolerance; - // should use extended overviews metadata to compute this properly. - if ( bbox ) { - var bbox_values = _.map(bbox.split(','), function(v) { return +v; }); - var w = Math.abs(bbox_values[2]-bbox_values[0]); - var h = Math.abs(bbox_values[3]-bbox_values[1]); - var max_dim = Math.min(w, h); - - // Find minimum suitable z - // note that the QueryRewirter will use the minimum level overview - // of level >= z if it exists, and otherwise the base table - var z = Math.ceil(-Math.log(max_dim*px_per_tile/earth_width/zoom_level_factor)/Math.log(2.0)); - return Math.max(z, 0); - } - return 0; -} - Formula.prototype.sql = function(psql, filters, override, callback) { var _query = this.query; var formulaQueryTpl = formulaQueryTpls[this.operation]; if ( formulaQueryTpl ) { // supported formula for use with overviews - var zoom_level = zoom_level_for_bbox(this.options.bbox); + var zoom_level = this.zoomLevelForBbox(this.options.bbox); _query = this.queryRewriter.query(_query, this.queryRewriteData, { zoom_level: zoom_level }); var formulaSql = formulaQueryTpl({ _query: _query, From 988412fc07ca54abb939f089505a0c52afda38e7 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Mon, 23 May 2016 16:53:28 +0200 Subject: [PATCH 02/10] Define default overviews dataview behaviour in base class --- lib/cartodb/models/dataview/overviews/base.js | 59 ++++++++++++++++++- .../models/dataview/overviews/formula.js | 24 ++------ 2 files changed, 60 insertions(+), 23 deletions(-) diff --git a/lib/cartodb/models/dataview/overviews/base.js b/lib/cartodb/models/dataview/overviews/base.js index f7ee7e5c..95901869 100644 --- a/lib/cartodb/models/dataview/overviews/base.js +++ b/lib/cartodb/models/dataview/overviews/base.js @@ -1,14 +1,29 @@ var _ = require('underscore'); var BaseWidget = require('../base'); -function BaseOverviewsDataview() {} +function BaseOverviewsDataview(query, queryOptions, BaseDataview, queryRewriter, queryRewriteData, options) { + this.BaseDataview = BaseDataview; + this.query = query; + this.queryOptions = queryOptions; + this.queryRewriter = queryRewriter; + this.queryRewriteData = queryRewriteData; + this.options = options; + this.baseDataview = new this.BaseDataview(this.query, this.queryOptions); +} module.exports = BaseOverviewsDataview; BaseOverviewsDataview.prototype = new BaseWidget(); BaseOverviewsDataview.prototype.constructor = BaseOverviewsDataview; -var zoomLevelFactor = 100.0; +// TODO: parameterized these settings +var SETTINGS = { + // use overviews as a default fallback strategy + defaultOverviews: false, + + // minimum ratio of bounding box size to grid size + zoomLevelFactor: 100.0 +}; // Compute zoom level so that the the resolution grid size of the // selected overview is smaller (zoomLevelFactor times smaller at least) @@ -27,8 +42,46 @@ BaseOverviewsDataview.prototype.zoomLevelForBbox = function(bbox) { // Find minimum suitable z // note that the QueryRewirter will use the minimum level overview // of level >= z if it exists, and otherwise the base table - var z = Math.ceil(-Math.log(maxDim*pxPerTile/earthWidth/zoomLevelFactor)/Math.log(2.0)); + var z = Math.ceil(-Math.log(maxDim*pxPerTile/earthWidth/SETTINGS.zoomLevelFactor)/Math.log(2.0)); return Math.max(z, 0); } return 0; }; + +BaseOverviewsDataview.prototype.rewrittenQuery = function(query) { + var zoom_level = this.zoomLevelForBbox(this.options.bbox); + return this.queryRewriter.query(query, this.queryRewriteData, { zoom_level: zoom_level }); +}; + +// Default behaviour +BaseOverviewsDataview.prototype.defaultSql = function(psql, filters, override, callback) { + var query = this.query; + var dataview = this.baseDataview; + if ( SETTINGS.defaultOverviews ) { + query = this.rewrittenQuery(query); + dataview = new this.BaseDataview(query, this.queryOptions); + } + return dataview.sql(psql, filters, override, callback); +}; + +// default implementation that can be override in derived classes: + +BaseOverviewsDataview.prototype.sql = function(psql, filters, override, callback) { + return this.defaultSql(psql, filters, override, callback); +}; + +BaseOverviewsDataview.prototype.search = function(psql, userQuery, callback) { + return this.baseDataview.search(psql, userQuery, callback); +}; + +BaseOverviewsDataview.prototype.format = function(result) { + return this.baseDataview.format(result); +}; + +BaseOverviewsDataview.prototype.getType = function() { + return this.baseDataview.getType(); +}; + +BaseOverviewsDataview.prototype.toString = function() { + return this.baseDataview.toString(); +}; diff --git a/lib/cartodb/models/dataview/overviews/formula.js b/lib/cartodb/models/dataview/overviews/formula.js index 54b4e0eb..c3fb442e 100644 --- a/lib/cartodb/models/dataview/overviews/formula.js +++ b/lib/cartodb/models/dataview/overviews/formula.js @@ -28,16 +28,12 @@ var formulaQueryTpls = { }; function Formula(query, options, queryRewriter, queryRewriteData, params) { - this.base_dataview = new BaseDataview(query, options); - this.query = query; + BaseOverviewsWidget.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); this.column = options.column || '1'; this.operation = options.operation; - this.queryRewriter = queryRewriter; - this.queryRewriteData = queryRewriteData; - this.options = params; } -Formula.prototype = new BaseOverviewsWidget(); +Formula.prototype = Object.create(BaseOverviewsWidget.prototype); Formula.prototype.constructor = Formula; module.exports = Formula; @@ -61,18 +57,6 @@ Formula.prototype.sql = function(psql, filters, override, callback) { return callback(null, formulaSql); } - // For non supported operations (min, max) we're not using overviews. - return this.base_dataview.sql(psql, filters, override, callback); -}; - -Formula.prototype.format = function(result) { - return this.base_dataview.format(result); -}; - -Formula.prototype.getType = function() { - return this.base_dataview.getType(); -}; - -Formula.prototype.toString = function() { - return this.base_dataview.toString(); + // default behaviour + return this.defaultSql(psql, filters, override, callback); }; From 657b262d9207923cf462a4ae0230309fc7116a48 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Mon, 23 May 2016 17:06:52 +0200 Subject: [PATCH 03/10] Override all dataview types for overviews All are using now the default behaviour defined in the base class. --- lib/cartodb/models/dataview/overviews/aggregation.js | 11 +++++++++++ lib/cartodb/models/dataview/overviews/histogram.js | 11 +++++++++++ lib/cartodb/models/dataview/overviews/index.js | 5 ++++- lib/cartodb/models/dataview/overviews/list.js | 11 +++++++++++ 4 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 lib/cartodb/models/dataview/overviews/aggregation.js create mode 100644 lib/cartodb/models/dataview/overviews/histogram.js create mode 100644 lib/cartodb/models/dataview/overviews/list.js diff --git a/lib/cartodb/models/dataview/overviews/aggregation.js b/lib/cartodb/models/dataview/overviews/aggregation.js new file mode 100644 index 00000000..3dc6f97a --- /dev/null +++ b/lib/cartodb/models/dataview/overviews/aggregation.js @@ -0,0 +1,11 @@ +var BaseOverviewsWidget = require('./base'); +var BaseDataview = require('../aggregation'); + +function Aggregation(query, options, queryRewriter, queryRewriteData, params) { + BaseOverviewsWidget.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); +} + +Aggregation.prototype = Object.create(BaseOverviewsWidget.prototype); +Aggregation.prototype.constructor = Aggregation; + +module.exports = Aggregation; diff --git a/lib/cartodb/models/dataview/overviews/histogram.js b/lib/cartodb/models/dataview/overviews/histogram.js new file mode 100644 index 00000000..9ce29ca6 --- /dev/null +++ b/lib/cartodb/models/dataview/overviews/histogram.js @@ -0,0 +1,11 @@ +var BaseOverviewsWidget = require('./base'); +var BaseDataview = require('../histogram'); + +function Histogram(query, options, queryRewriter, queryRewriteData, params) { + BaseOverviewsWidget.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); +} + +Histogram.prototype = Object.create(BaseOverviewsWidget.prototype); +Histogram.prototype.constructor = Histogram; + +module.exports = Histogram; diff --git a/lib/cartodb/models/dataview/overviews/index.js b/lib/cartodb/models/dataview/overviews/index.js index 775ceca1..c541973e 100644 --- a/lib/cartodb/models/dataview/overviews/index.js +++ b/lib/cartodb/models/dataview/overviews/index.js @@ -1,3 +1,6 @@ module.exports = { - Formula: require('./formula') + Aggregation: require('./aggregation'), + Formula: require('./formula'), + Histogram: require('./histogram'), + List: require('./list') }; diff --git a/lib/cartodb/models/dataview/overviews/list.js b/lib/cartodb/models/dataview/overviews/list.js new file mode 100644 index 00000000..5efb55f1 --- /dev/null +++ b/lib/cartodb/models/dataview/overviews/list.js @@ -0,0 +1,11 @@ +var BaseOverviewsWidget = require('./base'); +var BaseDataview = require('../list'); + +function List(query, options, queryRewriter, queryRewriteData, params) { + BaseOverviewsWidget.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); +} + +List.prototype = Object.create(BaseOverviewsWidget.prototype); +List.prototype.constructor = List; + +module.exports = List; From 4a646d47008d653370730a19af2577b16e819d7d Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Mon, 23 May 2016 17:20:04 +0200 Subject: [PATCH 04/10] Refactor overviews formula dataview --- lib/cartodb/models/dataview/overviews/formula.js | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/cartodb/models/dataview/overviews/formula.js b/lib/cartodb/models/dataview/overviews/formula.js index c3fb442e..3d5850ae 100644 --- a/lib/cartodb/models/dataview/overviews/formula.js +++ b/lib/cartodb/models/dataview/overviews/formula.js @@ -39,17 +39,14 @@ Formula.prototype.constructor = Formula; module.exports = Formula; Formula.prototype.sql = function(psql, filters, override, callback) { - var _query = this.query; var formulaQueryTpl = formulaQueryTpls[this.operation]; if ( formulaQueryTpl ) { // supported formula for use with overviews - var zoom_level = this.zoomLevelForBbox(this.options.bbox); - _query = this.queryRewriter.query(_query, this.queryRewriteData, { zoom_level: zoom_level }); var formulaSql = formulaQueryTpl({ - _query: _query, + _query: this.rewrittenQuery(this.query), _operation: this.operation, - _column: this.column + _column: this.column }); debug(formulaSql); callback = callback || override; From f008c74419c79f93fea9a80421a9572e6ce67e72 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Mon, 23 May 2016 17:42:26 +0200 Subject: [PATCH 05/10] Specific aggregation dataview implementation for overviews --- .../models/dataview/overviews/aggregation.js | 144 +++++++++++++++++- 1 file changed, 143 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/overviews/aggregation.js b/lib/cartodb/models/dataview/overviews/aggregation.js index 3dc6f97a..ff2e844e 100644 --- a/lib/cartodb/models/dataview/overviews/aggregation.js +++ b/lib/cartodb/models/dataview/overviews/aggregation.js @@ -1,11 +1,153 @@ var BaseOverviewsWidget = require('./base'); var BaseDataview = require('../aggregation'); +var debug = require('debug')('windshaft:widget:ovaggregation'); -function Aggregation(query, options, queryRewriter, queryRewriteData, params) { +var dot = require('dot'); +dot.templateSettings.strip = false; + +var summaryQueryTpl = dot.template([ + 'summary AS (', + ' SELECT', + ' sum(_feature_count) AS count,', + ' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count', + ' FROM ({{=it._query}}) _cdb_aggregation_nulls', + ')' +].join('\n')); + +var rankedCategoriesQueryTpl = dot.template([ + 'categories AS(', + ' SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,', + ' row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank', + ' FROM ({{=it._query}}) _cdb_aggregation_all', + ' GROUP BY {{=it._column}}', + ' ORDER BY 2 DESC', + ')' +].join('\n')); + +var categoriesSummaryQueryTpl = dot.template([ + 'categories_summary AS(', + ' SELECT count(1) categories_count, max(value) max_val, min(value) min_val', + ' FROM categories', + ')' +].join('\n')); + +var rankedAggregationQueryTpl = dot.template([ + 'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count', + ' FROM categories, summary, categories_summary', + ' WHERE rank < {{=it._limit}}', + 'UNION ALL', + 'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val, count, categories_count', + ' FROM categories, summary, categories_summary', + ' WHERE rank >= {{=it._limit}}', + 'GROUP BY nulls_count, min_val, max_val, count, categories_count' +].join('\n')); + +var aggregationQueryTpl = dot.template([ + 'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,', + ' nulls_count, min_val, max_val, count, categories_count', + 'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary', + 'GROUP BY category, nulls_count, min_val, max_val, count, categories_count', + 'ORDER BY value DESC' +].join('\n')); + +var CATEGORIES_LIMIT = 6; + +/** + { + type: 'aggregation', + options: { + column: 'name', + aggregation: 'count' // it could be, e.g., sum if column is numeric + } + } + */ + function Aggregation(query, options, queryRewriter, queryRewriteData, params) { BaseOverviewsWidget.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); + + this.query = query; + this.column = options.column; + this.aggregation = options.aggregation; + this.aggregationColumn = options.aggregationColumn; } Aggregation.prototype = Object.create(BaseOverviewsWidget.prototype); Aggregation.prototype.constructor = Aggregation; module.exports = Aggregation; + +Aggregation.prototype.sql = function(psql, filters, override, callback) { + if (!callback) { + callback = override; + override = {}; + } + + var _query = this.rewrittenQuery(this.query); + + var aggregationSql; + if (!!override.ownFilter) { + aggregationSql = [ + "WITH", + [ + summaryQueryTpl({ + _query: _query, + _column: this.column + }), + rankedCategoriesQueryTpl({ + _query: _query, + _column: this.column, + _aggregation: this.getAggregationSql() + }), + categoriesSummaryQueryTpl({ + _query: _query, + _column: this.column + }) + ].join(',\n'), + aggregationQueryTpl({ + _query: _query, + _column: this.column, + _aggregation: this.getAggregationSql(), + _limit: CATEGORIES_LIMIT + }) + ].join('\n'); + } else { + aggregationSql = [ + "WITH", + [ + summaryQueryTpl({ + _query: _query, + _column: this.column + }), + rankedCategoriesQueryTpl({ + _query: _query, + _column: this.column, + _aggregation: this.getAggregationSql() + }), + categoriesSummaryQueryTpl({ + _query: _query, + _column: this.column + }) + ].join(',\n'), + rankedAggregationQueryTpl({ + _query: _query, + _column: this.column, + _limit: CATEGORIES_LIMIT + }) + ].join('\n'); + } + + debug(aggregationSql); + + return callback(null, aggregationSql); +}; + +var aggregationFnQueryTpl = { + count: dot.template('sum(_feature_count)'), + sum: dot.template('sum({{=it._aggregationColumn}}*_feature_count)') +}; + +Aggregation.prototype.getAggregationSql = function() { + return aggregationFnQueryTpl[this.aggregation]({ + _aggregationFn: this.aggregation, + _aggregationColumn: this.aggregationColumn || 1 + }); +}; From ae9b8a03807c0eec05053fcb6996ff80e85cc6b7 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Mon, 23 May 2016 18:10:46 +0200 Subject: [PATCH 06/10] Remove comment --- lib/cartodb/models/dataview/overviews/aggregation.js | 9 --------- 1 file changed, 9 deletions(-) diff --git a/lib/cartodb/models/dataview/overviews/aggregation.js b/lib/cartodb/models/dataview/overviews/aggregation.js index ff2e844e..3fa81934 100644 --- a/lib/cartodb/models/dataview/overviews/aggregation.js +++ b/lib/cartodb/models/dataview/overviews/aggregation.js @@ -52,15 +52,6 @@ var aggregationQueryTpl = dot.template([ var CATEGORIES_LIMIT = 6; -/** - { - type: 'aggregation', - options: { - column: 'name', - aggregation: 'count' // it could be, e.g., sum if column is numeric - } - } - */ function Aggregation(query, options, queryRewriter, queryRewriteData, params) { BaseOverviewsWidget.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); From c0a24108ba98c6abf92f04ebfa8eb74c6d51aa46 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Mon, 23 May 2016 18:11:10 +0200 Subject: [PATCH 07/10] Implement overviews histogram dataviews --- .../models/dataview/overviews/histogram.js | 211 ++++++++++++++++++ 1 file changed, 211 insertions(+) diff --git a/lib/cartodb/models/dataview/overviews/histogram.js b/lib/cartodb/models/dataview/overviews/histogram.js index 9ce29ca6..88e55482 100644 --- a/lib/cartodb/models/dataview/overviews/histogram.js +++ b/lib/cartodb/models/dataview/overviews/histogram.js @@ -1,11 +1,222 @@ +var _ = require('underscore'); var BaseOverviewsWidget = require('./base'); var BaseDataview = require('../histogram'); +var debug = require('debug')('windshaft:dataview:ovhistogram'); + +var dot = require('dot'); +dot.templateSettings.strip = false; + +var columnTypeQueryTpl = dot.template( + 'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1' +); +var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})"); + +var BIN_MIN_NUMBER = 6; +var BIN_MAX_NUMBER = 48; + +var basicsQueryTpl = dot.template([ + 'basics AS (', + ' SELECT', + ' max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,', + ' sum({{=it._column}}*_feature_count)/sum(_feature_count) AS avg_val, sum(_feature_count) AS total_rows', + ' FROM ({{=it._query}}) _cdb_basics', + ')' +].join(' \n')); + +var overrideBasicsQueryTpl = dot.template([ + 'basics AS (', + ' SELECT', + ' max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,', + ' sum({{=it._column}}*_feature_count)/sum(_feature_count) AS avg_val, sum(_feature_count) AS total_rows', + ' FROM ({{=it._query}}) _cdb_basics', + ')' +].join('\n')); + +var iqrQueryTpl = dot.template([ + 'iqrange AS (', + ' SELECT max(quartile_max) - min(quartile_max) AS iqr', + ' FROM (', + ' SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (', + ' SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}', + ' ) AS quartile', + ' FROM ({{=it._query}}) _cdb_rank) _cdb_quartiles', + ' WHERE quartile = 1 or quartile = 3', + ' GROUP BY quartile', + ' ) _cdb_iqr', + ')' +].join('\n')); + +var binsQueryTpl = dot.template([ + 'bins AS (', + ' SELECT CASE WHEN total_rows = 0 OR iqr = 0', + ' THEN 1', + ' ELSE GREATEST(', + ' LEAST({{=it._minBins}}, CAST(total_rows AS INT)),', + ' LEAST(', + ' CAST(((max_val - min_val) / (2 * iqr * power(total_rows, 1/3))) AS INT),', + ' {{=it._maxBins}}', + ' )', + ' )', + ' END AS bins_number', + ' FROM basics, iqrange, ({{=it._query}}) _cdb_bins', + ' LIMIT 1', + ')' +].join('\n')); + +var overrideBinsQueryTpl = dot.template([ + 'bins AS (', + ' SELECT {{=it._bins}} AS bins_number', + ')' +].join('\n')); + +var nullsQueryTpl = dot.template([ + 'nulls AS (', + ' SELECT', + ' count(*) AS nulls_count', + ' FROM ({{=it._query}}) _cdb_histogram_nulls', + ' WHERE {{=it._column}} IS NULL', + ')' +].join('\n')); + +var histogramQueryTpl = dot.template([ + 'SELECT', + ' (max_val - min_val) / cast(bins_number as float) AS bin_width,', + ' bins_number,', + ' nulls_count,', + ' avg_val,', + ' CASE WHEN min_val = max_val', + ' THEN 0', + ' ELSE GREATEST(1, LEAST(WIDTH_BUCKET({{=it._column}}, min_val, max_val, bins_number), bins_number)) - 1', + ' END AS bin,', + ' min({{=it._column}})::numeric AS min,', + ' max({{=it._column}})::numeric AS max,', + ' sum({{=it._column}}*_feature_count)/sum(_feature_count)::numeric AS avg,', + ' sum(_feature_count) AS freq', + 'FROM ({{=it._query}}) _cdb_histogram, basics, nulls, bins', + 'WHERE {{=it._column}} IS NOT NULL', + 'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val', + 'ORDER BY bin' +].join('\n')); + function Histogram(query, options, queryRewriter, queryRewriteData, params) { BaseOverviewsWidget.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); + + this.query = query; + this.column = options.column; + this.bins = options.bins; + + this._columnType = null; } Histogram.prototype = Object.create(BaseOverviewsWidget.prototype); Histogram.prototype.constructor = Histogram; module.exports = Histogram; + + +var DATE_OIDS = { + 1082: true, + 1114: true, + 1184: true +}; + +Histogram.prototype.sql = function(psql, override, callback) { + if (!callback) { + callback = override; + override = {}; + } + + var self = this; + + var _column = this.column; + + var columnTypeQuery = columnTypeQueryTpl({ + column: _column, query: this.rewrittenQuery(this.query) + }); + + if (this._columnType === null) { + psql.query(columnTypeQuery, function(err, result) { + // assume numeric, will fail later + self._columnType = 'numeric'; + if (!err && !!result.rows[0]) { + var pgType = result.rows[0].pg_typeof; + if (DATE_OIDS.hasOwnProperty(pgType)) { + self._columnType = 'date'; + } + } + self.sql(psql, override, callback); + }, true); // use read-only transaction + return null; + } + + if (this._columnType === 'date') { + _column = columnCastTpl({column: _column}); + } + + var _query = this.rewrittenQuery(this.query); + + var basicsQuery, binsQuery; + + if (override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins')) { + debug('overriding with %j', override); + basicsQuery = overrideBasicsQueryTpl({ + _query: _query, + _column: _column, + _start: override.start, + _end: override.end + }); + + binsQuery = [ + overrideBinsQueryTpl({ + _bins: override.bins + }) + ].join(',\n'); + } else { + basicsQuery = basicsQueryTpl({ + _query: _query, + _column: _column + }); + + if (override && _.has(override, 'bins')) { + binsQuery = [ + overrideBinsQueryTpl({ + _bins: override.bins + }) + ].join(',\n'); + } else { + binsQuery = [ + iqrQueryTpl({ + _query: _query, + _column: _column + }), + binsQueryTpl({ + _query: _query, + _minBins: BIN_MIN_NUMBER, + _maxBins: BIN_MAX_NUMBER + }) + ].join(',\n'); + } + } + + + var histogramSql = [ + "WITH", + [ + basicsQuery, + binsQuery, + nullsQueryTpl({ + _query: _query, + _column: _column + }) + ].join(',\n'), + histogramQueryTpl({ + _query: _query, + _column: _column + }) + ].join('\n'); + + debug(histogramSql); + + return callback(null, histogramSql); +}; From 7ac3784f32cedf7b98e9609d6404bc2c5f86228f Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Mon, 30 May 2016 17:21:56 +0200 Subject: [PATCH 08/10] Increase the ratio used to select an overview level from a bounding box This value would ideally be adjusted to prevent the grid size of the overview used being greater that one pixel. So, this should be the larger dimension of the map window in pixels. --- lib/cartodb/models/dataview/overviews/base.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/overviews/base.js b/lib/cartodb/models/dataview/overviews/base.js index 95901869..9876e271 100644 --- a/lib/cartodb/models/dataview/overviews/base.js +++ b/lib/cartodb/models/dataview/overviews/base.js @@ -22,7 +22,8 @@ var SETTINGS = { defaultOverviews: false, // minimum ratio of bounding box size to grid size - zoomLevelFactor: 100.0 + // (this would ideally be based on the viewport size in pixels) + zoomLevelFactor: 1024.0 }; // Compute zoom level so that the the resolution grid size of the From 5838b7a4557de0838a6d18ac65e06a020d4f8c8e Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Tue, 31 May 2016 15:19:33 +0200 Subject: [PATCH 09/10] Remove debugging messages --- lib/cartodb/models/dataview/overviews/aggregation.js | 3 --- lib/cartodb/models/dataview/overviews/formula.js | 3 --- lib/cartodb/models/dataview/overviews/histogram.js | 5 ----- 3 files changed, 11 deletions(-) diff --git a/lib/cartodb/models/dataview/overviews/aggregation.js b/lib/cartodb/models/dataview/overviews/aggregation.js index 3fa81934..8da8baae 100644 --- a/lib/cartodb/models/dataview/overviews/aggregation.js +++ b/lib/cartodb/models/dataview/overviews/aggregation.js @@ -1,6 +1,5 @@ var BaseOverviewsWidget = require('./base'); var BaseDataview = require('../aggregation'); -var debug = require('debug')('windshaft:widget:ovaggregation'); var dot = require('dot'); dot.templateSettings.strip = false; @@ -126,8 +125,6 @@ Aggregation.prototype.sql = function(psql, filters, override, callback) { ].join('\n'); } - debug(aggregationSql); - return callback(null, aggregationSql); }; diff --git a/lib/cartodb/models/dataview/overviews/formula.js b/lib/cartodb/models/dataview/overviews/formula.js index 3d5850ae..5cf4af32 100644 --- a/lib/cartodb/models/dataview/overviews/formula.js +++ b/lib/cartodb/models/dataview/overviews/formula.js @@ -1,8 +1,6 @@ var BaseOverviewsWidget = require('./base'); var BaseDataview = require('../formula'); -var debug = require('debug')('windshaft:widget:formula:overviews'); - var dot = require('dot'); dot.templateSettings.strip = false; @@ -48,7 +46,6 @@ Formula.prototype.sql = function(psql, filters, override, callback) { _operation: this.operation, _column: this.column }); - debug(formulaSql); callback = callback || override; return callback(null, formulaSql); diff --git a/lib/cartodb/models/dataview/overviews/histogram.js b/lib/cartodb/models/dataview/overviews/histogram.js index 88e55482..7463b29f 100644 --- a/lib/cartodb/models/dataview/overviews/histogram.js +++ b/lib/cartodb/models/dataview/overviews/histogram.js @@ -2,8 +2,6 @@ var _ = require('underscore'); var BaseOverviewsWidget = require('./base'); var BaseDataview = require('../histogram'); -var debug = require('debug')('windshaft:dataview:ovhistogram'); - var dot = require('dot'); dot.templateSettings.strip = false; @@ -159,7 +157,6 @@ Histogram.prototype.sql = function(psql, override, callback) { var basicsQuery, binsQuery; if (override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins')) { - debug('overriding with %j', override); basicsQuery = overrideBasicsQueryTpl({ _query: _query, _column: _column, @@ -216,7 +213,5 @@ Histogram.prototype.sql = function(psql, override, callback) { }) ].join('\n'); - debug(histogramSql); - return callback(null, histogramSql); }; From d5423c88ea826de0fdad141e327f134ec6ab9125 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Tue, 31 May 2016 15:30:38 +0200 Subject: [PATCH 10/10] Replace use of the name widget by dataview for consistency --- lib/cartodb/models/dataview/overviews/aggregation.js | 6 +++--- lib/cartodb/models/dataview/overviews/base.js | 4 ++-- lib/cartodb/models/dataview/overviews/formula.js | 6 +++--- lib/cartodb/models/dataview/overviews/histogram.js | 6 +++--- lib/cartodb/models/dataview/overviews/list.js | 6 +++--- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/cartodb/models/dataview/overviews/aggregation.js b/lib/cartodb/models/dataview/overviews/aggregation.js index 8da8baae..41685ff5 100644 --- a/lib/cartodb/models/dataview/overviews/aggregation.js +++ b/lib/cartodb/models/dataview/overviews/aggregation.js @@ -1,4 +1,4 @@ -var BaseOverviewsWidget = require('./base'); +var BaseOverviewsDataview = require('./base'); var BaseDataview = require('../aggregation'); var dot = require('dot'); @@ -52,7 +52,7 @@ var aggregationQueryTpl = dot.template([ var CATEGORIES_LIMIT = 6; function Aggregation(query, options, queryRewriter, queryRewriteData, params) { - BaseOverviewsWidget.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); + BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); this.query = query; this.column = options.column; @@ -60,7 +60,7 @@ var CATEGORIES_LIMIT = 6; this.aggregationColumn = options.aggregationColumn; } -Aggregation.prototype = Object.create(BaseOverviewsWidget.prototype); +Aggregation.prototype = Object.create(BaseOverviewsDataview.prototype); Aggregation.prototype.constructor = Aggregation; module.exports = Aggregation; diff --git a/lib/cartodb/models/dataview/overviews/base.js b/lib/cartodb/models/dataview/overviews/base.js index 9876e271..bbf70742 100644 --- a/lib/cartodb/models/dataview/overviews/base.js +++ b/lib/cartodb/models/dataview/overviews/base.js @@ -1,5 +1,5 @@ var _ = require('underscore'); -var BaseWidget = require('../base'); +var BaseDataview = require('../base'); function BaseOverviewsDataview(query, queryOptions, BaseDataview, queryRewriter, queryRewriteData, options) { this.BaseDataview = BaseDataview; @@ -13,7 +13,7 @@ function BaseOverviewsDataview(query, queryOptions, BaseDataview, queryRewriter, module.exports = BaseOverviewsDataview; -BaseOverviewsDataview.prototype = new BaseWidget(); +BaseOverviewsDataview.prototype = new BaseDataview(); BaseOverviewsDataview.prototype.constructor = BaseOverviewsDataview; // TODO: parameterized these settings diff --git a/lib/cartodb/models/dataview/overviews/formula.js b/lib/cartodb/models/dataview/overviews/formula.js index 5cf4af32..adad5f58 100644 --- a/lib/cartodb/models/dataview/overviews/formula.js +++ b/lib/cartodb/models/dataview/overviews/formula.js @@ -1,4 +1,4 @@ -var BaseOverviewsWidget = require('./base'); +var BaseOverviewsDataview = require('./base'); var BaseDataview = require('../formula'); var dot = require('dot'); @@ -26,12 +26,12 @@ var formulaQueryTpls = { }; function Formula(query, options, queryRewriter, queryRewriteData, params) { - BaseOverviewsWidget.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); + BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); this.column = options.column || '1'; this.operation = options.operation; } -Formula.prototype = Object.create(BaseOverviewsWidget.prototype); +Formula.prototype = Object.create(BaseOverviewsDataview.prototype); Formula.prototype.constructor = Formula; module.exports = Formula; diff --git a/lib/cartodb/models/dataview/overviews/histogram.js b/lib/cartodb/models/dataview/overviews/histogram.js index 7463b29f..20dd7712 100644 --- a/lib/cartodb/models/dataview/overviews/histogram.js +++ b/lib/cartodb/models/dataview/overviews/histogram.js @@ -1,5 +1,5 @@ var _ = require('underscore'); -var BaseOverviewsWidget = require('./base'); +var BaseOverviewsDataview = require('./base'); var BaseDataview = require('../histogram'); var dot = require('dot'); @@ -98,7 +98,7 @@ var histogramQueryTpl = dot.template([ ].join('\n')); function Histogram(query, options, queryRewriter, queryRewriteData, params) { - BaseOverviewsWidget.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); + BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); this.query = query; this.column = options.column; @@ -107,7 +107,7 @@ function Histogram(query, options, queryRewriter, queryRewriteData, params) { this._columnType = null; } -Histogram.prototype = Object.create(BaseOverviewsWidget.prototype); +Histogram.prototype = Object.create(BaseOverviewsDataview.prototype); Histogram.prototype.constructor = Histogram; module.exports = Histogram; diff --git a/lib/cartodb/models/dataview/overviews/list.js b/lib/cartodb/models/dataview/overviews/list.js index 5efb55f1..7e3b3161 100644 --- a/lib/cartodb/models/dataview/overviews/list.js +++ b/lib/cartodb/models/dataview/overviews/list.js @@ -1,11 +1,11 @@ -var BaseOverviewsWidget = require('./base'); +var BaseOverviewsDataview = require('./base'); var BaseDataview = require('../list'); function List(query, options, queryRewriter, queryRewriteData, params) { - BaseOverviewsWidget.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); + BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params); } -List.prototype = Object.create(BaseOverviewsWidget.prototype); +List.prototype = Object.create(BaseOverviewsDataview.prototype); List.prototype.constructor = List; module.exports = List;