cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #468 from CartoDB/466-overviews-dataviews

Browse Source 
Implement overviews support for all dataview types
This commit is contained in:
Javier Goizueta
2016-06-01 10:35:07 +02:00
parent 81ff0152c0 d5423c88ea
commit bd35d4e78a
6 changed files with 468 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/cartodb/models/dataview/overviews/aggregation.js
+141
View File
@@ -0,0 +1,141 @@
var BaseOverviewsDataview = require('./base');
var BaseDataview = require('../aggregation');
var dot = require('dot');
dot.templateSettings.strip = false;
var summaryQueryTpl = dot.template([
'summary AS (',
' SELECT',
' sum(_feature_count) AS count,',
' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count',
' FROM ({{=it._query}}) _cdb_aggregation_nulls',
')'
].join('\n'));
var rankedCategoriesQueryTpl = dot.template([
'categories AS(',
' SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,',
' row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank',
' FROM ({{=it._query}}) _cdb_aggregation_all',
' GROUP BY {{=it._column}}',
' ORDER BY 2 DESC',
')'
].join('\n'));
var categoriesSummaryQueryTpl = dot.template([
'categories_summary AS(',
' SELECT count(1) categories_count, max(value) max_val, min(value) min_val',
' FROM categories',
')'
].join('\n'));
var rankedAggregationQueryTpl = dot.template([
'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count',
' FROM categories, summary, categories_summary',
' WHERE rank < {{=it._limit}}',
'UNION ALL',
'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val, count, categories_count',
' FROM categories, summary, categories_summary',
' WHERE rank >= {{=it._limit}}',
'GROUP BY nulls_count, min_val, max_val, count, categories_count'
].join('\n'));
var aggregationQueryTpl = dot.template([
'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,',
' nulls_count, min_val, max_val, count, categories_count',
'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary',
'GROUP BY category, nulls_count, min_val, max_val, count, categories_count',
'ORDER BY value DESC'
].join('\n'));
var CATEGORIES_LIMIT = 6;
function Aggregation(query, options, queryRewriter, queryRewriteData, params) {
BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params);
this.query = query;
this.column = options.column;
this.aggregation = options.aggregation;
this.aggregationColumn = options.aggregationColumn;
}
Aggregation.prototype = Object.create(BaseOverviewsDataview.prototype);
Aggregation.prototype.constructor = Aggregation;
module.exports = Aggregation;
Aggregation.prototype.sql = function(psql, filters, override, callback) {
if (!callback) {
callback = override;
override = {};
}
var _query = this.rewrittenQuery(this.query);
var aggregationSql;
if (!!override.ownFilter) {
aggregationSql = [
"WITH",
[
summaryQueryTpl({
_query: _query,
_column: this.column
}),
rankedCategoriesQueryTpl({
_query: _query,
_column: this.column,
_aggregation: this.getAggregationSql()
}),
categoriesSummaryQueryTpl({
_query: _query,
_column: this.column
})
].join(',\n'),
aggregationQueryTpl({
_query: _query,
_column: this.column,
_aggregation: this.getAggregationSql(),
_limit: CATEGORIES_LIMIT
})
].join('\n');
} else {
aggregationSql = [
"WITH",
[
summaryQueryTpl({
_query: _query,
_column: this.column
}),
rankedCategoriesQueryTpl({
_query: _query,
_column: this.column,
_aggregation: this.getAggregationSql()
}),
categoriesSummaryQueryTpl({
_query: _query,
_column: this.column
})
].join(',\n'),
rankedAggregationQueryTpl({
_query: _query,
_column: this.column,
_limit: CATEGORIES_LIMIT
})
].join('\n');
}
return callback(null, aggregationSql);
};
var aggregationFnQueryTpl = {
count: dot.template('sum(_feature_count)'),
sum: dot.template('sum({{=it._aggregationColumn}}*_feature_count)')
};
Aggregation.prototype.getAggregationSql = function() {
return aggregationFnQueryTpl[this.aggregation]({
_aggregationFn: this.aggregation,
_aggregationColumn: this.aggregationColumn || 1
});
};
lib/cartodb/models/dataview/overviews/base.js
+88
View File
@@ -0,0 +1,88 @@
var _ = require('underscore');
var BaseDataview = require('../base');
function BaseOverviewsDataview(query, queryOptions, BaseDataview, queryRewriter, queryRewriteData, options) {
this.BaseDataview = BaseDataview;
this.query = query;
this.queryOptions = queryOptions;
this.queryRewriter = queryRewriter;
this.queryRewriteData = queryRewriteData;
this.options = options;
this.baseDataview = new this.BaseDataview(this.query, this.queryOptions);
}
module.exports = BaseOverviewsDataview;
BaseOverviewsDataview.prototype = new BaseDataview();
BaseOverviewsDataview.prototype.constructor = BaseOverviewsDataview;
// TODO: parameterized these settings
var SETTINGS = {
// use overviews as a default fallback strategy
defaultOverviews: false,
// minimum ratio of bounding box size to grid size
// (this would ideally be based on the viewport size in pixels)
zoomLevelFactor: 1024.0
};
// Compute zoom level so that the the resolution grid size of the
// selected overview is smaller (zoomLevelFactor times smaller at least)
// than the bounding box size.
BaseOverviewsDataview.prototype.zoomLevelForBbox = function(bbox) {
var pxPerTile = 256.0;
var earthWidth = 360.0;
// TODO: now we assume overviews are computed for 1-pixel tolerance;
// should use extended overviews metadata to compute this properly.
if ( bbox ) {
var bboxValues = _.map(bbox.split(','), function(v) { return +v; });
var w = Math.abs(bboxValues[2]-bboxValues[0]);
var h = Math.abs(bboxValues[3]-bboxValues[1]);
var maxDim = Math.min(w, h);
// Find minimum suitable z
// note that the QueryRewirter will use the minimum level overview
// of level >= z if it exists, and otherwise the base table
var z = Math.ceil(-Math.log(maxDim*pxPerTile/earthWidth/SETTINGS.zoomLevelFactor)/Math.log(2.0));
return Math.max(z, 0);
}
return 0;
};
BaseOverviewsDataview.prototype.rewrittenQuery = function(query) {
var zoom_level = this.zoomLevelForBbox(this.options.bbox);
return this.queryRewriter.query(query, this.queryRewriteData, { zoom_level: zoom_level });
};
// Default behaviour
BaseOverviewsDataview.prototype.defaultSql = function(psql, filters, override, callback) {
var query = this.query;
var dataview = this.baseDataview;
if ( SETTINGS.defaultOverviews ) {
query = this.rewrittenQuery(query);
dataview = new this.BaseDataview(query, this.queryOptions);
}
return dataview.sql(psql, filters, override, callback);
};
// default implementation that can be override in derived classes:
BaseOverviewsDataview.prototype.sql = function(psql, filters, override, callback) {
return this.defaultSql(psql, filters, override, callback);
};
BaseOverviewsDataview.prototype.search = function(psql, userQuery, callback) {
return this.baseDataview.search(psql, userQuery, callback);
};
BaseOverviewsDataview.prototype.format = function(result) {
return this.baseDataview.format(result);
};
BaseOverviewsDataview.prototype.getType = function() {
return this.baseDataview.getType();
};
BaseOverviewsDataview.prototype.toString = function() {
return this.baseDataview.toString();
};
lib/cartodb/models/dataview/overviews/formula.js
+7 -55
View File
@@ -1,9 +1,6 @@
var _ = require('underscore');
var BaseWidget = require('../base');
var BaseOverviewsDataview = require('./base');
var BaseDataview = require('../formula');
var debug = require('debug')('windshaft:widget:formula:overviews');
var dot = require('dot');
dot.templateSettings.strip = false;
@@ -29,76 +26,31 @@ var formulaQueryTpls = {
};
function Formula(query, options, queryRewriter, queryRewriteData, params) {
this.base_dataview = new BaseDataview(query, options);
this.query = query;
BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params);
this.column = options.column || '1';
this.operation = options.operation;
this.queryRewriter = queryRewriter;
this.queryRewriteData = queryRewriteData;
this.options = params;
}
Formula.prototype = new BaseWidget();
Formula.prototype = Object.create(BaseOverviewsDataview.prototype);
Formula.prototype.constructor = Formula;
module.exports = Formula;
var zoom_level_factor = 100.0;
// Compute zoom level so that the the resolution grid size of the
// selected overview is smaller (zoom_level_factor times smaller at least)
// than the bounding box size.
function zoom_level_for_bbox(bbox) {
var px_per_tile = 256.0;
var earth_width = 360.0;
// TODO: now we assume overviews are computed for 1-pixel tolerance;
// should use extended overviews metadata to compute this properly.
if ( bbox ) {
var bbox_values = _.map(bbox.split(','), function(v) { return +v; });
var w = Math.abs(bbox_values[2]-bbox_values[0]);
var h = Math.abs(bbox_values[3]-bbox_values[1]);
var max_dim = Math.min(w, h);
// Find minimum suitable z
// note that the QueryRewirter will use the minimum level overview
// of level >= z if it exists, and otherwise the base table
var z = Math.ceil(-Math.log(max_dim*px_per_tile/earth_width/zoom_level_factor)/Math.log(2.0));
return Math.max(z, 0);
}
return 0;
}
Formula.prototype.sql = function(psql, filters, override, callback) {
var _query = this.query;
var formulaQueryTpl = formulaQueryTpls[this.operation];
if ( formulaQueryTpl ) {
// supported formula for use with overviews
var zoom_level = zoom_level_for_bbox(this.options.bbox);
_query = this.queryRewriter.query(_query, this.queryRewriteData, { zoom_level: zoom_level });
var formulaSql = formulaQueryTpl({
_query: _query,
_query: this.rewrittenQuery(this.query),
_operation: this.operation,
_column: this.column
_column: this.column
});
debug(formulaSql);
callback = callback || override;
return callback(null, formulaSql);
}
// For non supported operations (min, max) we're not using overviews.
return this.base_dataview.sql(psql, filters, override, callback);
};
Formula.prototype.format = function(result) {
return this.base_dataview.format(result);
};
Formula.prototype.getType = function() {
return this.base_dataview.getType();
};
Formula.prototype.toString = function() {
return this.base_dataview.toString();
// default behaviour
return this.defaultSql(psql, filters, override, callback);
};
lib/cartodb/models/dataview/overviews/histogram.js
+217
View File
@@ -0,0 +1,217 @@
var _ = require('underscore');
var BaseOverviewsDataview = require('./base');
var BaseDataview = require('../histogram');
var dot = require('dot');
dot.templateSettings.strip = false;
var columnTypeQueryTpl = dot.template(
'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1'
);
var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})");
var BIN_MIN_NUMBER = 6;
var BIN_MAX_NUMBER = 48;
var basicsQueryTpl = dot.template([
'basics AS (',
' SELECT',
' max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,',
' sum({{=it._column}}*_feature_count)/sum(_feature_count) AS avg_val, sum(_feature_count) AS total_rows',
' FROM ({{=it._query}}) _cdb_basics',
')'
].join(' \n'));
var overrideBasicsQueryTpl = dot.template([
'basics AS (',
' SELECT',
' max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,',
' sum({{=it._column}}*_feature_count)/sum(_feature_count) AS avg_val, sum(_feature_count) AS total_rows',
' FROM ({{=it._query}}) _cdb_basics',
')'
].join('\n'));
var iqrQueryTpl = dot.template([
'iqrange AS (',
' SELECT max(quartile_max) - min(quartile_max) AS iqr',
' FROM (',
' SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (',
' SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}',
' ) AS quartile',
' FROM ({{=it._query}}) _cdb_rank) _cdb_quartiles',
' WHERE quartile = 1 or quartile = 3',
' GROUP BY quartile',
' ) _cdb_iqr',
')'
].join('\n'));
var binsQueryTpl = dot.template([
'bins AS (',
' SELECT CASE WHEN total_rows = 0 OR iqr = 0',
' THEN 1',
' ELSE GREATEST(',
' LEAST({{=it._minBins}}, CAST(total_rows AS INT)),',
' LEAST(',
' CAST(((max_val - min_val) / (2 * iqr * power(total_rows, 1/3))) AS INT),',
' {{=it._maxBins}}',
' )',
' )',
' END AS bins_number',
' FROM basics, iqrange, ({{=it._query}}) _cdb_bins',
' LIMIT 1',
')'
].join('\n'));
var overrideBinsQueryTpl = dot.template([
'bins AS (',
' SELECT {{=it._bins}} AS bins_number',
')'
].join('\n'));
var nullsQueryTpl = dot.template([
'nulls AS (',
' SELECT',
' count(*) AS nulls_count',
' FROM ({{=it._query}}) _cdb_histogram_nulls',
' WHERE {{=it._column}} IS NULL',
')'
].join('\n'));
var histogramQueryTpl = dot.template([
'SELECT',
' (max_val - min_val) / cast(bins_number as float) AS bin_width,',
' bins_number,',
' nulls_count,',
' avg_val,',
' CASE WHEN min_val = max_val',
' THEN 0',
' ELSE GREATEST(1, LEAST(WIDTH_BUCKET({{=it._column}}, min_val, max_val, bins_number), bins_number)) - 1',
' END AS bin,',
' min({{=it._column}})::numeric AS min,',
' max({{=it._column}})::numeric AS max,',
' sum({{=it._column}}*_feature_count)/sum(_feature_count)::numeric AS avg,',
' sum(_feature_count) AS freq',
'FROM ({{=it._query}}) _cdb_histogram, basics, nulls, bins',
'WHERE {{=it._column}} IS NOT NULL',
'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val',
'ORDER BY bin'
].join('\n'));
function Histogram(query, options, queryRewriter, queryRewriteData, params) {
BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params);
this.query = query;
this.column = options.column;
this.bins = options.bins;
this._columnType = null;
}
Histogram.prototype = Object.create(BaseOverviewsDataview.prototype);
Histogram.prototype.constructor = Histogram;
module.exports = Histogram;
var DATE_OIDS = {
1082: true,
1114: true,
1184: true
};
Histogram.prototype.sql = function(psql, override, callback) {
if (!callback) {
callback = override;
override = {};
}
var self = this;
var _column = this.column;
var columnTypeQuery = columnTypeQueryTpl({
column: _column, query: this.rewrittenQuery(this.query)
});
if (this._columnType === null) {
psql.query(columnTypeQuery, function(err, result) {
// assume numeric, will fail later
self._columnType = 'numeric';
if (!err && !!result.rows[0]) {
var pgType = result.rows[0].pg_typeof;
if (DATE_OIDS.hasOwnProperty(pgType)) {
self._columnType = 'date';
}
}
self.sql(psql, override, callback);
}, true); // use read-only transaction
return null;
}
if (this._columnType === 'date') {
_column = columnCastTpl({column: _column});
}
var _query = this.rewrittenQuery(this.query);
var basicsQuery, binsQuery;
if (override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins')) {
basicsQuery = overrideBasicsQueryTpl({
_query: _query,
_column: _column,
_start: override.start,
_end: override.end
});
binsQuery = [
overrideBinsQueryTpl({
_bins: override.bins
})
].join(',\n');
} else {
basicsQuery = basicsQueryTpl({
_query: _query,
_column: _column
});
if (override && _.has(override, 'bins')) {
binsQuery = [
overrideBinsQueryTpl({
_bins: override.bins
})
].join(',\n');
} else {
binsQuery = [
iqrQueryTpl({
_query: _query,
_column: _column
}),
binsQueryTpl({
_query: _query,
_minBins: BIN_MIN_NUMBER,
_maxBins: BIN_MAX_NUMBER
})
].join(',\n');
}
}
var histogramSql = [
"WITH",
[
basicsQuery,
binsQuery,
nullsQueryTpl({
_query: _query,
_column: _column
})
].join(',\n'),
histogramQueryTpl({
_query: _query,
_column: _column
})
].join('\n');
return callback(null, histogramSql);
};
lib/cartodb/models/dataview/overviews/index.js
+4 -1
View File
@@ -1,3 +1,6 @@
module.exports = {
Formula: require('./formula')
Aggregation: require('./aggregation'),
Formula: require('./formula'),
Histogram: require('./histogram'),
List: require('./list')
};
lib/cartodb/models/dataview/overviews/list.js
+11
View File
@@ -0,0 +1,11 @@
var BaseOverviewsDataview = require('./base');
var BaseDataview = require('../list');
function List(query, options, queryRewriter, queryRewriteData, params) {
BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params);
}
List.prototype = Object.create(BaseOverviewsDataview.prototype);
List.prototype.constructor = List;
module.exports = List;