cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add overviews-dependent dataviews behaviour

Browse Source 
Now QueryRewriter is used in dataview objects they can decide
whether overviews are applicable, have the oportunity to
adapt queries for overviews, etc.
This is done by having overviews-related behaviour in models/dataview/overviews
and falling back to the regular models/dataview.
This commit is contained in:
Javier Goizueta
2016-05-13 18:46:58 +02:00
parent b574489950
commit 224eb392ba
5 changed files with 138 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/cartodb/backends/dataview.js
+6 -3
View File
@@ -8,7 +8,9 @@ var step = require('step');
var Timer = require('../stats/timer');
var BBoxFilter = require('../models/filter/bbox');
var DataviewFactory = require('../models/dataview/factory');
var DataviewFactoryWithOverviews = require('../models/dataview/overviews/factory');
var OverviewsQueryRewriter = require('../utils/overviews_query_rewriter');
var overviewsQueryRewriter = new OverviewsQueryRewriter({
zoom_level: 'CDB_ZoomFromScale(!scale_denominator!)'
@@ -132,16 +134,15 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param
{},
queryRewriteData, { filters: filters, unfiltered_query: unfiltered_query }
);
query = overviewsQueryRewriter.query(query, queryRewriteData, { bounding_box: params.bbox });
}
}
var dataviewFactory = DataviewFactoryWithOverviews.getFactory(overviewsQueryRewriter, queryRewriteData);
if (params.bbox) {
var bboxFilter = new BBoxFilter({column: 'the_geom', srid: 4326}, {bbox: params.bbox});
query = bboxFilter.sql(query);
}
var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins'),
function castNumbers(overrides, val, k) {
overrides[k] = Number.isFinite(+val) ? +val : val;
@@ -150,7 +151,7 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param
{ownFilter: ownFilter}
);
var dataview = DataviewFactory.getDataview(query, dataviewDefinition);
var dataview = dataviewFactory.getDataview(query, dataviewDefinition);
dataview.getResult(pg, overrideParams, this);
},
function returnCallback(err, result) {
@@ -253,6 +254,8 @@ DataviewBackend.prototype.search = function (mapConfigProvider, user, params, ca
query = node.getQuery(applyFilters);
}
// TODO: should handle overviews as getDataview ?
if (params.bbox) {
var bboxFilter = new BBoxFilter({column: 'the_geom', srid: 4326}, {bbox: params.bbox});
query = bboxFilter.sql(query);
lib/cartodb/models/dataview/overviews/factory.js
+29
View File
@@ -0,0 +1,29 @@
var parentFactory = require('../factory');
var dataviews = require('./');
function OverviewsDataviewFactory(queryRewriter, queryRewriteData) {
this.queryRewriter = queryRewriter;
this.queryRewriteData = queryRewriteData;
}
OverviewsDataviewFactory.prototype.getDataview = function(query, dataviewDefinition) {
var type = dataviewDefinition.type;
var dataviews = OverviewsDataviewMetaFactory.dataviews;
if ( !this.queryRewriter || !this.queryRewriteData || !dataviews[type] ) {
return parentFactory.getDataview(query, dataviewDefinition);
}
return new dataviews[type](query, dataviewDefinition.options, this.queryRewriter, this.queryRewriteData);
};
var OverviewsDataviewMetaFactory = {
dataviews: Object.keys(dataviews).reduce(function(allDataviews, dataviewClassName) {
allDataviews[dataviewClassName.toLowerCase()] = dataviews[dataviewClassName];
return allDataviews;
}, {}),
getFactory: function(queryRewriter, queryRewriteData) {
return new OverviewsDataviewFactory(queryRewriter, queryRewriteData);
},
};
module.exports = OverviewsDataviewMetaFactory;
lib/cartodb/models/dataview/overviews/formula.js
+84
View File
@@ -0,0 +1,84 @@
var BaseWidget = require('../base');
var BaseDataview = require('../formula');
var debug = require('debug')('windshaft:widget:formula:overviews');
var dot = require('dot');
dot.templateSettings.strip = false;
var formulaQueryTpls = {
'count': dot.template([
'SELECT',
'sum(_feature_count) AS result,',
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
'FROM ({{=it._query}}) _cdb_formula'
].join('\n')),
'sum': dot.template([
'SELECT',
'sum({{=it._column}}*_feature_count) AS result,',
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
'FROM ({{=it._query}}) _cdb_formula'
].join('\n')),
'avg': dot.template([
'SELECT',
'sum({{=it._column}}*_feature_count)/sum(_feature_count) AS result,',
'(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
'FROM ({{=it._query}}) _cdb_formula'
].join('\n')),
};
function Formula(query, options, queryRewriter, queryRewriteData) {
this.base_dataview = new BaseDataview(query, options);
this.query = query;
this.column = options.column || '1';
this.operation = options.operation;
this.queryRewriter = queryRewriter;
this.queryRewriteData = queryRewriteData;
}
Formula.prototype = new BaseWidget();
Formula.prototype.constructor = Formula;
module.exports = Formula;
Formula.prototype.sql = function(psql, filters, override, callback) {
var _query = this.query;
var formulaQueryTpl = formulaQueryTpls[this.operation];
if ( formulaQueryTpl ) {
// supported formula for use with overviews
// TODO: determine zoom level using bounding box so that the resolution
// (grid size) of the overview much smaller thatn the bounding box.
// This could be left to be computed by the queryRewriter passing the
// bounding box to it.
// The bounding box could be passed to the dataview constructor, or
// it could b extracted from the query.
var zoom_level = 0;
_query = this.queryRewriter.query(_query, this.queryRewriteData, { zoom_level: zoom_level });
var formulaSql = formulaQueryTpl({
_query: _query,
_operation: this.operation,
_column: this.column
});
debug(formulaSql);
callback = callback || override;
return callback(null, formulaSql);
}
// For non supported operations (min, max) we're not using overviews.
return this.base_dataview.sql(psql, filters, override, callback);
};
Formula.prototype.format = function(result) {
return this.base_dataview.format(result);
};
Formula.prototype.getType = function() {
return this.base_dataview.getType();
};
Formula.prototype.toString = function() {
return this.base_dataview.toString();
};
lib/cartodb/models/dataview/overviews/index.js
+3
View File
@@ -0,0 +1,3 @@
module.exports = {
Formula: require('./formula')
};
lib/cartodb/utils/overviews_query_rewriter.js
+16 -23
View File
@@ -1,9 +1,15 @@
var _ = require('underscore');
var TableNameParser = require('./table_name_parser');
// this is meant as a hack for development (otherwise we'd be coupling
// this to Camshaft impolementation)
// this is meant as a hack for development
var queryBuilder = require('camshaft/lib/filter/query-builder');
function apply_filters_to_query(query, filters) {
// TODO: implement filter application here
if ( filters && !_.isEmpty(filters)) {
query = queryBuilder.getSql(query, filters);
}
return query;
}
function OverviewsQueryRewriter(options) {
@@ -196,20 +202,6 @@ function overview_table_for_zoom_level(table_overviews, zoom_level) {
return overview_table;
}
function apply_filters_to_query(query, filters) {
// TODO: it'd be nice to have a public API in camshaft to apply filters!
if ( filters && !_.isEmpty(filters)) {
query = queryBuilder.getSql(query, filters);
}
return query;
}
function zoom_level_for_bounding_box(query, bounding_box) {
// TODO: implement
// jshint unused:false
return '100'; // force use of base table
}
// Transform an SQL query so that it uses overviews.
// overviews contains metadata about the overviews to be used:
// { 'table-name': {1: { table: 'overview-table-1' }, ... }, ... }
@@ -228,7 +220,7 @@ function zoom_level_for_bounding_box(query, bounding_box) {
// doesnn't prevent substitution inside literals).
// But the transformation will currently only be applied to simple queries
// of the form detected by the overviews_supported_query function.
// TODO: document recent changes (options bounding_box, zoom_level, filters...)
// TODO: document recent changes (zoom_level option, filters...)
OverviewsQueryRewriter.prototype.query = function(query, data, options) {
options = options || {};
@@ -255,6 +247,11 @@ OverviewsQueryRewriter.prototype.query = function(query, data, options) {
// could not or didn't need to alter the query
rewritten_query = query;
} else {
// TODO: determine if overviews can be used for the filters.
// Otherwise use the base table.
// Applicability of the filters should depend both on the expected
// row count of the result and on the columns to be filter and the
// way they're agregated in overviews.
rewritten_query = apply_filters_to_query(rewritten_query, filters);
}
@@ -263,12 +260,8 @@ OverviewsQueryRewriter.prototype.query = function(query, data, options) {
function zoom_level_for_query(query, zoom_level_expression, options) {
var zoom_level = null;
if ( _.has(options, 'bounding_box') ) {
// The presence of this key, even if it has a null value indicates the
// query cannot handle dynamic zoom expressinos (is not for the tiler)
zoom_level = zoom_level_for_bounding_box(query, options.bounding_box);
} else if ( _.has(options, 'zoom_level') ) {
zoom_level = options.zoom_level;
if ( _.has(options, 'zoom_level') ) {
zoom_level = options.zoom_level || '0';
}
if ( zoom_level === null && !zoom_level_expression ) {
zoom_level = '0';