cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add filter stats information to query rewriter data

Browse Source
This commit is contained in:
Javier Goizueta
2016-05-17 15:41:31 +02:00
parent 3c6d930434
commit 7f7204df6c
4 changed files with 133 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
lib/cartodb/api/filter_stats_api.js Normal file
View File

@@ -0,0 +1,63 @@
var _ = require('underscore');
var step = require('step');
var CamshaftFilter = require('../models/filter/camshaft');
function FilterStatsApi(pgQueryRunner) {
this.pgQueryRunner = pgQueryRunner;
}
module.exports = FilterStatsApi;
function getEstimatedRows(pgQueryRunner, username, query, callback) {
pgQueryRunner.run(username, "EXPLAIN "+query, function(err, result_rows) {
if (err){
callback(err);
return;
}
var rows;
var query_plan = result_rows[0]['QUERY PLAN'];
var match;
if ( query_plan ) {
match = query_plan.match(/rows=(\d+)/);
}
if ( match ) {
rows = +match[1];
}
return callback(null, rows);
});
}
FilterStatsApi.prototype.getFilterStats = function (username, unfiltered_query, filters, callback) {
var stats = {};
var self = this;
step(
function getUnfilteredRows() {
getEstimatedRows(self.pgQueryRunner, username, unfiltered_query, this);
},
function receiveUnfilteredRows(err, rows) {
if (err){
callback(err);
return;
}
stats.unfiltered_rows = rows;
this(null, rows);
},
function getFilteredRows() {
if ( filters && !_.isEmpty(filters)) {
var camshaftFilter = new CamshaftFilter(filters);
var query = camshaftFilter.sql(unfiltered_query);
getEstimatedRows(self.pgQueryRunner, username, query, this);
} else {
this(null, null);
}
},
function receiveFilteredRows(err, rows) {
if (err){
callback(err);
return;
}
stats.filtered_rows = rows;
callback(null, stats);
}
);
};

63
lib/cartodb/models/mapconfig_overviews_adapter.js
View File

@@ -1,8 +1,10 @@
var step = require('step');
var queue = require('queue-async');
var _ = require('underscore');
function MapConfigOverviewsAdapter(overviewsMetadataApi) {
function MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi) {
this.overviewsMetadataApi = overviewsMetadataApi;
this.filterStatsApi = filterStatsApi;
}
module.exports = MapConfigOverviewsAdapter;
@@ -25,26 +27,47 @@ MapConfigOverviewsAdapter.prototype.getLayers = function(username, layers, analy
done(err, layer);
} else {
var query_rewrite_data = { overviews: metadata };
if ( layer.options.source && analysesResults ) {
var sourceId = layer.options.source.id;
var node = _.find(analysesResults, function(a){ return a.rootNode.params.id === sourceId; });
if ( node ) {
node = node.rootNode;
var filters = node.filters; // TODO: node.getFilters() when available in camshaft
var filters_disabler = _.keys(filters).reduce(
function(disabler, filter_id){ disabler[filter_id] = false; return disabler; },
{}
);
var unfiltered_query = node.getQuery(filters_disabler);
query_rewrite_data.filters = filters;
query_rewrite_data.unfiltered_query = unfiltered_query;
step(
function collectFiltersData() {
var filters, unfiltered_query;
if ( layer.options.source && analysesResults ) {
var sourceId = layer.options.source.id;
var node = _.find(analysesResults, function(a){ return a.rootNode.params.id === sourceId; });
if ( node ) {
node = node.rootNode;
filters = node.filters; // TODO: node.getFilters() when available in camshaft
var filters_disabler = _.keys(filters).reduce(
function(disabler, filter_id){ disabler[filter_id] = false; return disabler; },
{}
);
unfiltered_query = node.getQuery(filters_disabler);
query_rewrite_data.filters = filters;
query_rewrite_data.unfiltered_query = unfiltered_query;
}
}
this(null, filters, unfiltered_query);
},
function collectStatsData(err, filters, unfiltered_query) {
var next_step = this;
if ( filters ) {
self.filterStatsApi.getFilterStats(username, unfiltered_query, filters, function(err, stats) {
if ( !err ) {
query_rewrite_data.filter_stats = stats;
}
return next_step(err);
});
} else {
return next_step(null);
}
},
function addDataToLayer(err) {
if ( !err && !_.isEmpty(metadata) ) {
layer = _.extend({}, layer);
layer.options = _.extend({}, layer.options, { query_rewrite_data: query_rewrite_data });
}
done(err, layer);
}
}
if ( !_.isEmpty(metadata) ) {
layer = _.extend({}, layer);
layer.options = _.extend({}, layer.options, { query_rewrite_data: query_rewrite_data });
}
done(null, layer);
);
}
});
}

4
lib/cartodb/server.js
View File

@@ -21,6 +21,7 @@ var mapnik = windshaft.mapnik;
var TemplateMaps = require('./backends/template_maps.js');
var OverviewsMetadataApi = require('./api/overviews_metadata_api');
var FilterStatsApi = require('./api/filter_stats_api');
var UserLimitsApi = require('./api/user_limits_api');
var AuthApi = require('./api/auth_api');
var LayergroupAffectedTablesCache = require('./cache/layergroup_affected_tables');
@@ -59,6 +60,7 @@ module.exports = function(serverOptions) {
var pgConnection = new PgConnection(metadataBackend);
var pgQueryRunner = new PgQueryRunner(pgConnection);
var overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner);
var filterStatsApi = new FilterStatsApi(pgQueryRunner);
var userLimitsApi = new UserLimitsApi(metadataBackend, {
limits: {
cacheOnTimeout: serverOptions.renderer.mapnik.limits.cacheOnTimeout || false,
@@ -148,7 +150,7 @@ module.exports = function(serverOptions) {
var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache();
app.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
var overviewsAdapter = new MapConfigOverviewsAdapter(overviewsMetadataApi);
var overviewsAdapter = new MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi);
var turboCartoParser = new TurboCartoParser(pgQueryRunner);
var turboCartoAdapter = new TurboCartoAdapter(turboCartoParser);

28
lib/cartodb/utils/overviews_query_rewriter.js
View File

@@ -4,9 +4,14 @@ var TableNameParser = require('./table_name_parser');
var BBoxFilter = require('../models/filter/bbox');
var CamshaftFilter = require('../models/filter/camshaft');
// Minimim number of filtered rows to use overviews
var FILTER_MIN_ROWS = 65536;
// Maximum filtered fraction to not apply overviews
var FILTER_MAX_FRACTION = 0.2;
function apply_filters_to_query(query, filters, bbox_filter) {
if ( filters && !_.isEmpty(filters)) {
var camshaftFilter = new CamshaftFilter(filters)
var camshaftFilter = new CamshaftFilter(filters);
query = camshaftFilter.sql(query);
}
if ( bbox_filter ) {
@@ -238,7 +243,7 @@ OverviewsQueryRewriter.prototype.query = function(query, data, options) {
unfiltered_query = query;
}
if ( !overviews || !this.is_supported_query(unfiltered_query)) {
if ( !should_use_overviews(unfiltered_query, data) ) {
return query;
}
@@ -283,7 +288,22 @@ function overviews_query(query, overviews, zoom_level, zoom_level_expression) {
}
}
OverviewsQueryRewriter.prototype.is_supported_query = function(sql) {
function should_use_overviews(query, data) {
data = data || {};
var use_overviews = data.overviews && is_supported_query(query);
if ( use_overviews ) {
if ( data.filters && data.filter_stats ) {
var filtered_rows = data.filter_stats.filtered_rows;
var unfiltered_rows = data.filter_stats.unfiltered_rows;
if ( unfiltered_rows && (filtered_rows || filtered_rows === 0) ) {
use_overviews = filtered_rows >= FILTER_MIN_ROWS || (filtered_rows/unfiltered_rows) > FILTER_MAX_FRACTION;
}
}
}
return use_overviews;
}
function is_supported_query(sql) {
var basic_query =
/\s*SELECT\s+[\*a-z0-9_,\s]+?\s+FROM\s+((\"[^"]+\"|[a-z0-9_]+)\.)?(\"[^"]+\"|[a-z0-9_]+)\s*;?\s*/i;
var unwrapped_query = new RegExp("^"+basic_query.source+"$", 'i');
@@ -295,4 +315,4 @@ OverviewsQueryRewriter.prototype.is_supported_query = function(sql) {
'i'
);
return !!(sql.match(unwrapped_query) || sql.match(wrapped_query));
};
}