From 7f7204df6cc993d7e84b08b38ecde6873f568547 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Tue, 17 May 2016 15:41:31 +0200 Subject: [PATCH] Add filter stats information to query rewriter data --- lib/cartodb/api/filter_stats_api.js | 63 +++++++++++++++++++ .../models/mapconfig_overviews_adapter.js | 63 +++++++++++++------ lib/cartodb/server.js | 4 +- lib/cartodb/utils/overviews_query_rewriter.js | 28 +++++++-- 4 files changed, 133 insertions(+), 25 deletions(-) create mode 100644 lib/cartodb/api/filter_stats_api.js diff --git a/lib/cartodb/api/filter_stats_api.js b/lib/cartodb/api/filter_stats_api.js new file mode 100644 index 00000000..5196d2fa --- /dev/null +++ b/lib/cartodb/api/filter_stats_api.js @@ -0,0 +1,63 @@ +var _ = require('underscore'); +var step = require('step'); +var CamshaftFilter = require('../models/filter/camshaft'); + +function FilterStatsApi(pgQueryRunner) { + this.pgQueryRunner = pgQueryRunner; +} + +module.exports = FilterStatsApi; + +function getEstimatedRows(pgQueryRunner, username, query, callback) { + pgQueryRunner.run(username, "EXPLAIN "+query, function(err, result_rows) { + if (err){ + callback(err); + return; + } + var rows; + var query_plan = result_rows[0]['QUERY PLAN']; + var match; + if ( query_plan ) { + match = query_plan.match(/rows=(\d+)/); + } + if ( match ) { + rows = +match[1]; + } + return callback(null, rows); + }); +} + +FilterStatsApi.prototype.getFilterStats = function (username, unfiltered_query, filters, callback) { + var stats = {}; + var self = this; + step( + function getUnfilteredRows() { + getEstimatedRows(self.pgQueryRunner, username, unfiltered_query, this); + }, + function receiveUnfilteredRows(err, rows) { + if (err){ + callback(err); + return; + } + stats.unfiltered_rows = rows; + this(null, rows); + }, + function getFilteredRows() { + if ( filters && !_.isEmpty(filters)) { + var camshaftFilter = new CamshaftFilter(filters); + var query = camshaftFilter.sql(unfiltered_query); + getEstimatedRows(self.pgQueryRunner, username, query, this); + } else { + this(null, null); + } + }, + function receiveFilteredRows(err, rows) { + if (err){ + callback(err); + return; + } + stats.filtered_rows = rows; + callback(null, stats); + } + ); +}; diff --git a/lib/cartodb/models/mapconfig_overviews_adapter.js b/lib/cartodb/models/mapconfig_overviews_adapter.js index 47185e00..d3fcedc0 100644 --- a/lib/cartodb/models/mapconfig_overviews_adapter.js +++ b/lib/cartodb/models/mapconfig_overviews_adapter.js @@ -1,8 +1,10 @@ +var step = require('step'); var queue = require('queue-async'); var _ = require('underscore'); -function MapConfigOverviewsAdapter(overviewsMetadataApi) { +function MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi) { this.overviewsMetadataApi = overviewsMetadataApi; + this.filterStatsApi = filterStatsApi; } module.exports = MapConfigOverviewsAdapter; @@ -25,26 +27,47 @@ MapConfigOverviewsAdapter.prototype.getLayers = function(username, layers, analy done(err, layer); } else { var query_rewrite_data = { overviews: metadata }; - if ( layer.options.source && analysesResults ) { - var sourceId = layer.options.source.id; - var node = _.find(analysesResults, function(a){ return a.rootNode.params.id === sourceId; }); - if ( node ) { - node = node.rootNode; - var filters = node.filters; // TODO: node.getFilters() when available in camshaft - var filters_disabler = _.keys(filters).reduce( - function(disabler, filter_id){ disabler[filter_id] = false; return disabler; }, - {} - ); - var unfiltered_query = node.getQuery(filters_disabler); - query_rewrite_data.filters = filters; - query_rewrite_data.unfiltered_query = unfiltered_query; + step( + function collectFiltersData() { + var filters, unfiltered_query; + if ( layer.options.source && analysesResults ) { + var sourceId = layer.options.source.id; + var node = _.find(analysesResults, function(a){ return a.rootNode.params.id === sourceId; }); + if ( node ) { + node = node.rootNode; + filters = node.filters; // TODO: node.getFilters() when available in camshaft + var filters_disabler = _.keys(filters).reduce( + function(disabler, filter_id){ disabler[filter_id] = false; return disabler; }, + {} + ); + unfiltered_query = node.getQuery(filters_disabler); + query_rewrite_data.filters = filters; + query_rewrite_data.unfiltered_query = unfiltered_query; + } + } + this(null, filters, unfiltered_query); + }, + function collectStatsData(err, filters, unfiltered_query) { + var next_step = this; + if ( filters ) { + self.filterStatsApi.getFilterStats(username, unfiltered_query, filters, function(err, stats) { + if ( !err ) { + query_rewrite_data.filter_stats = stats; + } + return next_step(err); + }); + } else { + return next_step(null); + } + }, + function addDataToLayer(err) { + if ( !err && !_.isEmpty(metadata) ) { + layer = _.extend({}, layer); + layer.options = _.extend({}, layer.options, { query_rewrite_data: query_rewrite_data }); + } + done(err, layer); } - } - if ( !_.isEmpty(metadata) ) { - layer = _.extend({}, layer); - layer.options = _.extend({}, layer.options, { query_rewrite_data: query_rewrite_data }); - } - done(null, layer); + ); } }); } diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index ed24f651..3feee602 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -21,6 +21,7 @@ var mapnik = windshaft.mapnik; var TemplateMaps = require('./backends/template_maps.js'); var OverviewsMetadataApi = require('./api/overviews_metadata_api'); +var FilterStatsApi = require('./api/filter_stats_api'); var UserLimitsApi = require('./api/user_limits_api'); var AuthApi = require('./api/auth_api'); var LayergroupAffectedTablesCache = require('./cache/layergroup_affected_tables'); @@ -59,6 +60,7 @@ module.exports = function(serverOptions) { var pgConnection = new PgConnection(metadataBackend); var pgQueryRunner = new PgQueryRunner(pgConnection); var overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner); + var filterStatsApi = new FilterStatsApi(pgQueryRunner); var userLimitsApi = new UserLimitsApi(metadataBackend, { limits: { cacheOnTimeout: serverOptions.renderer.mapnik.limits.cacheOnTimeout || false, @@ -148,7 +150,7 @@ module.exports = function(serverOptions) { var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache(); app.layergroupAffectedTablesCache = layergroupAffectedTablesCache; - var overviewsAdapter = new MapConfigOverviewsAdapter(overviewsMetadataApi); + var overviewsAdapter = new MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi); var turboCartoParser = new TurboCartoParser(pgQueryRunner); var turboCartoAdapter = new TurboCartoAdapter(turboCartoParser); diff --git a/lib/cartodb/utils/overviews_query_rewriter.js b/lib/cartodb/utils/overviews_query_rewriter.js index e38f53bd..4025e3f0 100644 --- a/lib/cartodb/utils/overviews_query_rewriter.js +++ b/lib/cartodb/utils/overviews_query_rewriter.js @@ -4,9 +4,14 @@ var TableNameParser = require('./table_name_parser'); var BBoxFilter = require('../models/filter/bbox'); var CamshaftFilter = require('../models/filter/camshaft'); +// Minimim number of filtered rows to use overviews +var FILTER_MIN_ROWS = 65536; +// Maximum filtered fraction to not apply overviews +var FILTER_MAX_FRACTION = 0.2; + function apply_filters_to_query(query, filters, bbox_filter) { if ( filters && !_.isEmpty(filters)) { - var camshaftFilter = new CamshaftFilter(filters) + var camshaftFilter = new CamshaftFilter(filters); query = camshaftFilter.sql(query); } if ( bbox_filter ) { @@ -238,7 +243,7 @@ OverviewsQueryRewriter.prototype.query = function(query, data, options) { unfiltered_query = query; } - if ( !overviews || !this.is_supported_query(unfiltered_query)) { + if ( !should_use_overviews(unfiltered_query, data) ) { return query; } @@ -283,7 +288,22 @@ function overviews_query(query, overviews, zoom_level, zoom_level_expression) { } } -OverviewsQueryRewriter.prototype.is_supported_query = function(sql) { +function should_use_overviews(query, data) { + data = data || {}; + var use_overviews = data.overviews && is_supported_query(query); + if ( use_overviews ) { + if ( data.filters && data.filter_stats ) { + var filtered_rows = data.filter_stats.filtered_rows; + var unfiltered_rows = data.filter_stats.unfiltered_rows; + if ( unfiltered_rows && (filtered_rows || filtered_rows === 0) ) { + use_overviews = filtered_rows >= FILTER_MIN_ROWS || (filtered_rows/unfiltered_rows) > FILTER_MAX_FRACTION; + } + } + } + return use_overviews; +} + +function is_supported_query(sql) { var basic_query = /\s*SELECT\s+[\*a-z0-9_,\s]+?\s+FROM\s+((\"[^"]+\"|[a-z0-9_]+)\.)?(\"[^"]+\"|[a-z0-9_]+)\s*;?\s*/i; var unwrapped_query = new RegExp("^"+basic_query.source+"$", 'i'); @@ -295,4 +315,4 @@ OverviewsQueryRewriter.prototype.is_supported_query = function(sql) { 'i' ); return !!(sql.match(unwrapped_query) || sql.match(wrapped_query)); -}; +}