diff --git a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js index b590f979..ac62ac2b 100644 --- a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js +++ b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -9,6 +9,8 @@ function MapnikLayerStats () { }; } +// TODO: estimatedFeatureCount is post-aggregation; the rest is pre; distinguish and complement + MapnikLayerStats.prototype.is = function (type) { return this._types[type] ? this._types[type] : false; }; @@ -41,7 +43,7 @@ function columnAggregations(field) { function firstPhaseQueries(queries, ctx) { if (queries.results.estimatedFeatureCount === undefined) { queries.task( - queryPromise(ctx.dbConnection, queryUtils.getQueryRowEstimation(ctx.query), function(err, res) { + queryPromise(ctx.dbConnection, queryUtils.getQueryRowEstimation(ctx.aggrQuery), function(err, res) { if (err) { // at least for debugging we should err queries.results.estimatedFeatureCount = -1; @@ -56,11 +58,13 @@ function firstPhaseQueries(queries, ctx) { } if (ctx.metaOptions.featureCount) { + // TODO: pre/aggr + // TODO: for pre, use ctx.aggrMeta.pre_aggregation_count // TODO: if ctx.metaOptions.columnStats we can combine this with column stats query queries.task( queryPromise( ctx.dbConnection, - queryUtils.getQueryActualRowCount(ctx.rawQuery), + queryUtils.getQueryActualRowCount(ctx.preQuery), (err, res) => { if (err) { queries.results.featureCount = -1; @@ -74,11 +78,13 @@ function firstPhaseQueries(queries, ctx) { } if (ctx.metaOptions.geometryType && queries.results.geometryType === undefined) { + // TODO: pre/aggr + // TODO: for pre, use ctx.aggrMeta.geometry_type const geometryColumn = AggregationMapConfig.getAggregationGeometryColumn(); queries.task( queryPromise( ctx.dbConnection, - queryUtils.getQueryGeometryType(ctx.rawQuery, geometryColumn), + queryUtils.getQueryGeometryType(ctx.preQuery, geometryColumn), (err, res) => { if (!err) { queries.results.geometryType = res.rows[0].geom_type; @@ -90,12 +96,15 @@ function firstPhaseQueries(queries, ctx) { } if (ctx.metaOptions.columns || ctx.metaOptions.columnStats) { + // TODO: pre/aggr + // TODO: for aggr, use layer.options.columns (will need to pass in ctx) + // note: post-aggregation columns are in layer.options.columns when aggregation is present queries.task( // TODO: note we have getLayerColumns in aggregation mapconfig. // and also getLayerAggregationColumns which either uses getLayerColumns or derives columns from parameters queryPromise( ctx.dbConnection, - queryUtils.getQueryLimited(ctx.rawQuery, 0), + queryUtils.getQueryLimited(ctx.preQuery, 0), (err, res) => { if (!err) { queries.results.columns = formatResultFields(ctx.dbConnection, res.fields); @@ -116,7 +125,7 @@ function secondPhaseQueries(queries, ctx) { queries.task( queryPromise( ctx.dbConnection, - queryUtils.getQuerySample(ctx.rawQuery, sampleProb), + queryUtils.getQuerySample(ctx.preQuery, sampleProb), (err, res) => { if (err) { queries.results.sample = []; @@ -130,6 +139,7 @@ function secondPhaseQueries(queries, ctx) { } if (ctx.metaOptions.columnStats) { + // TODO: pre/aggr let aggr = []; Object.keys(queries.results.columns).forEach(name => { aggr = aggr.concat( @@ -143,7 +153,7 @@ function secondPhaseQueries(queries, ctx) { queries.task( queryPromise( ctx.dbConnection, - queryUtils.getQueryTopCategories(ctx.rawQuery, name, topN), + queryUtils.getQueryTopCategories(ctx.preQuery, name, topN), (err, res) => { if (!err) { queries.results.columns[name].categories = res.rows; @@ -157,7 +167,7 @@ function secondPhaseQueries(queries, ctx) { queries.task( queryPromise( ctx.dbConnection, - `SELECT ${aggr.join(',')} FROM (${ctx.rawQuery}) AS __cdb_query`, + `SELECT ${aggr.join(',')} FROM (${ctx.preQuery}) AS __cdb_query`, (err, res) => { if (!err) { Object.keys(queries.results.columns).forEach(name => { @@ -218,10 +228,14 @@ function formatResultFields(dbConnection, flds) { MapnikLayerStats.prototype.getStats = function (layer, dbConnection, callback) { + let aggrQuery = layer.options.sql_raw || layer.options.sql; + let preQuery = layer.options.aggregation_metadata ? layer.options.aggregation_metadata.pre_aggregation_sql : aggrQuery; + let context = { dbConnection, - query: layer.options.sql, - rawQuery: layer.options.sql_raw ? layer.options.sql_raw : layer.options.sql, + preQuery, + aggrQuery, + aggrMeta: layer.options.aggregation_metadata, metaOptions: layer.options.metadata || {} }; diff --git a/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js b/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js index 67c5eac4..8639b85b 100644 --- a/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js @@ -73,7 +73,7 @@ module.exports = class AggregationMapConfigAdapter { if (adapted) { requestMapConfig.layers[index] = layer; } - const aggregatedFormats = this._getAggregationMetadata(mapConfig, layer, adapted); + const aggregatedFormats = this._getAggregationMetadata(mapConfig, layer, adapted); // <<- context.aggregation.layers.push(aggregatedFormats); }); @@ -84,7 +84,7 @@ module.exports = class AggregationMapConfigAdapter { _adaptLayer (connection, mapConfig, layer, index) { return new Promise((resolve, reject) => { - this._shouldAdaptLayer(connection, mapConfig, layer, index, (err, shouldAdapt) => { + this._shouldAdaptLayer(connection, mapConfig, layer, index, (err, shouldAdapt, aggrMeta) => { if (err) { return reject(err); } @@ -93,6 +93,7 @@ module.exports = class AggregationMapConfigAdapter { return resolve({ layer, index, adapted: shouldAdapt }); } + const sqlQuery = layer.options.sql; const sqlQueryWrap = layer.options.sql_wrap; let aggregationSql = mapConfig.getAggregatedQuery(index); @@ -110,6 +111,12 @@ module.exports = class AggregationMapConfigAdapter { layer.options.columns = columns; + layer.options.aggregation_metadata = { + pre_aggregation_sql: sqlQueryWrap || sqlQuery, + geometry_type: aggrMeta.type, + pre_aggregation_count: aggrMeta.count + }; + return resolve({ layer, index, adapted: shouldAdapt }); }); }); @@ -154,11 +161,12 @@ module.exports = class AggregationMapConfigAdapter { return callback(null, false); } - callback(null, true); + callback(null, true, result); }); } _getAggregationMetadata (mapConfig, layer, adapted) { + // also: pre-aggr query, columns, ... if (!adapted) { return { png: false, mvt: false }; } diff --git a/test/acceptance/stats/mapnik_stats_layergroup.js b/test/acceptance/stats/mapnik_stats_layergroup.js index 508878cf..d8a9a06b 100644 --- a/test/acceptance/stats/mapnik_stats_layergroup.js +++ b/test/acceptance/stats/mapnik_stats_layergroup.js @@ -474,4 +474,6 @@ describe('Create mapnik layergroup', function() { testClient.drain(done); }); }); + + // TODO: add tests for metadata with aggregation });