From 506e16fc879aeba2de898c483934a3944271751e Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Mon, 18 Dec 2017 20:18:37 +0100 Subject: [PATCH 01/27] Experimental full-sample aggregation --- .../models/aggregation/aggregation-query.js | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/models/aggregation/aggregation-query.js b/lib/cartodb/models/aggregation/aggregation-query.js index 69118021..03c99081 100644 --- a/lib/cartodb/models/aggregation/aggregation-query.js +++ b/lib/cartodb/models/aggregation/aggregation-query.js @@ -186,5 +186,30 @@ const aggregationQueryTemplates = { FROM _cdb_clusters INNER JOIN (${ctx.sourceQuery}) _cdb_query ON (_cdb_clusters.cartodb_id = _cdb_query.cartodb_id) - ` + `, + + 'full-sample': ctx => ` + WITH _cdb_params AS ( + SELECT + ${gridResolution(ctx)} AS res, + !bbox! AS bbox + ), _cdb_clusters AS ( + SELECT + MIN(cartodb_id) AS cartodb_id + ${dimensionDefs(ctx)} + ${aggregateColumnDefs(ctx)} + FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params + WHERE _cdb_query.the_geom_webmercator && _cdb_params.bbox + GROUP BY + Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res), + Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res) + ${dimensionNames(ctx)} + ) SELECT + _cdb_query.* + ${aggregateColumnNames(ctx)} + FROM + _cdb_clusters INNER JOIN (${ctx.sourceQuery}) _cdb_query + ON (_cdb_clusters.cartodb_id = _cdb_query.cartodb_id) +` + }; From bcd2fd8f88fee6997882400a39b2f80c4a81c721 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 19 Dec 2017 12:59:33 +0100 Subject: [PATCH 02/27] Export supported placements --- .../models/aggregation/aggregation-query.js | 43 ++++++++++--------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/lib/cartodb/models/aggregation/aggregation-query.js b/lib/cartodb/models/aggregation/aggregation-query.js index 06697732..b98d0dbb 100644 --- a/lib/cartodb/models/aggregation/aggregation-query.js +++ b/lib/cartodb/models/aggregation/aggregation-query.js @@ -142,26 +142,26 @@ const aggregationQueryTemplates = { `, 'point-grid': ctx => ` - WITH _cdb_params AS ( + WITH _cdb_params AS ( + SELECT + ${gridResolution(ctx)} AS res, + !bbox! AS bbox + ), + _cdb_clusters AS ( + SELECT + Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res)::int AS _cdb_gx, + Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)::int AS _cdb_gy + ${dimensionDefs(ctx)} + ${aggregateColumnDefs(ctx)} + FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params + WHERE the_geom_webmercator && _cdb_params.bbox + GROUP BY _cdb_gx, _cdb_gy ${dimensionNames(ctx)} + ) SELECT - ${gridResolution(ctx)} AS res, - !bbox! AS bbox - ), - _cdb_clusters AS ( - SELECT - Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res)::int AS _cdb_gx, - Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)::int AS _cdb_gy - ${dimensionDefs(ctx)} - ${aggregateColumnDefs(ctx)} - FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params - WHERE the_geom_webmercator && _cdb_params.bbox - GROUP BY _cdb_gx, _cdb_gy ${dimensionNames(ctx)} - ) - SELECT - ST_SetSRID(ST_MakePoint((_cdb_gx+0.5)*res, (_cdb_gy+0.5)*res), 3857) AS the_geom_webmercator - ${dimensionNames(ctx)} - ${aggregateColumnNames(ctx)} - FROM _cdb_clusters, _cdb_params + ST_SetSRID(ST_MakePoint((_cdb_gx+0.5)*res, (_cdb_gy+0.5)*res), 3857) AS the_geom_webmercator + ${dimensionNames(ctx)} + ${aggregateColumnNames(ctx)} + FROM _cdb_clusters, _cdb_params `, 'point-sample': ctx => ` @@ -212,6 +212,7 @@ const aggregationQueryTemplates = { FROM _cdb_clusters INNER JOIN (${ctx.sourceQuery}) _cdb_query ON (_cdb_clusters.cartodb_id = _cdb_query.cartodb_id) -` - + ` }; + +module.exports.SUPPORTED_PLACEMENTS = Object.keys(aggregationQueryTemplates); From 5a3dd6a91467b9e8e3c3ae465a0821dabdd193d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 19 Dec 2017 13:00:18 +0100 Subject: [PATCH 03/27] Use supported placemets of aggregation-query --- .../models/aggregation/aggregation-mapconfig.js | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/cartodb/models/aggregation/aggregation-mapconfig.js b/lib/cartodb/models/aggregation/aggregation-mapconfig.js index c6dbcad4..f5d40c0c 100644 --- a/lib/cartodb/models/aggregation/aggregation-mapconfig.js +++ b/lib/cartodb/models/aggregation/aggregation-mapconfig.js @@ -10,15 +10,13 @@ const { module.exports = class AggregationMapConfig extends MapConfig { static get PLACEMENTS () { - return [ - 'centroid', - 'point-grid', - 'point-sample' - ]; + return aggregationQuery.SUPPORTED_PLACEMENTS; } static get PLACEMENT () { - return AggregationMapConfig.PLACEMENTS[0]; + return AggregationMapConfig.PLACEMENTS[ + AggregationMapConfig.PLACEMENTS.indexOf('centroid') + ]; } static get THRESHOLD () { From d2828ecaff1d985fd915f467527f7744ed6ef0b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 19 Dec 2017 13:07:57 +0100 Subject: [PATCH 04/27] Update test --- test/acceptance/aggregation.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/acceptance/aggregation.js b/test/acceptance/aggregation.js index 73284582..4c2d790e 100644 --- a/test/acceptance/aggregation.js +++ b/test/acceptance/aggregation.js @@ -762,10 +762,10 @@ describe('aggregation', function () { } assert.deepEqual(body, { - errors: [ 'Invalid placement. Valid values: centroid, point-grid, point-sample'], + errors: [ 'Invalid placement. Valid values: centroid, point-grid, point-sample, full-sample'], errors_with_context:[{ type: 'layer', - message: 'Invalid placement. Valid values: centroid, point-grid, point-sample', + message: 'Invalid placement. Valid values: centroid, point-grid, point-sample, full-sample', layer: { id: "layer0", index: 0, From 4946ca688c0c86912d43fc67f09b2f0660e2787c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 19 Dec 2017 16:17:13 +0100 Subject: [PATCH 05/27] Add test to check full-sample query --- test/acceptance/aggregation.js | 45 ++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/test/acceptance/aggregation.js b/test/acceptance/aggregation.js index 4c2d790e..517a5692 100644 --- a/test/acceptance/aggregation.js +++ b/test/acceptance/aggregation.js @@ -692,6 +692,51 @@ describe('aggregation', function () { }); }); + it('aggregates with full-sample placement', function (done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + resolution: 256, + aggregation: { + placement: 'point-grid', + columns: { + total: { + aggregate_function: 'sum', + aggregated_column: 'value' + }, + v_avg: { + aggregate_function: 'avg', + aggregated_column: 'value' + } + }, + threshold: 1 + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + + this.testClient.getTile(0, 0, 0, { format: 'mvt' }, function (err, res, mvt) { + if (err) { + return done(err); + } + + const geojsonTile = JSON.parse(mvt.toGeoJSONSync(0)); + + assert.ok(Array.isArray(geojsonTile.features)); + assert.ok(geojsonTile.features.length > 0); + + const feature = geojsonTile.features[0]; + + assert.ok(feature.properties.hasOwnProperty('value'), 'Missing value property'); + + done(); + }); + }); + it('should fail with bad resolution', function (done) { this.mapConfig = createVectorMapConfig([ { From cc9b190e5d6eb04e8970a3acc57fefd2de6425b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 19 Dec 2017 16:17:37 +0100 Subject: [PATCH 06/27] Minor style formats --- .../models/aggregation/aggregation-query.js | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/lib/cartodb/models/aggregation/aggregation-query.js b/lib/cartodb/models/aggregation/aggregation-query.js index b98d0dbb..fd860318 100644 --- a/lib/cartodb/models/aggregation/aggregation-query.js +++ b/lib/cartodb/models/aggregation/aggregation-query.js @@ -118,7 +118,8 @@ const gridResolution = ctx => `(${256*0.00028/ctx.res}*!scale_denominator!)::dou const aggregationQueryTemplates = { 'centroid': ctx => ` - WITH _cdb_params AS ( + WITH + _cdb_params AS ( SELECT ${gridResolution(ctx)} AS res, !bbox! AS bbox @@ -142,7 +143,8 @@ const aggregationQueryTemplates = { `, 'point-grid': ctx => ` - WITH _cdb_params AS ( + WITH + _cdb_params AS ( SELECT ${gridResolution(ctx)} AS res, !bbox! AS bbox @@ -165,11 +167,13 @@ const aggregationQueryTemplates = { `, 'point-sample': ctx => ` - WITH _cdb_params AS ( + WITH + _cdb_params AS ( SELECT ${gridResolution(ctx)} AS res, !bbox! AS bbox - ), _cdb_clusters AS ( + ), + _cdb_clusters AS ( SELECT MIN(cartodb_id) AS cartodb_id ${dimensionDefs(ctx)} @@ -180,7 +184,8 @@ const aggregationQueryTemplates = { Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res), Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res) ${dimensionNames(ctx)} - ) SELECT + ) + SELECT _cdb_clusters.cartodb_id, the_geom, the_geom_webmercator ${dimensionNames(ctx)} @@ -191,11 +196,13 @@ const aggregationQueryTemplates = { `, 'full-sample': ctx => ` - WITH _cdb_params AS ( + WITH + _cdb_params AS ( SELECT ${gridResolution(ctx)} AS res, !bbox! AS bbox - ), _cdb_clusters AS ( + ), + _cdb_clusters AS ( SELECT MIN(cartodb_id) AS cartodb_id ${dimensionDefs(ctx)} From 3c00266666fc749d7ae99eb5a16515ed3cc35c58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 21 Dec 2017 20:00:17 +0100 Subject: [PATCH 07/27] Add support for aggregated columns in mvt format --- .../models/aggregation/aggregation-mapconfig.js | 12 ++++++++++++ .../adapter/aggregation-mapconfig-adapter.js | 4 ++++ 2 files changed, 16 insertions(+) diff --git a/lib/cartodb/models/aggregation/aggregation-mapconfig.js b/lib/cartodb/models/aggregation/aggregation-mapconfig.js index bf334338..6e9b71d0 100644 --- a/lib/cartodb/models/aggregation/aggregation-mapconfig.js +++ b/lib/cartodb/models/aggregation/aggregation-mapconfig.js @@ -72,6 +72,18 @@ module.exports = class AggregationMapConfig extends MapConfig { }); } + getAggregatedColumns (index) { + const { aggregation } = this.getLayer(index).options; + + if (aggregation === undefined || aggregation.columns === undefined) { + return []; + } + + return aggregation.columns + .map(({ aggregated_column }) => aggregated_column) + .filter(aggregated_column => typeof aggregated_column === 'string'); + } + isAggregationMapConfig () { return this.isVectorOnlyMapConfig() || this.hasAnyLayerAggregation(); } diff --git a/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js b/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js index 425aade7..99f68a0f 100644 --- a/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js @@ -99,6 +99,10 @@ module.exports = class AggregationMapConfigAdapter { } layer.options.sql = aggregationSql; + + const aggregatedColumns = mapConfig.getAggregatedColumns(index); + + layer.options.columns = aggregatedColumns; } return resolve({ layer, index, adapted: shouldAdapt }); From e9a4fc4b2c1188e3262a820de95ac38e31e36d4b Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Fri, 22 Dec 2017 11:31:33 +0100 Subject: [PATCH 08/27] Use full-sample aggregation only as default Sampling is performed only when placement, columns or dimensions are specified; otherwise the regular centroid/grid-point/grid-center is used without sampling. --- .../aggregation/aggregation-mapconfig.js | 8 +- .../models/aggregation/aggregation-query.js | 98 ++++++--- test/acceptance/aggregation.js | 194 ++++++++++++++++-- 3 files changed, 249 insertions(+), 51 deletions(-) diff --git a/lib/cartodb/models/aggregation/aggregation-mapconfig.js b/lib/cartodb/models/aggregation/aggregation-mapconfig.js index bf334338..320239bf 100644 --- a/lib/cartodb/models/aggregation/aggregation-mapconfig.js +++ b/lib/cartodb/models/aggregation/aggregation-mapconfig.js @@ -16,10 +16,6 @@ module.exports = class AggregationMapConfig extends MapConfig { return aggregationQuery.SUPPORTED_PLACEMENTS; } - static get PLACEMENT () { - return AggregationMapConfig.PLACEMENTS.find(placement => placement === 'centroid'); - } - static get THRESHOLD () { return 1e5; // 100K } @@ -55,9 +51,11 @@ module.exports = class AggregationMapConfig extends MapConfig { getAggregatedQuery (index) { const { sql_raw, sql } = this.getLayer(index).options; const { + // The default aggregation has no placement, columns or dimensions; + // this enables the special "full-sample" aggregation. resolution = AggregationMapConfig.RESOLUTION, threshold = AggregationMapConfig.THRESHOLD, - placement = AggregationMapConfig.PLACEMENT, + placement = null, columns = {}, dimensions = {} } = this.getAggregation(index); diff --git a/lib/cartodb/models/aggregation/aggregation-query.js b/lib/cartodb/models/aggregation/aggregation-query.js index fd860318..ab56e9b5 100644 --- a/lib/cartodb/models/aggregation/aggregation-query.js +++ b/lib/cartodb/models/aggregation/aggregation-query.js @@ -1,8 +1,12 @@ +const DEFAULT_PLACEMENT = 'centroid'; + /** * Returns a template function (function that accepts template parameters and returns a string) * to generate an aggregation query. * Valid options to define the query template are: * - placement + * - columns + * - dimensions* * The query template parameters taken by the result template function are: * - sourceQuery * - res @@ -10,13 +14,28 @@ * - dimensions */ const templateForOptions = (options) => { - let templateFn = aggregationQueryTemplates[options.placement]; - if (!templateFn) { - throw new Error("Invalid Aggregation placement: '" + options.placement + "'"); + let templateFn = defaultAggregationQueryTemplate; + if (!isDefaultAggregation(options)) { + templateFn = aggregationQueryTemplates[options.placement || DEFAULT_PLACEMENT]; + if (!templateFn) { + throw new Error("Invalid Aggregation placement: '" + options.placement + "'"); + } } return templateFn; }; +const isEmptyParameter = (parameter) => !parameter || Object.keys(parameter).length === 0; + +/** + * When no placement, columns or dimensions are specified in the aggregation + * a special default aggregation is used. + */ +const isDefaultAggregation = (options) => { + return !options || ( + !options.placement && isEmptyParameter(options.columns) && isEmptyParameter(options.dimensions) + ); +}; + /** * Generates an aggregation query given the aggregation options: * - query @@ -25,6 +44,11 @@ const templateForOptions = (options) => { * - columns * - placement * - dimensions + * + * The default aggregation (when no explicit placement, columns or dimensions are present) returns + * a sample record (with all the original columns and _cdb_feature_count) for each aggregation group. + * When placement, columns or dimensions are specified, columns are aggregated as requested + * (by default only _cdb_feature_count) and with the_geom_webmercator as defined by placement. */ const queryForOptions = (options) => templateForOptions(options)({ sourceQuery: options.query, @@ -35,6 +59,9 @@ const queryForOptions = (options) => templateForOptions(options)({ module.exports = queryForOptions; +// Checks if the aggregation parameters represent the default (full-sample) aggregation +module.exports.isDefaultAggregation = isDefaultAggregation; + const SUPPORTED_AGGREGATE_FUNCTIONS = { 'count': { sql: (column_name, params) => `count(${params.aggregated_column || '*'})` @@ -92,7 +119,13 @@ const aggregateColumnDefs = ctx => { const aggregateDimensions = ctx => ctx.dimensions || {}; -const dimensionNames = ctx => { +const dimensionNames = (ctx, table) => { + if (table) { + let dimensions = aggregateDimensions(ctx); + return sep(Object.keys(dimensions).map( + dimension_name => `${table}.${dimension_name}` + )); + } return sep(Object.keys(aggregateDimensions(ctx))); }; @@ -116,6 +149,34 @@ const gridResolution = ctx => `(${256*0.00028/ctx.res}*!scale_denominator!)::dou // is only applied after the aggregation. // * This queries are used for rendering and the_geom is omitted in the results for better performance +// The special default aggregation includes all the columns of a sample row per grid cell and +// the count (_cdb_feature_count) of the aggregated rows. +const defaultAggregationQueryTemplate = ctx => ` + WITH + _cdb_params AS ( + SELECT + ${gridResolution(ctx)} AS res, + !bbox! AS bbox + ), + _cdb_clusters AS ( + SELECT + MIN(cartodb_id) AS cartodb_id + ${dimensionDefs(ctx)} + ${aggregateColumnDefs(ctx)} + FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params + WHERE _cdb_query.the_geom_webmercator && _cdb_params.bbox + GROUP BY + Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res), + Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res) + ${dimensionNames(ctx)} + ) SELECT + _cdb_query.* + ${aggregateColumnNames(ctx)} + FROM + _cdb_clusters INNER JOIN (${ctx.sourceQuery}) _cdb_query + ON (_cdb_clusters.cartodb_id = _cdb_query.cartodb_id) +`; + const aggregationQueryTemplates = { 'centroid': ctx => ` WITH @@ -188,38 +249,13 @@ const aggregationQueryTemplates = { SELECT _cdb_clusters.cartodb_id, the_geom, the_geom_webmercator - ${dimensionNames(ctx)} - ${aggregateColumnNames(ctx)} - FROM - _cdb_clusters INNER JOIN (${ctx.sourceQuery}) _cdb_query - ON (_cdb_clusters.cartodb_id = _cdb_query.cartodb_id) - `, - - 'full-sample': ctx => ` - WITH - _cdb_params AS ( - SELECT - ${gridResolution(ctx)} AS res, - !bbox! AS bbox - ), - _cdb_clusters AS ( - SELECT - MIN(cartodb_id) AS cartodb_id - ${dimensionDefs(ctx)} - ${aggregateColumnDefs(ctx)} - FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params - WHERE _cdb_query.the_geom_webmercator && _cdb_params.bbox - GROUP BY - Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res), - Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res) - ${dimensionNames(ctx)} - ) SELECT - _cdb_query.* + ${dimensionNames(ctx, '_cdb_query')} ${aggregateColumnNames(ctx)} FROM _cdb_clusters INNER JOIN (${ctx.sourceQuery}) _cdb_query ON (_cdb_clusters.cartodb_id = _cdb_query.cartodb_id) ` + }; module.exports.SUPPORTED_PLACEMENTS = Object.keys(aggregationQueryTemplates); diff --git a/test/acceptance/aggregation.js b/test/acceptance/aggregation.js index 14040592..2d31fe98 100644 --- a/test/acceptance/aggregation.js +++ b/test/acceptance/aggregation.js @@ -289,7 +289,8 @@ describe('aggregation', function () { options: { sql: POINTS_SQL_2, aggregation: { - threshold: 1 + threshold: 1, + placement: 'centroid' }, cartocss: '#layer { marker-width: [value]; }', cartocss_version: '2.3.0' @@ -309,6 +310,45 @@ describe('aggregation', function () { }); }); + it('should provide all columns in the default aggregation ', + function (done) { + const response = { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_2, + aggregation: { + threshold: 1 + }, + cartocss: '#layer { marker-width: [value]; }', + cartocss_version: '2.3.0' + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + this.testClient.getLayergroup({ response }, (err, body) => { + if (err) { + return done(err); + } + + + assert.equal(typeof body.metadata, 'object'); + assert.ok(Array.isArray(body.metadata.layers)); + + body.metadata.layers.forEach(layer => assert.ok(layer.meta.aggregation.mvt)); + body.metadata.layers.forEach(layer => assert.ok(layer.meta.aggregation.png)); + done(); + }); + }); + it('should skip aggregation to create a layergroup with aggregation defined already', function (done) { const mapConfig = createVectorMapConfig([ { @@ -569,6 +609,140 @@ describe('aggregation', function () { }); }); + ['centroid', 'point-sample', 'point-grid'].forEach(placement => { + it(`dimensions should work for ${placement} placement`, function(done) { + + // FIXME: skip until pg-mvt renderer is able to return all columns + if (process.env.POSTGIS_VERSION === '2.4') { + return done(); + } + + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + placement: placement , + threshold: 1, + dimensions: { + value: "value" + } + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + + const tileJSON = tile.toJSON(); + + tileJSON[0].features.forEach( + feature => assert.equal(typeof feature.properties.value, 'number') + ); + + done(); + }); + }); + }); + + it(`dimensions should trigger non-default aggregation`, function(done) { + + // FIXME: skip until pg-mvt renderer is able to return all columns + if (process.env.POSTGIS_VERSION === '2.4') { + return done(); + } + + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_2, + aggregation: { + threshold: 1, + dimensions: { + value: "value" + } + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + + const tileJSON = tile.toJSON(); + + tileJSON[0].features.forEach( + feature => assert.equal(typeof feature.properties.value, 'number') + ); + tileJSON[0].features.forEach( + feature => assert.equal(typeof feature.properties.sqrt_value, 'undefined') + ); + + done(); + }); + }); + + it(`aggregations should trigger non-default aggregation`, function(done) { + + // FIXME: skip until pg-mvt renderer is able to return all columns + if (process.env.POSTGIS_VERSION === '2.4') { + return done(); + } + + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_2, + aggregation: { + threshold: 1, + columns: { + value: { + aggregate_function: 'sum', + aggregated_column: 'value' + } + } + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + + const tileJSON = tile.toJSON(); + + tileJSON[0].features.forEach( + feature => assert.equal(typeof feature.properties.value, 'number') + ); + tileJSON[0].features.forEach( + feature => assert.equal(typeof feature.properties.sqrt_value, 'undefined') + ); + + done(); + }); + }); it('should work when the sql has single quotes', function (done) { this.mapConfig = createVectorMapConfig([ { @@ -576,6 +750,7 @@ describe('aggregation', function () { options: { sql: ` SELECT + cartodb_id, the_geom_webmercator, the_geom, value, @@ -732,7 +907,7 @@ describe('aggregation', function () { }); }); - it('aggregates with full-sample placement', function (done) { + it('aggregates with full-sample placement by default', function (done) { this.mapConfig = createVectorMapConfig([ { type: 'cartodb', @@ -740,17 +915,6 @@ describe('aggregation', function () { sql: POINTS_SQL_1, resolution: 256, aggregation: { - placement: 'point-grid', - columns: { - total: { - aggregate_function: 'sum', - aggregated_column: 'value' - }, - v_avg: { - aggregate_function: 'avg', - aggregated_column: 'value' - } - }, threshold: 1 } } @@ -847,10 +1011,10 @@ describe('aggregation', function () { } assert.deepEqual(body, { - errors: [ 'Invalid placement. Valid values: centroid, point-grid, point-sample, full-sample'], + errors: [ 'Invalid placement. Valid values: centroid, point-grid, point-sample'], errors_with_context:[{ type: 'layer', - message: 'Invalid placement. Valid values: centroid, point-grid, point-sample, full-sample', + message: 'Invalid placement. Valid values: centroid, point-grid, point-sample', layer: { id: "layer0", index: 0, From efdfabf3e98d9b6cc4d3b17ecf68a2dab5efc358 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 22 Dec 2017 12:14:34 +0100 Subject: [PATCH 09/27] Remove method --- .../models/aggregation/aggregation-mapconfig.js | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/lib/cartodb/models/aggregation/aggregation-mapconfig.js b/lib/cartodb/models/aggregation/aggregation-mapconfig.js index 6e9b71d0..bf334338 100644 --- a/lib/cartodb/models/aggregation/aggregation-mapconfig.js +++ b/lib/cartodb/models/aggregation/aggregation-mapconfig.js @@ -72,18 +72,6 @@ module.exports = class AggregationMapConfig extends MapConfig { }); } - getAggregatedColumns (index) { - const { aggregation } = this.getLayer(index).options; - - if (aggregation === undefined || aggregation.columns === undefined) { - return []; - } - - return aggregation.columns - .map(({ aggregated_column }) => aggregated_column) - .filter(aggregated_column => typeof aggregated_column === 'string'); - } - isAggregationMapConfig () { return this.isVectorOnlyMapConfig() || this.hasAnyLayerAggregation(); } From a2f804d79f130bb4b3a276fcc56b7ef8c0882ea0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 22 Dec 2017 12:15:37 +0100 Subject: [PATCH 10/27] Use full-sample aggregation mode --- test/acceptance/aggregation.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/aggregation.js b/test/acceptance/aggregation.js index 14040592..abb9a376 100644 --- a/test/acceptance/aggregation.js +++ b/test/acceptance/aggregation.js @@ -740,7 +740,7 @@ describe('aggregation', function () { sql: POINTS_SQL_1, resolution: 256, aggregation: { - placement: 'point-grid', + placement: 'full-sample', columns: { total: { aggregate_function: 'sum', From b7ecde5c9d183a11c70bf057f1e2fd5c362545ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 22 Dec 2017 13:43:30 +0100 Subject: [PATCH 11/27] Add function get columns for layer's query --- .../aggregation/aggregation-mapconfig.js | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/models/aggregation/aggregation-mapconfig.js b/lib/cartodb/models/aggregation/aggregation-mapconfig.js index bf334338..7966693e 100644 --- a/lib/cartodb/models/aggregation/aggregation-mapconfig.js +++ b/lib/cartodb/models/aggregation/aggregation-mapconfig.js @@ -38,7 +38,7 @@ module.exports = class AggregationMapConfig extends MapConfig { return AggregationMapConfig.SUPPORTED_GEOMETRY_TYPES.includes(geometryType); } - constructor (config, datasource) { + constructor (config, connection, datasource) { super(config, datasource); const validate = aggregationValidator(this); @@ -50,6 +50,8 @@ module.exports = class AggregationMapConfig extends MapConfig { validate('placement', includesValidPlacementsValidator); validate('threshold', positiveNumberValidator); validate('columns', aggregationColumnsValidator); + + this.connection = connection; } getAggregatedQuery (index) { @@ -120,4 +122,26 @@ module.exports = class AggregationMapConfig extends MapConfig { return featureCount >= threshold; } + + getLayerColumns (index, skipGeoms, callback) { + const geomColumns = ['the_geom', 'the_geom_webmercator']; + const limitedQuery = ctx => `SELECT * FROM (${ctx.query}) __cdb_schema LIMIT 0`; + const layer = this.getLayer(index); + + this.connection.query(limitedQuery({ query: layer.options.sql }), (err, result) => { + if (err) { + return callback(err); + } + + let columns = result.fields || []; + + columns = columns.map(({ name, type }) => ({ name, type })); + + if (skipGeoms) { + columns = columns.filter((column) => !geomColumns.includes(column)); + } + + return callback(err, columns); + }); + } }; From 19bf079f2dcd2cfbf9443335eb760d2ddb6794b1 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Fri, 22 Dec 2017 15:45:14 +0100 Subject: [PATCH 12/27] Exclude test from PostGIS 2.4 --- test/acceptance/aggregation.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/acceptance/aggregation.js b/test/acceptance/aggregation.js index 2d31fe98..1809c8a0 100644 --- a/test/acceptance/aggregation.js +++ b/test/acceptance/aggregation.js @@ -908,6 +908,12 @@ describe('aggregation', function () { }); it('aggregates with full-sample placement by default', function (done) { + + // FIXME: skip until pg-mvt renderer is able to return all columns + if (process.env.POSTGIS_VERSION === '2.4') { + return done(); + } + this.mapConfig = createVectorMapConfig([ { type: 'cartodb', From 54f32113f348d7fa0d0deb84b27f7d21928ba968 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Fri, 22 Dec 2017 15:45:34 +0100 Subject: [PATCH 13/27] Add some aggregation tests --- test/acceptance/aggregation.js | 50 +++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/test/acceptance/aggregation.js b/test/acceptance/aggregation.js index 1809c8a0..afc27282 100644 --- a/test/acceptance/aggregation.js +++ b/test/acceptance/aggregation.js @@ -697,7 +697,7 @@ describe('aggregation', function () { }); }); - it(`aggregations should trigger non-default aggregation`, function(done) { + it(`aggregation columns should trigger non-default aggregation`, function(done) { // FIXME: skip until pg-mvt renderer is able to return all columns if (process.env.POSTGIS_VERSION === '2.4') { @@ -743,6 +743,54 @@ describe('aggregation', function () { done(); }); }); + + ['centroid', 'point-sample', 'point-grid'].forEach(placement => { + it(`aggregations with base column names should work for ${placement} placement`, function(done) { + + // FIXME: skip until pg-mvt renderer is able to return all columns + if (process.env.POSTGIS_VERSION === '2.4') { + return done(); + } + + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + placement: placement , + threshold: 1, + columns: { + value: { + aggregate_function: 'sum', + aggregated_column: 'value' + } + } + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + + const tileJSON = tile.toJSON(); + + tileJSON[0].features.forEach( + feature => assert.equal(typeof feature.properties.value, 'number') + ); + + done(); + }); + }); + }); + it('should work when the sql has single quotes', function (done) { this.mapConfig = createVectorMapConfig([ { From 1ce80766998b4e1d4f7c3620962f9d88103f5fb0 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Fri, 22 Dec 2017 15:46:29 +0100 Subject: [PATCH 14/27] Change default aggregation placement to point-sample For consistency with the default aggregation. --- lib/cartodb/models/aggregation/aggregation-query.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/aggregation/aggregation-query.js b/lib/cartodb/models/aggregation/aggregation-query.js index ab56e9b5..5ecf09aa 100644 --- a/lib/cartodb/models/aggregation/aggregation-query.js +++ b/lib/cartodb/models/aggregation/aggregation-query.js @@ -1,4 +1,4 @@ -const DEFAULT_PLACEMENT = 'centroid'; +const DEFAULT_PLACEMENT = 'point-sample'; /** * Returns a template function (function that accepts template parameters and returns a string) From d726c9ad013d072b03c6eea0d42ed68a7c0deaff Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Fri, 22 Dec 2017 15:48:30 +0100 Subject: [PATCH 15/27] Fix point-sample aggregation it failed in the case of aggregate columns with the name of base columns --- lib/cartodb/models/aggregation/aggregation-query.js | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/cartodb/models/aggregation/aggregation-query.js b/lib/cartodb/models/aggregation/aggregation-query.js index 5ecf09aa..965d201f 100644 --- a/lib/cartodb/models/aggregation/aggregation-query.js +++ b/lib/cartodb/models/aggregation/aggregation-query.js @@ -98,8 +98,13 @@ const aggregateColumns = ctx => { }, ctx.columns || {}); }; -const aggregateColumnNames = ctx => { +const aggregateColumnNames = (ctx, table) => { let columns = aggregateColumns(ctx); + if (table) { + return sep(Object.keys(columns).map( + column_name => `${table}.${column_name}` + )); + } return sep(Object.keys(columns)); }; @@ -120,13 +125,13 @@ const aggregateColumnDefs = ctx => { const aggregateDimensions = ctx => ctx.dimensions || {}; const dimensionNames = (ctx, table) => { + let dimensions = aggregateDimensions(ctx); if (table) { - let dimensions = aggregateDimensions(ctx); return sep(Object.keys(dimensions).map( dimension_name => `${table}.${dimension_name}` )); } - return sep(Object.keys(aggregateDimensions(ctx))); + return sep(Object.keys(dimensions)); }; const dimensionDefs = ctx => { @@ -250,7 +255,7 @@ const aggregationQueryTemplates = { _cdb_clusters.cartodb_id, the_geom, the_geom_webmercator ${dimensionNames(ctx, '_cdb_query')} - ${aggregateColumnNames(ctx)} + ${aggregateColumnNames(ctx, '_cdb_clusters')} FROM _cdb_clusters INNER JOIN (${ctx.sourceQuery}) _cdb_query ON (_cdb_clusters.cartodb_id = _cdb_query.cartodb_id) From a43779b05017120e25369c72b6e8eebf1473a622 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 22 Dec 2017 18:19:57 +0100 Subject: [PATCH 16/27] Get columns from layer query a set them into layer opptions --- .../aggregation/aggregation-mapconfig.js | 40 ++++++++++++++----- .../adapter/aggregation-mapconfig-adapter.js | 15 +++++-- package.json | 2 +- yarn.lock | 22 +++++----- 4 files changed, 55 insertions(+), 24 deletions(-) diff --git a/lib/cartodb/models/aggregation/aggregation-mapconfig.js b/lib/cartodb/models/aggregation/aggregation-mapconfig.js index 7966693e..95ae1641 100644 --- a/lib/cartodb/models/aggregation/aggregation-mapconfig.js +++ b/lib/cartodb/models/aggregation/aggregation-mapconfig.js @@ -38,7 +38,7 @@ module.exports = class AggregationMapConfig extends MapConfig { return AggregationMapConfig.SUPPORTED_GEOMETRY_TYPES.includes(geometryType); } - constructor (config, connection, datasource) { + constructor (user, config, connection, datasource) { super(config, datasource); const validate = aggregationValidator(this); @@ -51,7 +51,8 @@ module.exports = class AggregationMapConfig extends MapConfig { validate('threshold', positiveNumberValidator); validate('columns', aggregationColumnsValidator); - this.connection = connection; + this.user = user; + this.pgConnection = connection; } getAggregatedQuery (index) { @@ -128,20 +129,41 @@ module.exports = class AggregationMapConfig extends MapConfig { const limitedQuery = ctx => `SELECT * FROM (${ctx.query}) __cdb_schema LIMIT 0`; const layer = this.getLayer(index); - this.connection.query(limitedQuery({ query: layer.options.sql }), (err, result) => { + this.pgConnection.getConnection(this.user, (err, connection) => { if (err) { return callback(err); } - let columns = result.fields || []; + const sql = limitedQuery({ query: layer.options.sql }); - columns = columns.map(({ name, type }) => ({ name, type })); + connection.query(sql, (err, result) => { + if (err) { + return callback(err); + } - if (skipGeoms) { - columns = columns.filter((column) => !geomColumns.includes(column)); - } + let columns = result.fields || []; - return callback(err, columns); + columns = columns.map(({ name, type }) => ({ name, type })); + + if (skipGeoms) { + columns = columns.filter((column) => !geomColumns.includes(column)); + } + + return callback(err, columns); + }); }); } + + isDefaultAggregation (index) { + const aggregation = this.getAggregation(index); + return !aggregation || ( + !aggregation.placement && + this._isEmptyParameter(aggregation.columns) && + this._isEmptyParameter(aggregation.dimensions) + ); + } + + _isEmptyParameter (parameter) { + return !parameter || Object.keys(parameter).length === 0; + } }; diff --git a/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js b/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js index 99f68a0f..1540faf7 100644 --- a/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js @@ -20,7 +20,7 @@ module.exports = class AggregationMapConfigAdapter { let mapConfig; try { - mapConfig = new AggregationMapConfig(requestMapConfig); + mapConfig = new AggregationMapConfig(user, requestMapConfig, this.pgConnection); } catch (err) { return callback(err); } @@ -100,9 +100,18 @@ module.exports = class AggregationMapConfigAdapter { layer.options.sql = aggregationSql; - const aggregatedColumns = mapConfig.getAggregatedColumns(index); + if (mapConfig.isDefaultAggregation(index)) { + const skipGeoms = true; + mapConfig.getLayerColumns(index, skipGeoms, (err, columns) => { + if (err) { + return reject(err); + } - layer.options.columns = aggregatedColumns; + layer.options.columns = columns; + + return resolve({ layer, index, adapted: shouldAdapt }); + }); + } } return resolve({ layer, index, adapted: shouldAdapt }); diff --git a/package.json b/package.json index dd210fc0..669ec59c 100644 --- a/package.json +++ b/package.json @@ -46,7 +46,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.20.2", "underscore": "~1.6.0", - "windshaft": "4.1.0", + "windshaft": "cartodb/windshaft#pg-mvt-do-not-filter-columns", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index 78e663f0..a8d4808b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -244,14 +244,6 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" -carto@CartoDB/carto#0.15.1-cdb1: - version "0.15.1-cdb1" - resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" - dependencies: - mapnik-reference "~6.0.2" - optimist "~0.6.0" - underscore "~1.6.0" - carto@cartodb/carto#0.15.1-cdb3: version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" @@ -260,6 +252,14 @@ carto@cartodb/carto#0.15.1-cdb3: optimist "~0.6.0" underscore "1.8.3" +"carto@github:cartodb/carto#0.15.1-cdb1": + version "0.15.1-cdb1" + resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + dependencies: + mapnik-reference "~6.0.2" + optimist "~0.6.0" + underscore "~1.6.0" + cartocolor@4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/cartocolor/-/cartocolor-4.0.0.tgz#841a3222d8b5b22718d9d545b1e5b972cb26eb36" @@ -2392,9 +2392,9 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@4.1.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.1.0.tgz#dc17c8369570c305171d1ab5ca130369bba04d58" +windshaft@cartodb/windshaft#pg-mvt-do-not-filter-columns: + version "4.1.1" + resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/2de4e21b41787acd1630193fdd42c308815bfee5" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 From 9885045b41b7c7e869f07447ea6650d1f0f982be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 27 Dec 2017 12:48:06 +0100 Subject: [PATCH 17/27] Do not default to null --- lib/cartodb/models/aggregation/aggregation-mapconfig.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/aggregation/aggregation-mapconfig.js b/lib/cartodb/models/aggregation/aggregation-mapconfig.js index 94f7257f..8852a821 100644 --- a/lib/cartodb/models/aggregation/aggregation-mapconfig.js +++ b/lib/cartodb/models/aggregation/aggregation-mapconfig.js @@ -58,7 +58,7 @@ module.exports = class AggregationMapConfig extends MapConfig { // this enables the special "full-sample" aggregation. resolution = AggregationMapConfig.RESOLUTION, threshold = AggregationMapConfig.THRESHOLD, - placement = null, + placement, columns = {}, dimensions = {} } = this.getAggregation(index); From 418c8691d1b80e568a1c257ae459bdfc7ffc1d8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 27 Dec 2017 20:08:43 +0100 Subject: [PATCH 18/27] Support default full-sample aggregation for postgis vector renderer --- .../aggregation/aggregation-mapconfig.js | 42 ++++++++++++++----- .../adapter/aggregation-mapconfig-adapter.js | 42 ++++++++++--------- test/acceptance/aggregation.js | 6 --- yarn.lock | 18 ++++---- 4 files changed, 63 insertions(+), 45 deletions(-) diff --git a/lib/cartodb/models/aggregation/aggregation-mapconfig.js b/lib/cartodb/models/aggregation/aggregation-mapconfig.js index 8852a821..a463d136 100644 --- a/lib/cartodb/models/aggregation/aggregation-mapconfig.js +++ b/lib/cartodb/models/aggregation/aggregation-mapconfig.js @@ -7,6 +7,17 @@ const { createAggregationColumnsValidator } = aggregationValidator; +const SubstitutionTokens = require('../../utils/substitution-tokens'); + +function prepareSql(sql) { + return sql && SubstitutionTokens.replace(sql, { + bbox: 'ST_MakeEnvelope(0,0,0,0)', + scale_denominator: '0', + pixel_width: '1', + pixel_height: '1' + }); +} + module.exports = class AggregationMapConfig extends MapConfig { static get AGGREGATIONS () { return aggregationQuery.SUPPORTED_AGGREGATE_FUNCTIONS; @@ -132,7 +143,9 @@ module.exports = class AggregationMapConfig extends MapConfig { return callback(err); } - const sql = limitedQuery({ query: layer.options.sql }); + const sql = limitedQuery({ + query: prepareSql(layer.options.sql) + }); connection.query(sql, (err, result) => { if (err) { @@ -141,7 +154,7 @@ module.exports = class AggregationMapConfig extends MapConfig { let columns = result.fields || []; - columns = columns.map(({ name, type }) => ({ name, type })); + columns = columns.map(({ name }) => name); if (skipGeoms) { columns = columns.filter((column) => !geomColumns.includes(column)); @@ -152,16 +165,25 @@ module.exports = class AggregationMapConfig extends MapConfig { }); } - isDefaultAggregation (index) { + isDefaultLayerAggregation (index) { const aggregation = this.getAggregation(index); - return !aggregation || ( - !aggregation.placement && - this._isEmptyParameter(aggregation.columns) && - this._isEmptyParameter(aggregation.dimensions) - ); + + return (this.isVectorOnlyMapConfig() && !this.hasLayerAggregation(index)) || + aggregation === true || + this._isEmptyAggregation(aggregation); } - _isEmptyParameter (parameter) { - return !parameter || Object.keys(parameter).length === 0; + _isEmptyAggregation (aggregation) { + return aggregation.placement === undefined && + aggregation.columns === undefined && + this._isEmptyParameter(aggregation.dimensions); + } + + _isEmptyParameter(parameter) { + return parameter === undefined || parameter === null || this._isEmptyObject(parameter); + } + + _isEmptyObject (parameter) { + return typeof parameter === 'object' && Object.keys(parameter).length === 0; } }; diff --git a/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js b/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js index 1540faf7..79bd6527 100644 --- a/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js @@ -89,32 +89,34 @@ module.exports = class AggregationMapConfigAdapter { return reject(err); } - if (shouldAdapt) { - const sqlQueryWrap = layer.options.sql_wrap; + if (!shouldAdapt) { + return resolve({ layer, index, adapted: shouldAdapt }); + } - let aggregationSql = mapConfig.getAggregatedQuery(index); + const sqlQueryWrap = layer.options.sql_wrap; - if (sqlQueryWrap) { - aggregationSql = sqlQueryWrap.replace(/<%=\s*sql\s*%>/g, aggregationSql); + let aggregationSql = mapConfig.getAggregatedQuery(index); + + if (sqlQueryWrap) { + aggregationSql = sqlQueryWrap.replace(/<%=\s*sql\s*%>/g, aggregationSql); + } + + if (!mapConfig.isDefaultLayerAggregation(index)) { + layer.options.sql = aggregationSql; + return resolve({ layer, index, adapted: shouldAdapt }); + } + + const skipGeoms = true; + mapConfig.getLayerColumns(index, skipGeoms, (err, columns) => { + if (err) { + return reject(err); } layer.options.sql = aggregationSql; + layer.options.columns = columns; - if (mapConfig.isDefaultAggregation(index)) { - const skipGeoms = true; - mapConfig.getLayerColumns(index, skipGeoms, (err, columns) => { - if (err) { - return reject(err); - } - - layer.options.columns = columns; - - return resolve({ layer, index, adapted: shouldAdapt }); - }); - } - } - - return resolve({ layer, index, adapted: shouldAdapt }); + return resolve({ layer, index, adapted: shouldAdapt }); + }); }); }); } diff --git a/test/acceptance/aggregation.js b/test/acceptance/aggregation.js index afc27282..7879817a 100644 --- a/test/acceptance/aggregation.js +++ b/test/acceptance/aggregation.js @@ -956,12 +956,6 @@ describe('aggregation', function () { }); it('aggregates with full-sample placement by default', function (done) { - - // FIXME: skip until pg-mvt renderer is able to return all columns - if (process.env.POSTGIS_VERSION === '2.4') { - return done(); - } - this.mapConfig = createVectorMapConfig([ { type: 'cartodb', diff --git a/yarn.lock b/yarn.lock index 4ad1dabc..300bf086 100644 --- a/yarn.lock +++ b/yarn.lock @@ -244,14 +244,6 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" -carto@CartoDB/carto#0.15.1-cdb1: - version "0.15.1-cdb1" - resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" - dependencies: - mapnik-reference "~6.0.2" - optimist "~0.6.0" - underscore "~1.6.0" - carto@cartodb/carto#0.15.1-cdb3: version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" @@ -260,6 +252,14 @@ carto@cartodb/carto#0.15.1-cdb3: optimist "~0.6.0" underscore "1.8.3" +"carto@github:cartodb/carto#0.15.1-cdb1": + version "0.15.1-cdb1" + resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + dependencies: + mapnik-reference "~6.0.2" + optimist "~0.6.0" + underscore "~1.6.0" + cartocolor@4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/cartocolor/-/cartocolor-4.0.0.tgz#841a3222d8b5b22718d9d545b1e5b972cb26eb36" @@ -2394,7 +2394,7 @@ window-size@^0.2.0: windshaft@cartodb/windshaft#pg-mvt-do-not-filter-columns: version "4.1.1" - resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/2de4e21b41787acd1630193fdd42c308815bfee5" + resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/04df77330ea983193345ef65601ee131bc1ce796" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 From 5a730c6df18db9479ad1769e0f10c71cb84a5f6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 28 Dec 2017 13:10:11 +0100 Subject: [PATCH 19/27] Remove exited containers after running test against docker --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 669ec59c..fb2fc37c 100644 --- a/package.json +++ b/package.json @@ -65,7 +65,7 @@ "update-internal-deps": "rm -rf node_modules && rm -f yarn.lock && yarn", "docker-install": "sudo apt install docker.io && sudo usermod -aG docker $(whoami)", "docker-pull": "docker pull cartoimages/windshaft-testing", - "docker-test": "docker run -v `pwd`:/srv cartoimages/windshaft-testing bash docker-test.sh", + "docker-test": "docker run -v `pwd`:/srv cartoimages/windshaft-testing bash docker-test.sh && docker ps --filter status=dead --filter status=exited -aq | xargs -r docker rm -v", "docker-bash": "docker run -it -v `pwd`:/srv cartoimages/windshaft-testing bash", "docker-publish": "docker push cartoimages/windshaft-carto-testing" }, From d42257127b8e664d4f27e1dc2482f549f907b11c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 28 Dec 2017 13:11:32 +0100 Subject: [PATCH 20/27] Add method to discover required columns or all of them if it's a default aggregation --- .../aggregation/aggregation-mapconfig.js | 41 ++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/models/aggregation/aggregation-mapconfig.js b/lib/cartodb/models/aggregation/aggregation-mapconfig.js index a463d136..ef12518e 100644 --- a/lib/cartodb/models/aggregation/aggregation-mapconfig.js +++ b/lib/cartodb/models/aggregation/aggregation-mapconfig.js @@ -9,6 +9,8 @@ const { const SubstitutionTokens = require('../../utils/substitution-tokens'); +const removeDuplicates = arr => [...new Set(arr)]; + function prepareSql(sql) { return sql && SubstitutionTokens.replace(sql, { bbox: 'ST_MakeEnvelope(0,0,0,0)', @@ -125,6 +127,43 @@ module.exports = class AggregationMapConfig extends MapConfig { return aggregation; } + getLayerAggregationColumns (index, callback) { + if (this._isDefaultLayerAggregation(index)) { + const skipGeoms = true; + return this.getLayerColumns(index, skipGeoms, (err, columns) => { + if (err) { + return callback(err); + } + + return callback(null, columns); + }); + } + + const columns = this._getLayerAggregationRequiredColumns(index); + + return callback(null, columns); + } + + _getLayerAggregationRequiredColumns (index) { + const { columns, dimensions } = this.getAggregation(index); + + let aggregatedColumns = []; + if (columns) { + aggregatedColumns = Object.keys(columns) + .map(key => columns[key].aggregated_column) + .filter(aggregatedColumn => typeof aggregatedColumn === 'string'); + } + + let dimensionsColumns = []; + if (dimensions) { + dimensionsColumns = Object.keys(dimensions) + .map(key => dimensions[key]) + .filter(dimension => typeof dimension === 'string'); + } + + return removeDuplicates(aggregatedColumns.concat(dimensionsColumns)); + } + doesLayerReachThreshold(index, featureCount) { const threshold = this.getAggregation(index) && this.getAggregation(index).threshold ? this.getAggregation(index).threshold : @@ -165,7 +204,7 @@ module.exports = class AggregationMapConfig extends MapConfig { }); } - isDefaultLayerAggregation (index) { + _isDefaultLayerAggregation (index) { const aggregation = this.getAggregation(index); return (this.isVectorOnlyMapConfig() && !this.hasLayerAggregation(index)) || From 9a1bc51fdbff79ee5d38f4198ebd3ea1ab5db123 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 28 Dec 2017 13:12:41 +0100 Subject: [PATCH 21/27] Use aggregation-mapconfig's method to discover columns to be exposed used to aggregate --- .../mapconfig/adapter/aggregation-mapconfig-adapter.js | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js b/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js index 79bd6527..28cbffc9 100644 --- a/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js @@ -101,18 +101,13 @@ module.exports = class AggregationMapConfigAdapter { aggregationSql = sqlQueryWrap.replace(/<%=\s*sql\s*%>/g, aggregationSql); } - if (!mapConfig.isDefaultLayerAggregation(index)) { - layer.options.sql = aggregationSql; - return resolve({ layer, index, adapted: shouldAdapt }); - } + layer.options.sql = aggregationSql; - const skipGeoms = true; - mapConfig.getLayerColumns(index, skipGeoms, (err, columns) => { + mapConfig.getLayerAggregationColumns(index, (err, columns) => { if (err) { return reject(err); } - layer.options.sql = aggregationSql; layer.options.columns = columns; return resolve({ layer, index, adapted: shouldAdapt }); From 9fbc56b82cae5ee36955785038318e545a486d75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 28 Dec 2017 13:13:32 +0100 Subject: [PATCH 22/27] Remove FIXME notes to test against PostGIS vector renderer --- test/acceptance/aggregation.js | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/test/acceptance/aggregation.js b/test/acceptance/aggregation.js index 7879817a..30cfd9db 100644 --- a/test/acceptance/aggregation.js +++ b/test/acceptance/aggregation.js @@ -571,12 +571,6 @@ describe('aggregation', function () { it('when dimensions is provided should return a tile returning the column used as dimensions', function (done) { - - // FIXME: skip until pg-mvt renderer is able to return all columns - if (process.env.POSTGIS_VERSION === '2.4') { - return done(); - } - this.mapConfig = createVectorMapConfig([ { type: 'cartodb', @@ -611,12 +605,6 @@ describe('aggregation', function () { ['centroid', 'point-sample', 'point-grid'].forEach(placement => { it(`dimensions should work for ${placement} placement`, function(done) { - - // FIXME: skip until pg-mvt renderer is able to return all columns - if (process.env.POSTGIS_VERSION === '2.4') { - return done(); - } - this.mapConfig = createVectorMapConfig([ { type: 'cartodb', @@ -654,12 +642,6 @@ describe('aggregation', function () { }); it(`dimensions should trigger non-default aggregation`, function(done) { - - // FIXME: skip until pg-mvt renderer is able to return all columns - if (process.env.POSTGIS_VERSION === '2.4') { - return done(); - } - this.mapConfig = createVectorMapConfig([ { type: 'cartodb', @@ -698,12 +680,6 @@ describe('aggregation', function () { }); it(`aggregation columns should trigger non-default aggregation`, function(done) { - - // FIXME: skip until pg-mvt renderer is able to return all columns - if (process.env.POSTGIS_VERSION === '2.4') { - return done(); - } - this.mapConfig = createVectorMapConfig([ { type: 'cartodb', @@ -746,12 +722,6 @@ describe('aggregation', function () { ['centroid', 'point-sample', 'point-grid'].forEach(placement => { it(`aggregations with base column names should work for ${placement} placement`, function(done) { - - // FIXME: skip until pg-mvt renderer is able to return all columns - if (process.env.POSTGIS_VERSION === '2.4') { - return done(); - } - this.mapConfig = createVectorMapConfig([ { type: 'cartodb', From 99b95cf839c603c88b675f92df6a2d72e7385744 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 28 Dec 2017 13:50:59 +0100 Subject: [PATCH 23/27] Move check of default-aggregation to mapconfig model --- .../models/aggregation/aggregation-mapconfig.js | 3 ++- .../models/aggregation/aggregation-query.js | 17 +---------------- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/lib/cartodb/models/aggregation/aggregation-mapconfig.js b/lib/cartodb/models/aggregation/aggregation-mapconfig.js index ef12518e..e08f3024 100644 --- a/lib/cartodb/models/aggregation/aggregation-mapconfig.js +++ b/lib/cartodb/models/aggregation/aggregation-mapconfig.js @@ -82,7 +82,8 @@ module.exports = class AggregationMapConfig extends MapConfig { threshold, placement, columns, - dimensions + dimensions, + isDefaultAggregation: this._isDefaultLayerAggregation(index) }); } diff --git a/lib/cartodb/models/aggregation/aggregation-query.js b/lib/cartodb/models/aggregation/aggregation-query.js index 965d201f..2a09efde 100644 --- a/lib/cartodb/models/aggregation/aggregation-query.js +++ b/lib/cartodb/models/aggregation/aggregation-query.js @@ -15,7 +15,7 @@ const DEFAULT_PLACEMENT = 'point-sample'; */ const templateForOptions = (options) => { let templateFn = defaultAggregationQueryTemplate; - if (!isDefaultAggregation(options)) { + if (!options.isDefaultAggregation) { templateFn = aggregationQueryTemplates[options.placement || DEFAULT_PLACEMENT]; if (!templateFn) { throw new Error("Invalid Aggregation placement: '" + options.placement + "'"); @@ -24,18 +24,6 @@ const templateForOptions = (options) => { return templateFn; }; -const isEmptyParameter = (parameter) => !parameter || Object.keys(parameter).length === 0; - -/** - * When no placement, columns or dimensions are specified in the aggregation - * a special default aggregation is used. - */ -const isDefaultAggregation = (options) => { - return !options || ( - !options.placement && isEmptyParameter(options.columns) && isEmptyParameter(options.dimensions) - ); -}; - /** * Generates an aggregation query given the aggregation options: * - query @@ -59,9 +47,6 @@ const queryForOptions = (options) => templateForOptions(options)({ module.exports = queryForOptions; -// Checks if the aggregation parameters represent the default (full-sample) aggregation -module.exports.isDefaultAggregation = isDefaultAggregation; - const SUPPORTED_AGGREGATE_FUNCTIONS = { 'count': { sql: (column_name, params) => `count(${params.aggregated_column || '*'})` From 75f72c4d07c5b502a8fb4d514732b1d010c2272c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 29 Dec 2017 17:52:28 +0100 Subject: [PATCH 24/27] Return empty aggregation configuration when the map-config is vector-only and the layer has no aggregation --- lib/cartodb/models/aggregation/aggregation-mapconfig.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/aggregation/aggregation-mapconfig.js b/lib/cartodb/models/aggregation/aggregation-mapconfig.js index e08f3024..bc93ceab 100644 --- a/lib/cartodb/models/aggregation/aggregation-mapconfig.js +++ b/lib/cartodb/models/aggregation/aggregation-mapconfig.js @@ -115,8 +115,8 @@ module.exports = class AggregationMapConfig extends MapConfig { } getAggregation (index) { - if (!this.hasLayerAggregation(index)) { - return; + if (this.isVectorOnlyMapConfig() && !this.hasLayerAggregation(index)) { + return {}; } const { aggregation } = this.getLayer(index).options; From 83d0ce4040c2762115c1f72182f6994bc232d60c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 29 Dec 2017 18:25:08 +0100 Subject: [PATCH 25/27] Rename method --- lib/cartodb/models/aggregation/aggregation-mapconfig.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/models/aggregation/aggregation-mapconfig.js b/lib/cartodb/models/aggregation/aggregation-mapconfig.js index bc93ceab..99ed32c1 100644 --- a/lib/cartodb/models/aggregation/aggregation-mapconfig.js +++ b/lib/cartodb/models/aggregation/aggregation-mapconfig.js @@ -210,10 +210,10 @@ module.exports = class AggregationMapConfig extends MapConfig { return (this.isVectorOnlyMapConfig() && !this.hasLayerAggregation(index)) || aggregation === true || - this._isEmptyAggregation(aggregation); + this._isDefaultAggregation(aggregation); } - _isEmptyAggregation (aggregation) { + _isDefaultAggregation (aggregation) { return aggregation.placement === undefined && aggregation.columns === undefined && this._isEmptyParameter(aggregation.dimensions); From ba002fdb2c6c3d55cdb25706777d705f5637bbe0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 2 Jan 2018 12:38:58 +0100 Subject: [PATCH 26/27] Update windshaft to 4.20 --- package.json | 2 +- yarn.lock | 40 ++++++++++++++++++++-------------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/package.json b/package.json index fb2fc37c..83e14970 100644 --- a/package.json +++ b/package.json @@ -46,7 +46,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.20.2", "underscore": "~1.6.0", - "windshaft": "cartodb/windshaft#pg-mvt-do-not-filter-columns", + "windshaft": "4.2.0", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index 518849d3..6e1c0707 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -abaculus@cartodb/abaculus#2.0.3-cdb1: +"abaculus@github:cartodb/abaculus#2.0.3-cdb1": version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -226,7 +226,7 @@ camshaft@0.60.0: dot "^1.0.3" request "^2.69.0" -canvas@cartodb/node-canvas#1.6.2-cdb2: +"canvas@github:cartodb/node-canvas#1.6.2-cdb2": version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -244,14 +244,6 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" -carto@cartodb/carto#0.15.1-cdb3: - version "0.15.1-cdb3" - resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" - dependencies: - mapnik-reference "~6.0.2" - optimist "~0.6.0" - underscore "1.8.3" - "carto@github:cartodb/carto#0.15.1-cdb1": version "0.15.1-cdb1" resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" @@ -260,6 +252,14 @@ carto@cartodb/carto#0.15.1-cdb3: optimist "~0.6.0" underscore "~1.6.0" +"carto@github:cartodb/carto#0.15.1-cdb3": + version "0.15.1-cdb3" + resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" + dependencies: + mapnik-reference "~6.0.2" + optimist "~0.6.0" + underscore "1.8.3" + cartocolor@4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/cartocolor/-/cartocolor-4.0.0.tgz#841a3222d8b5b22718d9d545b1e5b972cb26eb36" @@ -842,9 +842,9 @@ graceful-fs@^4.1.2: version "1.0.1" resolved "https://registry.yarnpkg.com/graceful-readlink/-/graceful-readlink-1.0.1.tgz#4cafad76bc62f02fa039b2f94e9a3dd3a391a725" -grainstore@1.7.0: - version "1.7.0" - resolved "https://registry.yarnpkg.com/grainstore/-/grainstore-1.7.0.tgz#28d78895c82e6201f7d0ff63af1056f3c0fda0d3" +grainstore@~1.8.0: + version "1.8.0" + resolved "https://registry.yarnpkg.com/grainstore/-/grainstore-1.8.0.tgz#9398729df88f3aecb55ffbb415d541dcca4420af" dependencies: carto "0.16.3" debug "~3.1.0" @@ -1633,7 +1633,7 @@ pg-types@1.*: postgres-date "~1.0.0" postgres-interval "^1.1.0" -"pg@github:cartodb/node-postgres#6.1.6-cdb1": +pg@cartodb/node-postgres#6.1.6-cdb1: version "6.1.6" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3eef52dd1e655f658a4ee8ac5697688b3ecfed44" dependencies: @@ -2223,7 +2223,7 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb4: +"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb4": version "2.3.1-cdb4" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/faa2b638da2d119b78281575d40255cb523f6ca6" dependencies: @@ -2231,7 +2231,7 @@ tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb4: mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb3: +"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb3": version "0.6.18-cdb3" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/23bd1c31dd57d0b76c86b9f1eaf62462b3c17d01" dependencies: @@ -2392,9 +2392,9 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@cartodb/windshaft#pg-mvt-do-not-filter-columns: - version "4.1.2" - resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/3a62a4e9f5fdbcb3e462adf047f97b1c8433971c" +windshaft@4.2.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.2.0.tgz#6a0409832a0d3bccfa09a88a8ab8288686b6762d" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 @@ -2402,7 +2402,7 @@ windshaft@cartodb/windshaft#pg-mvt-do-not-filter-columns: cartodb-psql "^0.10.1" debug "^3.1.0" dot "~1.0.2" - grainstore "1.7.0" + grainstore "~1.8.0" mapnik "3.5.14" queue-async "~1.0.7" redis-mpool "0.4.1" From 6bbe715aa63827a87cdf3c12020f5ee7e50820f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 2 Jan 2018 12:57:54 +0100 Subject: [PATCH 27/27] Update NEWS --- NEWS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.md b/NEWS.md index 24a86655..230774ac 100644 --- a/NEWS.md +++ b/NEWS.md @@ -7,6 +7,7 @@ Announcements: - Upgrades windshaft to [4.1.1](https://github.com/CartoDB/windshaft/releases/tag/4.1.1). - Validate aggregation input params - Fix column names collisions in histograms [#828](https://github.com/CartoDB/Windshaft-cartodb/pull/828) + - Add full-sample aggregation support for vector map-config. ## 4.5.0