diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 6b50fc47..185a4f8d 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -6,6 +6,7 @@ var ResourceLocator = require('../models/resource-locator'); var cors = require('../middleware/cors'); var userMiddleware = require('../middleware/user'); +const allowQueryParams = require('../middleware/allow-query-params'); var MapConfig = windshaft.model.MapConfig; var Datasource = windshaft.model.Datasource; @@ -69,6 +70,7 @@ MapController.prototype.composeCreateMapMiddleware = function (useTemplate = fal return [ cors(), userMiddleware, + allowQueryParams(['aggregation']), this.prepareContext, this.initProfiler(isTemplateInstantiation), this.checkJsonContentType(), @@ -87,6 +89,7 @@ MapController.prototype.composeCreateMapMiddleware = function (useTemplate = fal this.setDataviewsAndWidgetsUrlsToLayergroupMetadata(), this.setAnalysesMetadataToLayergroup(includeQuery), this.setTurboCartoMetadataToLayergroup(), + this.setAggregationMetadataToLayergroup(), this.setSurrogateKeyHeader(), this.sendResponse(), this.augmentError({ label, addContext }) @@ -540,13 +543,13 @@ MapController.prototype.setTurboCartoMetadataToLayergroup = function () { return function setTurboCartoMetadataToLayergroupMiddleware (req, res, next) { const { layergroup, mapconfig, context } = res.locals; - addContextMetadata(layergroup, mapconfig.obj(), context); + addTurboCartoContextMetadata(layergroup, mapconfig.obj(), context); next(); }; }; -function addContextMetadata(layergroup, mapConfig, context) { +function addTurboCartoContextMetadata(layergroup, mapConfig, context) { if (layergroup.metadata && Array.isArray(layergroup.metadata.layers) && Array.isArray(mapConfig.layers)) { layergroup.metadata.layers = layergroup.metadata.layers.map(function(layer, layerIndex) { if (context.turboCarto && Array.isArray(context.turboCarto.layers)) { @@ -557,6 +560,28 @@ function addContextMetadata(layergroup, mapConfig, context) { } } +// TODO: see how evolve this function, it's a good candidate to be refactored +MapController.prototype.setAggregationMetadataToLayergroup = function () { + return function setAggregationMetadataToLayergroupMiddleware (req, res, next) { + const { layergroup, mapconfig, context } = res.locals; + + addAggregationContextMetadata(layergroup, mapconfig.obj(), context); + + next(); + }; +}; + +function addAggregationContextMetadata(layergroup, mapConfig, context) { + if (layergroup.metadata && Array.isArray(layergroup.metadata.layers) && Array.isArray(mapConfig.layers)) { + layergroup.metadata.layers = layergroup.metadata.layers.map(function(layer, layerIndex) { + if (context.aggregation && Array.isArray(context.aggregation.layers)) { + layer.meta.aggregation = context.aggregation.layers[layerIndex]; + } + return layer; + }); + } +} + MapController.prototype.setSurrogateKeyHeader = function () { return function setSurrogateKeyHeaderMiddleware(req, res, next) { const { affectedTables, user, templateName } = res.locals; diff --git a/lib/cartodb/models/aggregation/aggregation-map-config.js b/lib/cartodb/models/aggregation/aggregation-map-config.js new file mode 100644 index 00000000..e6524e3d --- /dev/null +++ b/lib/cartodb/models/aggregation/aggregation-map-config.js @@ -0,0 +1,34 @@ +const MapConfig = require('windshaft').model.MapConfig; + +module.exports = class AggregationMapConfig extends MapConfig { + constructor (config, datasource) { + super(config, datasource); + } + + isAggregationMapConfig () { + return this.isVectorOnlyMapConfig() || this.hasAnyLayerAggregation(); + } + + isAggregationLayer (index) { + return this.isVectorOnlyMapConfig() || this.hasLayerAggregation(index); + } + + hasAnyLayerAggregation () { + const layers = this.getLayers(); + + for (let index = 0; index < layers.length; index++) { + if (this.hasLayerAggregation(index)) { + return true; + } + } + + return false; + } + + hasLayerAggregation (index) { + const layer = this.getLayer(index); + const { aggregation } = layer.options; + + return aggregation !== undefined && (typeof aggregation === 'object' || typeof aggregation === 'boolean'); + } +}; diff --git a/lib/cartodb/models/aggregation/aggregation-query.js b/lib/cartodb/models/aggregation/aggregation-query.js new file mode 100644 index 00000000..b42b1c57 --- /dev/null +++ b/lib/cartodb/models/aggregation/aggregation-query.js @@ -0,0 +1,190 @@ +/** + * Returns a template function (function that accepts template parameters and returns a string) + * to generate an aggregation query. + * Valid options to define the query template are: + * - placement + * The query template parameters taken by the result template function are: + * - sourceQuery + * - res + * - columns + * - dimensions + */ +const templateForOptions = (options) => { + let templateFn = aggregationQueryTemplates[options.placement]; + if (!templateFn) { + throw new Error("Invalid Aggregation placement: '" + options.placement + "'"); + } + return templateFn; +}; + +/** + * Generates an aggregation query given the aggregation options: + * - query + * - resolution - defined as in torque: + * aggregation cell is resolution*resolution pixels, where tiles are always 256x256 pixels + * - columns + * - placement + * - dimensions + */ +const queryForOptions = (options) => templateForOptions(options)({ + sourceQuery: options.query, + res: 256/options.resolution, + columns: options.columns, + dimensions: options.dimensions +}); + +module.exports = queryForOptions; + +const SUPPORTED_AGGREGATE_FUNCTIONS = { + 'count': { + sql: (column_name, params) => `count(${params.aggregated_column || '*'})` + }, + 'avg': { + sql: (column_name, params) => `avg(${params.aggregated_column || column_name})` + }, + 'sum': { + sql: (column_name, params) => `sum(${params.aggregated_column || column_name})` + }, + 'min': { + sql: (column_name, params) => `min(${params.aggregated_column || column_name})` + }, + 'max': { + sql: (column_name, params) => `max(${params.aggregated_column || column_name})` + }, + 'mode': { + sql: (column_name, params) => `_cdb_mode(${params.aggregated_column || column_name})` + } +}; + +const sep = (list) => { + let expr = list.join(', '); + return expr ? ', ' + expr : expr; +}; + +const aggregateColumns = ctx => { + return Object.assign({ + _cdb_feature_count: { + aggregate_function: 'count' + } + }, ctx.columns || {}); +}; + +const aggregateColumnNames = ctx => { + let columns = aggregateColumns(ctx); + return sep(Object.keys(columns)); +}; + +const aggregateColumnDefs = ctx => { + let columns = aggregateColumns(ctx); + return sep(Object.keys(columns).map(column_name => { + const aggregate_function = columns[column_name].aggregate_function || 'count'; + const aggregate_definition = SUPPORTED_AGGREGATE_FUNCTIONS[aggregate_function]; + if (!aggregate_definition) { + throw new Error("Invalid Aggregate function: '" + aggregate_function + "'"); + } + const aggregate_expression = aggregate_definition.sql(column_name, columns[column_name]); + return `${aggregate_expression} AS ${column_name}`; + })); +}; + + +const aggregateDimensions = ctx => ctx.dimensions || {}; + +const dimensionNames = ctx => { + return sep(Object.keys(aggregateDimensions(ctx))); +}; + +const dimensionDefs = ctx => { + let dimensions = aggregateDimensions(ctx); + return sep(Object.keys(dimensions).map(dimension_name => { + const expression = dimensions[dimension_name]; + return `${expression} AS ${dimension_name}`; + })); +}; + +// SQL expression to compute the aggregation resolution (grid cell size). +// This is equivalent to `${256/ctx.res}*CDB_XYZ_Resolution(CDB_ZoomFromScale(!scale_denominator!))` +// This is defined by the ctx.res parameter, which is the number of grid cells per tile linear dimension +// (i.e. each tile is divided into ctx.res*ctx.res cells). +const gridResolution = ctx => `(${256*0.00028/ctx.res}*!scale_denominator!)::double precision`; + +// Notes: +// * We need to filter spatially using !bbox! to make the queries efficient because +// the filter added by Mapnik (wrapping the query) +// is only applied after the aggregation. +// * This queries are used for rendering and the_geom is omitted in the results for better performance + +const aggregationQueryTemplates = { + 'centroid': ctx => ` + WITH _cdb_params AS ( + SELECT + ${gridResolution(ctx)} AS res, + !bbox! AS bbox + ) + SELECT + row_number() over() AS cartodb_id, + ST_SetSRID( + ST_MakePoint( + AVG(ST_X(_cdb_query.the_geom_webmercator)), + AVG(ST_Y(_cdb_query.the_geom_webmercator)) + ), 3857 + ) AS the_geom_webmercator + ${dimensionDefs(ctx)} + ${aggregateColumnDefs(ctx)} + FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params + WHERE _cdb_query.the_geom_webmercator && _cdb_params.bbox + GROUP BY + Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res), + Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res) + ${dimensionNames(ctx)} + `, + + 'point-grid': ctx => ` + WITH _cdb_params AS ( + SELECT + ${gridResolution(ctx)} AS res, + !bbox! AS bbox + ), + _cdb_clusters AS ( + SELECT + Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res)::int AS _cdb_gx, + Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)::int AS _cdb_gy + ${dimensionDefs(ctx)} + ${aggregateColumnDefs(ctx)} + FROM (${ctx.sourceQuery(ctx)}) _cdb_query, _cdb_params + WHERE the_geom_webmercator && _cdb_params.bbox + GROUP BY _cdb_gx, _cdb_gy ${dimensionNames} + ) + SELECT + ST_SetSRID(ST_MakePoint(_cdb_gx*(res+0.5), _cdb_gy*(res+0.5)), 3857) AS the_geom_webmercator + ${dimensionNames(ctx)} + ${aggregateColumnNames(ctx)} + FROM _cdb_clusters, _cdb_params + `, + + 'point-sample': ctx => ` + WITH _cdb_params AS ( + SELECT + ${gridResolution(ctx)} AS res, + !bbox! AS bbox + ), _cdb_clusters AS ( + SELECT + MIN(cartodb_id) AS cartodb_id + ${dimensionDefs(ctx)} + ${aggregateColumnDefs(ctx)} + FROM (${ctx.sourceQuery(ctx)}) _cdb_query, _cdb_params + WHERE _cdb_query.the_geom_webmercator && _cdb_params.bbox + GROUP BY + Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res), + Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res) + ${dimensionNames(ctx)} + ) SELECT + _cdb_clusters.cartodb_id, + the_geom, the_geom_webmercator + ${dimensionNames(ctx)} + ${aggregateColumnNames(ctx)} + FROM + _cdb_clusters INNER JOIN (${ctx.sourceQuery(ctx)}) _cdb_query + ON (_cdb_clusters.cartodb_id = _cdb_query.cartodb_id) + ` +}; diff --git a/lib/cartodb/models/aggregation/aggregation.js b/lib/cartodb/models/aggregation/aggregation.js new file mode 100644 index 00000000..2642898c --- /dev/null +++ b/lib/cartodb/models/aggregation/aggregation.js @@ -0,0 +1,26 @@ +const aggregationQuery = require('./aggregation-query'); + +module.exports = class Aggregation { + static get THRESHOLD() { + return 1e5; // 100K + } + + constructor (mapconfig, query, { + resolution = 1, + threshold = Aggregation.THRESHOLD, + placement = 'centroid', + columns = {}, + dimensions = {} + } = {}) { + this.mapconfig = mapconfig; + this.query = query; + this.resolution = resolution; + this.threshold = threshold; + this.placement = placement; + this.columns = columns; + this.dimensions = dimensions; + } + sql () { + return aggregationQuery(this); + } +}; diff --git a/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js b/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js new file mode 100644 index 00000000..cbbcd50f --- /dev/null +++ b/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js @@ -0,0 +1,155 @@ +const Aggregation = require('../../aggregation/aggregation'); +const AggregationMapConfig = require('../../aggregation/aggregation-map-config'); +const queryUtils = require('../../../utils/query-utils'); + +const unsupportedGeometryTypeErrorMessage = ctx => +`Unsupported geometry type: ${ctx.geometryType}. Aggregation is available only for geometry type: ST_Point`; + +const invalidAggregationParamValueErrorMessage = ctx => +`Invalid value for 'aggregation' query param: ${ctx.value}. Valid ones are 'true' or 'false'`; + +module.exports = class AggregationMapConfigAdapter { + constructor (pgConnection) { + this.pgConnection = pgConnection; + } + + getMapConfig (user, requestMapConfig, params, context, callback) { + if (!this._isValidAggregationParam(params)) { + return callback(new Error(invalidAggregationParamValueErrorMessage({ value: params.aggregation }))); + } + + const mapConfig = new AggregationMapConfig(requestMapConfig); + + if (!this._shouldAdapt(mapConfig, params)) { + return callback(null, requestMapConfig); + } + + this.pgConnection.getConnection(user, (err, connection) => { + if (err) { + return callback(err); + } + + this._adaptLayers(connection, mapConfig, requestMapConfig, context, callback); + }); + } + + _isValidAggregationParam (params) { + const { aggregation } = params; + return aggregation === undefined || aggregation === 'true' || aggregation === 'false'; + } + + _shouldAdapt (mapConfig, params) { + const { aggregation } = params; + + if (aggregation === 'false') { + return false; + } + + if (aggregation === 'true' || mapConfig.isAggregationMapConfig()) { + return true; + } + + return false; + } + + _adaptLayers (connection, mapConfig, requestMapConfig, context, callback) { + const adaptLayerPromises = requestMapConfig.layers.map((layer, index) => { + return this._adaptLayer(connection, mapConfig, layer, index); + }); + + Promise.all(adaptLayerPromises) + .then(results => { + context.aggregation = { + layers: [] + }; + + results.forEach(({ layer, index, adapted }) => { + if (adapted) { + requestMapConfig.layers[index] = layer; + } + const aggregatedFormats = this._getAggregationMetadata(mapConfig, layer, adapted); + context.aggregation.layers.push(aggregatedFormats); + }); + + callback(null, requestMapConfig); + }) + .catch(err => callback(err)); + } + + _adaptLayer (connection, mapConfig, layer, index) { + return new Promise((resolve, reject) => { + this._shouldAdaptLayer(connection, mapConfig, layer, index, (err, shouldAdapt) => { + if (err) { + return reject(err); + } + + if (shouldAdapt) { + const sql = layer.options.sql_raw ? layer.options.sql_raw : layer.options.sql; + const aggregation = new Aggregation(mapConfig, sql, layer.options.aggregation); + const sqlQueryWrap = layer.options.sql_wrap; + + let aggregationSql = aggregation.sql(); + + if (sqlQueryWrap) { + aggregationSql = sqlQueryWrap.replace(/<%=\s*sql\s*%>/g, aggregationSql); + } + + layer.options.sql = aggregationSql; + } + + return resolve({ layer, index, adapted: shouldAdapt }); + }); + }); + } + + _shouldAdaptLayer (connection, mapConfig, layer, index, callback) { + let shouldAdapt = false; + + if (!mapConfig.isAggregationLayer(index)) { + return callback(null, shouldAdapt); + } + + const aggregationMetadata = queryUtils.getAggregationMetadata({ + query: layer.options.sql_raw ? layer.options.sql_raw : layer.options.sql + }); + + connection.query(aggregationMetadata, (err, res) => { + if (err) { + return callback(null, shouldAdapt); + } + + const result = res.rows[0] || {}; + const estimatedFeatureCount = result.count; + + const threshold = layer.options.aggregation && layer.options.aggregation.threshold ? + layer.options.aggregation.threshold : + Aggregation.THRESHOLD; + + if (estimatedFeatureCount < threshold) { + return callback(null, shouldAdapt); + } + + const geometryType = result.type; + + if (geometryType !== 'ST_Point') { + return callback(new Error(unsupportedGeometryTypeErrorMessage({ geometryType }))); + } + + shouldAdapt = true; + + callback(null, shouldAdapt); + }); + } + + _getAggregationMetadata (mapConfig, layer, adapted) { + if (!adapted) { + return { png: false, mvt: false }; + } + + if (mapConfig.isVectorOnlyMapConfig()) { + return { png: false, mvt: true }; + } + + return { png: true, mvt: true }; + } +}; diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 08ec1a66..46ff7122 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -41,6 +41,7 @@ var AnalysisMapConfigAdapter = require('./models/mapconfig/adapter/analysis-mapc var MapConfigOverviewsAdapter = require('./models/mapconfig/adapter/mapconfig-overviews-adapter'); var TurboCartoAdapter = require('./models/mapconfig/adapter/turbo-carto-adapter'); var DataviewsWidgetsAdapter = require('./models/mapconfig/adapter/dataviews-widgets-adapter'); +var AggregationMapConfigAdapter = require('./models/mapconfig/adapter/aggregation-mapconfig-adapter'); var MapConfigAdapter = require('./models/mapconfig/adapter'); var StatsBackend = require('./backends/stats'); @@ -190,6 +191,7 @@ module.exports = function(serverOptions) { new SqlWrapMapConfigAdapter(), new DataviewsWidgetsAdapter(), new AnalysisMapConfigAdapter(analysisBackend), + new AggregationMapConfigAdapter(pgConnection), new MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi), new TurboCartoAdapter() ); diff --git a/lib/cartodb/utils/query-utils.js b/lib/cartodb/utils/query-utils.js index f8b2a0f5..c0d0aec2 100644 --- a/lib/cartodb/utils/query-utils.js +++ b/lib/cartodb/utils/query-utils.js @@ -21,9 +21,25 @@ module.exports.extractTableNames = function extractTableNames(query) { ].join(''); }; -module.exports.getQueryRowCount = function getQueryRowEstimation(query) { - return 'select CDB_EstimateRowCount($$' + query + '$$) as rows'; -}; +function getQueryRowEstimation(query) { + return 'select CDB_EstimateRowCount($windshaft$' + query + '$windshaft$) as rows'; +} +module.exports.getQueryRowCount = getQueryRowEstimation; + +module.exports.getAggregationMetadata = ctx => ` + WITH + rowEstimation AS ( + ${getQueryRowEstimation(ctx.query)} + ), + geometryType AS ( + SELECT ST_GeometryType(the_geom) as geom_type + FROM (${ctx.query}) AS __cdb_query WHERE the_geom IS NOT NULL LIMIT 1 + ) + SELECT + rows AS count, + geom_type AS type + FROM rowEstimation, geometryType; +`; /** Cast the column to epoch */ module.exports.columnCastTpl = function columnCastTpl(ctx) { @@ -54,4 +70,4 @@ module.exports.countNaNs = function countNaNs(ctx) { return `${!ctx.isFloatColumn ? `0` : `sum(CASE WHEN (${ctx.column} = 'NaN'::float) THEN 1 ELSE 0 END)` }`; -}; \ No newline at end of file +}; diff --git a/test/acceptance/aggregation.js b/test/acceptance/aggregation.js new file mode 100644 index 00000000..aa245fac --- /dev/null +++ b/test/acceptance/aggregation.js @@ -0,0 +1,565 @@ +require('../support/test_helper'); + +const assert = require('../support/assert'); +const TestClient = require('../support/test-client'); +const serverOptions = require('../../lib/cartodb/server_options'); + +const suites = [{ + desc: 'mvt (mapnik)', + usePostGIS: false +}]; + +if (process.env.POSTGIS_VERSION === '2.4') { + suites.push({ + desc: 'mvt (postgis)', + usePostGIS: true + }); +} + +describe('aggregation', function () { + + const POINTS_SQL_1 = ` + select + x + 4 as cartodb_id, + st_setsrid(st_makepoint(x*10, x*10), 4326) as the_geom, + st_transform(st_setsrid(st_makepoint(x*10, x*10), 4326), 3857) as the_geom_webmercator, + x as value + from generate_series(-3, 3) x + `; + + const POINTS_SQL_TIMESTAMP_1 = ` + select + row_number() over() AS cartodb_id, + st_setsrid(st_makepoint(x*10, x*10), 4326) as the_geom, + st_transform(st_setsrid(st_makepoint(x*10, x*10), 4326), 3857) as the_geom_webmercator, + x as value, + date + from + generate_series(-3, 3) x, + generate_series( + '2007-02-15 01:00:00'::timestamp, '2007-02-18 01:00:00'::timestamp, '1 day'::interval + ) date + `; + + const POINTS_SQL_2 = ` + select + x + 4 as cartodb_id, + st_setsrid(st_makepoint(x*10, x*10*(-1)), 4326) as the_geom, + st_transform(st_setsrid(st_makepoint(x*10, x*10*(-1)), 4326), 3857) as the_geom_webmercator, + x as value, + x*x as sqrt_value + from generate_series(-3, 3) x + `; + + const POLYGONS_SQL_1 = ` + select + x + 4 as cartodb_id, + st_buffer(st_setsrid(st_makepoint(x*10, x*10), 4326)::geography, 100000)::geometry as the_geom, + st_transform( + st_buffer(st_setsrid(st_makepoint(x*10, x*10), 4326)::geography, 100000)::geometry, + 3857 + ) as the_geom_webmercator, + x as value + from generate_series(-3, 3) x + `; + + const SQL_WRAP = ` + WITH hgrid AS ( + SELECT + CDB_RectangleGrid ( + ST_Expand(!bbox!, CDB_XYZ_Resolution(1) * 12), + CDB_XYZ_Resolution(1) * 12, + CDB_XYZ_Resolution(1) * 12 + ) as cell + ) + SELECT + hgrid.cell as the_geom_webmercator, + count(1) as agg_value, + count(1) /power( 12 * CDB_XYZ_Resolution(1), 2 ) as agg_value_density, + row_number() over () as cartodb_id + FROM hgrid, (<%= sql %>) i + WHERE ST_Intersects(i.the_geom_webmercator, hgrid.cell) GROUP BY hgrid.cell + `; + + const TURBO_CARTOCSS_SQL_WRAP = ` + #layer { + polygon-fill: ramp([agg_value], (#245668, #04817E, #39AB7E, #8BD16D, #EDEF5D), quantiles); + } + #layer::outline { + line-width: 1; + line-color: #FFFFFF; + line-opacity: 1; + } + `; + + function createVectorMapConfig (layers = [ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: true + } + }, + { + type: 'cartodb', + options: { + sql: POINTS_SQL_2, + aggregation: true + } + } + ]) { + return { + version: '1.6.0', + layers: layers + }; + } + + suites.forEach((suite) => { + const { desc, usePostGIS } = suite; + + describe(desc, function () { + const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS; + + before(function () { + serverOptions.renderer.mvt.usePostGIS = usePostGIS; + }); + + after(function (){ + serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; + }); + + afterEach(function (done) { + this.testClient.drain(done); + }); + + it('should return a layergroup indicating the mapconfig was aggregated', function (done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + threshold: 1 + } + } + }, + { + type: 'cartodb', + options: { + sql: POINTS_SQL_2, + aggregation: { + threshold: 1 + } + } + } + ]); + this.testClient = new TestClient(this.mapConfig); + + this.testClient.getLayergroup((err, body) => { + if (err) { + return done(err); + } + + assert.equal(typeof body.metadata, 'object'); + assert.ok(Array.isArray(body.metadata.layers)); + + body.metadata.layers.forEach(layer => assert.ok(layer.meta.aggregation.mvt)); + body.metadata.layers.forEach(layer => assert.ok(!layer.meta.aggregation.png)); + + done(); + }); + }); + + it('should return a NOT aggregated layergroup', function (done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + cartocss: '#layer { marker-width: [value]; }', + cartocss_version: '2.3.0' + } + } + ]); + this.testClient = new TestClient(this.mapConfig); + + this.testClient.getLayergroup((err, body) => { + if (err) { + return done(err); + } + + assert.equal(typeof body.metadata, 'object'); + assert.ok(Array.isArray(body.metadata.layers)); + + body.metadata.layers.forEach(layer => assert.equal(layer.meta.aggregation, undefined)); + + done(); + }); + }); + + it('should return a layergroup with aggregation and cartocss compatible', function (done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + columns: { + total: { + aggregate_function: 'sum', + aggregated_column: 'value' + } + }, + threshold: 1 + }, + cartocss: '#layer { marker-width: [total]; }', + cartocss_version: '2.3.0' + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + this.testClient.getLayergroup((err, body) => { + if (err) { + return done(err); + } + + assert.equal(typeof body.metadata, 'object'); + assert.ok(Array.isArray(body.metadata.layers)); + + body.metadata.layers.forEach(layer => assert.ok(layer.meta.aggregation.mvt)); + body.metadata.layers.forEach(layer => assert.ok(layer.meta.aggregation.png)); + + done(); + }); + }); + + it('should fail when aggregation and cartocss are not compatible', function (done) { + const response = { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + columns: { + total: { + aggregate_function: 'sum', + aggregated_column: 'value' + } + }, + threshold: 1 + }, + cartocss: '#layer { marker-width: [value]; }', + cartocss_version: '2.3.0' + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + this.testClient.getLayergroup({ response }, (err, body) => { + if (err) { + return done(err); + } + + assert.ok(body.errors[0].match(/column "value" does not exist/)); + + done(); + }); + }); + + it('should fail if cartocss uses "value" column and it\'s not defined in the aggregation', + function (done) { + const response = { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_2, + aggregation: { + threshold: 1 + }, + cartocss: '#layer { marker-width: [value]; }', + cartocss_version: '2.3.0' + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + this.testClient.getLayergroup({ response }, (err, body) => { + if (err) { + return done(err); + } + + assert.ok(body.errors[0].match(/column "value" does not exist/)); + + done(); + }); + }); + + it('should skip aggregation to create a layergroup with aggregation defined already', function (done) { + const mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + columns: { + total: { + aggregate_function: 'sum', + aggregated_column: 'value' + } + }, + threshold: 1 + } + } + } + ]); + + this.testClient = new TestClient(mapConfig); + const options = { aggregation: false }; + + this.testClient.getLayergroup(options, (err, body) => { + if (err) { + return done(err); + } + + assert.equal(typeof body.metadata, 'object'); + assert.ok(Array.isArray(body.metadata.layers)); + + body.metadata.layers.forEach(layer => assert.equal(layer.meta.aggregation, undefined)); + + done(); + }); + }); + + it('when the aggregation param is not valid should respond with error', function (done) { + const mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + threshold: 1 + } + } + } + ]); + + this.testClient = new TestClient(mapConfig); + const options = { + response: { + status: 400 + }, + aggregation: 'wadus' + }; + + this.testClient.getLayergroup(options, (err, body) => { + if (err) { + return done(err); + } + + assert.deepEqual(body, { + errors: [ + "Invalid value for 'aggregation' query param: wadus." + + " Valid ones are 'true' or 'false'" + ], + errors_with_context:[{ + type: 'unknown', + message: "Invalid value for 'aggregation' query param: wadus." + + " Valid ones are 'true' or 'false'" + }] + }); + + done(); + }); + }); + + it('when the layer\'s row count is lower than threshold should skip aggregation', function (done) { + const mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + columns: { + total: { + aggregate_function: 'sum', + aggregated_column: 'value' + } + }, + threshold: 1001 + } + } + } + ]); + + this.testClient = new TestClient(mapConfig); + const options = {}; + + this.testClient.getLayergroup(options, (err, body) => { + if (err) { + return done(err); + } + + assert.equal(typeof body.metadata, 'object'); + assert.ok(Array.isArray(body.metadata.layers)); + + body.metadata.layers.forEach(layer =>{ + assert.deepEqual(layer.meta.aggregation, { png: false, mvt: false }); + }); + + done(); + }); + }); + + it('when the layer\'s geometry type is not point should respond with error', function (done) { + const mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POLYGONS_SQL_1, + aggregation: { + threshold: 1 + } + } + } + ]); + + this.testClient = new TestClient(mapConfig); + const options = { + response: { + status: 400 + } + }; + + this.testClient.getLayergroup(options, (err, body) => { + if (err) { + return done(err); + } + + assert.deepEqual(body, { + errors: [ + 'Unsupported geometry type: ST_Polygon.' + + ' Aggregation is available only for geometry type: ST_Point' + ], + errors_with_context:[{ + type: 'unknown', + message: 'Unsupported geometry type: ST_Polygon.' + + ' Aggregation is available only for geometry type: ST_Point' + }] + }); + + done(); + }); + }); + + it('when sql_wrap is provided should return a layergroup', function (done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql_wrap: SQL_WRAP, + sql: POINTS_SQL_1, + aggregation: { + threshold: 1 + }, + cartocss: TURBO_CARTOCSS_SQL_WRAP, + cartocss_version: '3.0.12' + } + } + ]); + this.testClient = new TestClient(this.mapConfig); + + this.testClient.getLayergroup((err, body) => { + if (err) { + return done(err); + } + + assert.equal(typeof body.metadata, 'object'); + assert.ok(Array.isArray(body.metadata.layers)); + + body.metadata.layers.forEach(layer => assert.ok(layer.meta.aggregation.mvt)); + body.metadata.layers.forEach(layer => assert.ok(layer.meta.aggregation.png)); + + done(); + }); + }); + + it('when sql_wrap is provided should return a tile', function (done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql_wrap: SQL_WRAP, + sql: POINTS_SQL_1, + aggregation: { + threshold: 1 + }, + cartocss: TURBO_CARTOCSS_SQL_WRAP, + cartocss_version: '3.0.12' + } + } + ]); + this.testClient = new TestClient(this.mapConfig); + + this.testClient.getTile(0, 0, 0, {}, (err) => { + if (err) { + return done(err); + } + + done(); + }); + }); + + it('should work when the sql has single quotes', function (done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: ` + SELECT + the_geom_webmercator, + the_geom, + value, + DATE_PART('day', date::timestamp - '1912-12-31 01:00:00'::timestamp )::numeric AS day + FROM (${POINTS_SQL_TIMESTAMP_1}) _query + `, + aggregation: { + threshold: 1 + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + + this.testClient.getLayergroup((err, body) => { + if (err) { + return done(err); + } + + assert.equal(typeof body.metadata, 'object'); + assert.ok(Array.isArray(body.metadata.layers)); + + body.metadata.layers.forEach(layer => assert.ok(layer.meta.aggregation.mvt)); + body.metadata.layers.forEach(layer => assert.ok(!layer.meta.aggregation.png)); + + done(); + }); + }); + }); + }); +}); diff --git a/test/acceptance/analysis/analysis-layers.js b/test/acceptance/analysis/analysis-layers.js index a745fb97..1eb3d4e6 100644 --- a/test/acceptance/analysis/analysis-layers.js +++ b/test/acceptance/analysis/analysis-layers.js @@ -174,7 +174,9 @@ describe('analysis-layers', function() { } }; - testClient.getLayergroup(PERMISSION_DENIED_RESPONSE, function(err, layergroupResult) { + + + testClient.getLayergroup({ response: PERMISSION_DENIED_RESPONSE }, function(err, layergroupResult) { assert.ok(!err, err); assert.deepEqual( layergroupResult.errors, diff --git a/test/acceptance/analysis/error-cases.js b/test/acceptance/analysis/error-cases.js index b79e1046..376d7606 100644 --- a/test/acceptance/analysis/error-cases.js +++ b/test/acceptance/analysis/error-cases.js @@ -62,7 +62,7 @@ describe('analysis-layers error cases', function() { var testClient = new TestClient(mapConfig, 1234); - testClient.getLayergroup(ERROR_RESPONSE, function(err, layergroupResult) { + testClient.getLayergroup({ response: ERROR_RESPONSE }, function(err, layergroupResult) { assert.ok(!err, err); assert.equal(layergroupResult.errors.length, 1); @@ -97,7 +97,7 @@ describe('analysis-layers error cases', function() { var testClient = new TestClient(mapConfig, 1234); - testClient.getLayergroup(ERROR_RESPONSE, function(err, layergroupResult) { + testClient.getLayergroup({ response: ERROR_RESPONSE }, function(err, layergroupResult) { assert.ok(!err, err); assert.equal(layergroupResult.errors.length, 1); @@ -144,7 +144,7 @@ describe('analysis-layers error cases', function() { var testClient = new TestClient(mapConfig, 1234); - testClient.getLayergroup(ERROR_RESPONSE, function(err, layergroupResult) { + testClient.getLayergroup({ response: ERROR_RESPONSE }, function(err, layergroupResult) { assert.ok(!err, err); assert.equal(layergroupResult.errors.length, 1); @@ -190,7 +190,7 @@ describe('analysis-layers error cases', function() { var testClient = new TestClient(mapConfig, 11111); - testClient.getLayergroup(AUTH_ERROR_RESPONSE, function(err, layergroupResult) { + testClient.getLayergroup({ response: AUTH_ERROR_RESPONSE }, function(err, layergroupResult) { assert.ok(!err, err); assert.equal(layergroupResult.errors.length, 1); @@ -246,7 +246,7 @@ describe('analysis-layers error cases', function() { var testClient = new TestClient(mapConfig, 1234); - testClient.getLayergroup(ERROR_RESPONSE, function(err, layergroupResult) { + testClient.getLayergroup({ response: ERROR_RESPONSE }, function(err, layergroupResult) { assert.ok(!err, err); assert.equal(layergroupResult.errors.length, 1); @@ -298,7 +298,7 @@ describe('analysis-layers error cases', function() { var testClient = new TestClient(mapConfig, 1234); - testClient.getLayergroup(ERROR_RESPONSE, function(err, layergroupResult) { + testClient.getLayergroup({ response: ERROR_RESPONSE }, function(err, layergroupResult) { assert.ok(!err, err); assert.equal(layergroupResult.errors.length, 1); @@ -351,7 +351,7 @@ describe('analysis-layers error cases', function() { var testClient = new TestClient(mapConfig, 1234); - testClient.getLayergroup(ERROR_RESPONSE, function(err, layergroupResult) { + testClient.getLayergroup({ response: ERROR_RESPONSE }, function(err, layergroupResult) { assert.ok(!err, err); assert.equal(layergroupResult.errors.length, 1); @@ -415,7 +415,7 @@ describe('analysis-layers error cases', function() { var testClient = new TestClient(mapConfig, 1234); - testClient.getLayergroup(ERROR_RESPONSE, function(err, layergroupResult) { + testClient.getLayergroup({ response: ERROR_RESPONSE }, function(err, layergroupResult) { assert.ok(!err, err); assert.equal(layergroupResult.errors.length, 1); diff --git a/test/acceptance/dataviews/error-cases.js b/test/acceptance/dataviews/error-cases.js index 7e37e8ee..3e6cc2a8 100644 --- a/test/acceptance/dataviews/error-cases.js +++ b/test/acceptance/dataviews/error-cases.js @@ -51,7 +51,7 @@ describe('histogram-dataview', function() { it('should fail when invalid dataviews object is provided, string case', function(done) { var mapConfig = createMapConfig("wadus-string"); this.testClient = new TestClient(mapConfig, 1234); - this.testClient.getLayergroup(ERROR_RESPONSE, function(err, errObj) { + this.testClient.getLayergroup({ response: ERROR_RESPONSE }, function(err, errObj) { assert.ok(!err, err); assert.deepEqual(errObj.errors, [ '"dataviews" must be a valid JSON object: "string" type found' ]); @@ -63,7 +63,7 @@ describe('histogram-dataview', function() { it('should fail when invalid dataviews object is provided, array case', function(done) { var mapConfig = createMapConfig([]); this.testClient = new TestClient(mapConfig, 1234); - this.testClient.getLayergroup(ERROR_RESPONSE, function(err, errObj) { + this.testClient.getLayergroup({ response: ERROR_RESPONSE }, function(err, errObj) { assert.ok(!err, err); assert.deepEqual(errObj.errors, [ '"dataviews" must be a valid JSON object: "array" type found' ]); diff --git a/test/acceptance/error-middleware.js b/test/acceptance/error-middleware.js index 3ad22774..4df5b7e6 100644 --- a/test/acceptance/error-middleware.js +++ b/test/acceptance/error-middleware.js @@ -17,22 +17,24 @@ describe('error middleware', function () { message: "Missing cartocss for layer 0 options", name: "Error", label: "ANONYMOUS LAYERGROUP", - type: "layer", + type: "layer", }, moreErrors: [] }; this.testClient = new TestClient(mapConfig, 1234); - - const expectedResponse = { - status: 400, - headers: { - 'Content-Type': 'application/json; charset=utf-8', - 'X-Tiler-Errors': JSON.stringify(errorHeader) + + const params = { + response: { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8', + 'X-Tiler-Errors': JSON.stringify(errorHeader) + } } }; - this.testClient.getLayergroup(expectedResponse, (err) => { + this.testClient.getLayergroup(params, (err) => { assert.ifError(err); done(); }); diff --git a/test/acceptance/ported/multilayer_error_cases.js b/test/acceptance/ported/multilayer_error_cases.js index 9efd7c7b..6e7f5964 100644 --- a/test/acceptance/ported/multilayer_error_cases.js +++ b/test/acceptance/ported/multilayer_error_cases.js @@ -151,7 +151,7 @@ describe('multilayer error cases', function() { }; ServerOptions.afterLayergroupCreateCalls = 0; this.client = new TestClient(layergroup); - this.client.getLayergroup({status: 400}, function(err, parsed) { + this.client.getLayergroup({ response: { status: 400 } }, function(err, parsed) { assert.ok(!err, err); // See http://github.com/CartoDB/Windshaft/issues/159 assert.equal(ServerOptions.afterLayergroupCreateCalls, 0); diff --git a/test/acceptance/regressions.js b/test/acceptance/regressions.js index 43dad070..421df02f 100644 --- a/test/acceptance/regressions.js +++ b/test/acceptance/regressions.js @@ -28,7 +28,7 @@ describe('regressions', function() { var testClient = new TestClient(mapConfig, 1234); - testClient.getLayergroup(ERROR_RESPONSE, function(err, layergroupResult) { + testClient.getLayergroup({ response: ERROR_RESPONSE }, function(err, layergroupResult) { assert.ok(!err, err); assert.equal(layergroupResult.errors.length, 1); diff --git a/test/acceptance/turbo-carto/error-cases.js b/test/acceptance/turbo-carto/error-cases.js index 0d25ae93..b6424c72 100644 --- a/test/acceptance/turbo-carto/error-cases.js +++ b/test/acceptance/turbo-carto/error-cases.js @@ -51,7 +51,7 @@ describe('turbo-carto error cases', function() { it('should return invalid number of ramp error', function(done) { this.testClient = new TestClient(makeMapconfig('ramp([pop_max], 8, 96, 3, (8,24,96,128))')); - this.testClient.getLayergroup(ERROR_RESPONSE, function(err, layergroup) { + this.testClient.getLayergroup({ response: ERROR_RESPONSE }, function(err, layergroup) { assert.ok(!err, err); assert.ok(layergroup.hasOwnProperty('errors')); @@ -65,7 +65,7 @@ describe('turbo-carto error cases', function() { it('should return invalid column from datasource', function(done) { this.testClient = new TestClient(makeMapconfig(null, 'ramp([wadus_column], (red, green, blue))')); - this.testClient.getLayergroup(ERROR_RESPONSE, function(err, layergroup) { + this.testClient.getLayergroup({ response: ERROR_RESPONSE }, function(err, layergroup) { assert.ok(!err, err); assert.ok(layergroup.hasOwnProperty('errors')); @@ -80,7 +80,7 @@ describe('turbo-carto error cases', function() { it('should return invalid method from datasource', function(done) { this.testClient = new TestClient(makeMapconfig(null, 'ramp([wadus_column], (red, green, blue), wadusmethod)')); - this.testClient.getLayergroup(ERROR_RESPONSE, function(err, layergroup) { + this.testClient.getLayergroup({ response: ERROR_RESPONSE }, function(err, layergroup) { assert.ok(!err, err); assert.ok(layergroup.hasOwnProperty('errors')); @@ -95,7 +95,7 @@ describe('turbo-carto error cases', function() { it('should fail by falling back to normal carto parser', function(done) { this.testClient = new TestClient(makeMapconfig('ramp([price], (8,24,96), (8,24,96));//(red, green, blue))')); - this.testClient.getLayergroup(ERROR_RESPONSE, function(err, layergroup) { + this.testClient.getLayergroup({ response: ERROR_RESPONSE }, function(err, layergroup) { assert.ok(!err, err); assert.ok(layergroup.hasOwnProperty('errors')); @@ -109,7 +109,7 @@ describe('turbo-carto error cases', function() { it('turbo-carto: should return error invalid column from datasource with some context', function(done) { this.testClient = new TestClient(makeMapconfig(null, 'ramp([wadus_column], (red, green, blue))')); - this.testClient.getLayergroup(ERROR_RESPONSE, function(err, layergroup) { + this.testClient.getLayergroup({ response: ERROR_RESPONSE }, function(err, layergroup) { assert.ok(!err, err); assert.ok(layergroup.hasOwnProperty('errors')); @@ -164,7 +164,7 @@ describe('turbo-carto error cases', function() { }; this.testClient = new TestClient(multipleErrorsMapConfig); - this.testClient.getLayergroup(ERROR_RESPONSE, function(err, layergroup) { + this.testClient.getLayergroup({ response: ERROR_RESPONSE }, function(err, layergroup) { assert.ok(!err, err); assert.ok(layergroup.hasOwnProperty('errors')); diff --git a/test/acceptance/turbo-carto/regressions.js b/test/acceptance/turbo-carto/regressions.js index f7f39ccb..9e2d377f 100644 --- a/test/acceptance/turbo-carto/regressions.js +++ b/test/acceptance/turbo-carto/regressions.js @@ -79,7 +79,7 @@ describe('turbo-carto regressions', function() { ].join('\n'); this.testClient = new TestClient(makeMapconfig('SELECT * FROM test_table_private_1', cartocss)); - this.testClient.getLayergroup(TestClient.RESPONSE.ERROR, function(err, layergroup) { + this.testClient.getLayergroup({ response: TestClient.RESPONSE.ERROR }, function(err, layergroup) { assert.ok(!err, err); assert.ok(!layergroup.hasOwnProperty('layergroupid')); diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js index 9893f00a..52ead026 100644 --- a/test/acceptance/user-database-timeout-limit.js +++ b/test/acceptance/user-database-timeout-limit.js @@ -138,7 +138,7 @@ describe('user database timeout limit', function () { } }; - this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { + this.testClient.getLayergroup({ response: expectedResponse }, (err, timeoutError) => { assert.deepEqual(timeoutError, { errors: [ 'You are over platform\'s limits. Please contact us to know more details' ], errors_with_context: [{ @@ -177,7 +177,7 @@ describe('user database timeout limit', function () { return done(err); } - this.testClient.getLayergroup(expectedResponse, (err, res) => { + this.testClient.getLayergroup({ response: expectedResponse }, (err, res) => { if (err) { return done(err); } @@ -259,7 +259,7 @@ describe('user database timeout limit', function () { return done(err); } - this.testClient.getLayergroup(expectedResponse, (err, res) => { + this.testClient.getLayergroup({ response: expectedResponse }, (err, res) => { if (err) { return done(err); } @@ -360,7 +360,7 @@ describe('user database timeout limit', function () { } }; - this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { + this.testClient.getLayergroup({ response: expectedResponse }, (err, timeoutError) => { assert.deepEqual(timeoutError, { errors: [ 'You are over platform\'s limits. Please contact us to know more details' ], errors_with_context: [{ @@ -387,7 +387,7 @@ describe('user database timeout limit', function () { } }; - this.testClient.getLayergroup(expectedResponse, (err, res) => { + this.testClient.getLayergroup({ response: expectedResponse }, (err, res) => { if (err) { return done(err); } @@ -426,16 +426,16 @@ describe('user database timeout limit', function () { this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { assert.ifError(err); - + var tileJSON = tile.toJSON(); assert.equal(Array.isArray(tileJSON), true); assert.equal(tileJSON.length, 2); assert.equal(tileJSON[0].name, 'errorTileSquareLayer'); assert.equal(tileJSON[1].name, 'errorTileStripesLayer'); - + done(); }); - + }); }); }); @@ -467,7 +467,7 @@ describe('user database timeout limit', function () { } }; - this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { + this.testClient.getLayergroup({ response: expectedResponse }, (err, timeoutError) => { assert.deepEqual(timeoutError, { errors: [ 'You are over platform\'s limits. Please contact us to know more details' ], errors_with_context: [{ @@ -494,7 +494,7 @@ describe('user database timeout limit', function () { } }; - this.testClient.getLayergroup(expectedResponse, (err, res) => { + this.testClient.getLayergroup({ response: expectedResponse }, (err, res) => { if (err) { return done(err); } @@ -571,7 +571,7 @@ describe('user database timeout limit', function () { } }; - this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { + this.testClient.getLayergroup({ response: expectedResponse }, (err, timeoutError) => { assert.deepEqual(timeoutError, { errors: [ 'You are over platform\'s limits. Please contact us to know more details' ], errors_with_context: [{ @@ -601,7 +601,7 @@ describe('user database timeout limit', function () { } }; - this.testClient.getLayergroup(expectedResponse, (err, res) => { + this.testClient.getLayergroup({ response: expectedResponse }, (err, res) => { if (err) { return done(err); } @@ -702,7 +702,7 @@ describe('user database timeout limit', function () { } }; - this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { + this.testClient.getLayergroup({ response: expectedResponse }, (err, timeoutError) => { assert.deepEqual(timeoutError, { errors: [ 'You are over platform\'s limits. Please contact us to know more details' ], errors_with_context: [{ @@ -740,7 +740,7 @@ describe('user database timeout limit', function () { } }; - this.testClient.getLayergroup(expectedResponse, (err, res) => { + this.testClient.getLayergroup({ response: expectedResponse }, (err, res) => { if (err) { return done(err); } diff --git a/test/acceptance/user-render-timeout-limit.js b/test/acceptance/user-render-timeout-limit.js index 5942d045..93770080 100644 --- a/test/acceptance/user-render-timeout-limit.js +++ b/test/acceptance/user-render-timeout-limit.js @@ -92,7 +92,7 @@ describe('user render timeout limit', function () { } }; - this.testClient.getLayergroup(expectedResponse, (err, timeoutError) => { + this.testClient.getLayergroup({ response: expectedResponse }, (err, timeoutError) => { assert.ifError(err); assert.deepEqual(timeoutError, { @@ -245,7 +245,7 @@ describe('user render timeout limit', function () { assert.equal(tileJSON.length, 2); assert.equal(tileJSON[0].name, 'errorTileSquareLayer'); assert.equal(tileJSON[1].name, 'errorTileStripesLayer'); - + done(); }); }); @@ -399,4 +399,3 @@ describe('user render timeout limit', function () { }); }); }); - diff --git a/test/acceptance/vector-layergroup.js b/test/acceptance/vector-layergroup.js index 54ec29ef..0778453e 100644 --- a/test/acceptance/vector-layergroup.js +++ b/test/acceptance/vector-layergroup.js @@ -206,7 +206,7 @@ suites.forEach((suite) => { this.testClient.mapConfig.layers[0].options.cartocss = cartocss; this.testClient.mapConfig.layers[0].options.cartocss_version = cartocssVersion; - this.testClient.getLayergroup(response, (err, body) => { + this.testClient.getLayergroup({ response }, (err, body) => { if (err) { return done(err); } diff --git a/test/support/prepare_db.sh b/test/support/prepare_db.sh index b31b2c9b..03d97fc7 100755 --- a/test/support/prepare_db.sh +++ b/test/support/prepare_db.sh @@ -77,7 +77,7 @@ if test x"$PREPARE_PGSQL" = xyes; then createdb -Ttemplate_postgis -EUTF8 "${TEST_DB}" || die "Could not create test database" LOCAL_SQL_SCRIPTS='analysis_catalog windshaft.test gadm4 ported/populated_places_simple_reduced cdb_analysis_check cdb_invalidate_varnish' - REMOTE_SQL_SCRIPTS='CDB_QueryStatements CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_OverviewsSupport CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins CDB_Hexagon CDB_XYZ CDB_EstimateRowCount' + REMOTE_SQL_SCRIPTS='CDB_QueryStatements CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_OverviewsSupport CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins CDB_Hexagon CDB_XYZ CDB_EstimateRowCount CDB_RectangleGrid' CURL_ARGS="" for i in ${REMOTE_SQL_SCRIPTS} @@ -99,7 +99,7 @@ if test x"$PREPARE_PGSQL" = xyes; then sed -e 's/PARALLEL \= [A-Z]*,/''/g' \ -e 's/PARALLEL [A-Z]*/''/g' sql/$i.sql > $TMPFILE mv $TMPFILE sql/$i.sql - fi + fi cat sql/${i}.sql | sed -e 's/cartodb\./public./g' -e "s/''cartodb''/''public''/g" | sed "s/:PUBLICUSER/${PUBLICUSER}/" | diff --git a/test/support/test-client.js b/test/support/test-client.js index 3188d0fc..f4437a39 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -618,8 +618,19 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { } var data = templateId ? params.placeholders : self.mapConfig; + + const queryParams = {}; + + if (self.apiKey) { + queryParams.api_key = self.apiKey; + } + + if (params.aggregation !== undefined) { + queryParams.aggregation = params.aggregation; + } + var path = templateId ? - urlNamed + '/' + templateId + '?' + qs.stringify({api_key: self.apiKey}) : + urlNamed + '/' + templateId + '?' + qs.stringify(queryParams) : url; assert.response(self.server, @@ -647,7 +658,7 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { ); }, function getTileResult(err, layergroupId) { - // jshint maxcomplexity:12 + // jshint maxcomplexity:13 assert.ifError(err); self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; @@ -671,8 +682,14 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { url += [z,x,y].join('/'); url += '.' + format; + const queryParams = {}; + if (self.apiKey) { - url += '?' + qs.stringify({api_key: self.apiKey}); + queryParams.api_key = self.apiKey; + } + + if (Object.keys(queryParams).length) { + url += '?' + qs.stringify(queryParams); } var request = { @@ -754,12 +771,21 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { ); }; -TestClient.prototype.getLayergroup = function(expectedResponse, callback) { +TestClient.prototype.getLayergroup = function (params, callback) { + // jshint maxcomplexity: 7 var self = this; if (!callback) { - callback = expectedResponse; - expectedResponse = { + callback = params; + params = null; + } + + if (!params) { + params = {}; + } + + if (!params.response) { + params.response = { status: 200, headers: { 'Content-Type': 'application/json; charset=utf-8' @@ -769,8 +795,18 @@ TestClient.prototype.getLayergroup = function(expectedResponse, callback) { var url = '/api/v1/map'; - if (this.apiKey) { - url += '?' + qs.stringify({api_key: this.apiKey}); + const queryParams = {}; + + if (self.apiKey) { + queryParams.api_key = self.apiKey; + } + + if (params.aggregation !== undefined) { + queryParams.aggregation = params.aggregation; + } + + if (Object.keys(queryParams).length) { + url += '?' + qs.stringify(queryParams); } assert.response(self.server, @@ -783,7 +819,7 @@ TestClient.prototype.getLayergroup = function(expectedResponse, callback) { }, data: JSON.stringify(self.mapConfig) }, - expectedResponse, + params.response, function(res, err) { var parsedBody; // If there is a response, we are still interested in catching the created keys diff --git a/yarn.lock b/yarn.lock index 34122d94..78e663f0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -"abaculus@github:cartodb/abaculus#2.0.3-cdb1": +abaculus@cartodb/abaculus#2.0.3-cdb1: version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -226,7 +226,7 @@ camshaft@0.60.0: dot "^1.0.3" request "^2.69.0" -"canvas@github:cartodb/node-canvas#1.6.2-cdb2": +canvas@cartodb/node-canvas#1.6.2-cdb2: version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -244,15 +244,15 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" -"carto@github:cartodb/carto#0.15.1-cdb1": +carto@CartoDB/carto#0.15.1-cdb1: version "0.15.1-cdb1" - resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" dependencies: mapnik-reference "~6.0.2" optimist "~0.6.0" underscore "~1.6.0" -"carto@github:cartodb/carto#0.15.1-cdb3": +carto@cartodb/carto#0.15.1-cdb3: version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -1633,7 +1633,7 @@ pg-types@1.*: postgres-date "~1.0.0" postgres-interval "^1.1.0" -"pg@github:cartodb/node-postgres#6.1.6-cdb1": +pg@cartodb/node-postgres#6.1.6-cdb1: version "6.1.6" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3eef52dd1e655f658a4ee8ac5697688b3ecfed44" dependencies: @@ -2223,7 +2223,7 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb4": +tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb4: version "2.3.1-cdb4" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/faa2b638da2d119b78281575d40255cb523f6ca6" dependencies: @@ -2231,7 +2231,7 @@ through@2: mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb3": +tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb3: version "0.6.18-cdb3" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/23bd1c31dd57d0b76c86b9f1eaf62462b3c17d01" dependencies: