diff --git a/NEWS.md b/NEWS.md index 1e592442..605a9751 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,8 +1,11 @@ # Changelog -## 7.0.1 +## 7.1.0 Released 2019-mm-dd +Announcements: +- Experimental support for listing features in a grid when the map uses the dynamic agregation. + ## 7.0.0 Released 2019-02-22 diff --git a/lib/cartodb/api/api-router.js b/lib/cartodb/api/api-router.js index fafc90b3..af82a3b8 100644 --- a/lib/cartodb/api/api-router.js +++ b/lib/cartodb/api/api-router.js @@ -21,6 +21,8 @@ const OverviewsMetadataBackend = require('../backends/overviews-metadata'); const FilterStatsApi = require('../backends/filter-stats'); const TablesExtentBackend = require('../backends/tables-extent'); +const ClusterBackend = require('../backends/cluster'); + const LayergroupAffectedTablesCache = require('../cache/layergroup_affected_tables'); const SurrogateKeysCache = require('../cache/surrogate_keys_cache'); const VarnishHttpCacheBackend = require('../cache/backend/varnish_http'); @@ -110,6 +112,7 @@ module.exports = class ApiRouter { const analysisBackend = new AnalysisBackend(metadataBackend, serverOptions.analysis); const dataviewBackend = new DataviewBackend(analysisBackend); const statsBackend = new StatsBackend(); + const clusterBackend = new ClusterBackend(); const userLimitsBackend = new UserLimitsBackend(metadataBackend, { limits: { @@ -179,7 +182,8 @@ module.exports = class ApiRouter { statsBackend, layergroupMetadata, namedMapProviderCache, - tablesExtentBackend + tablesExtentBackend, + clusterBackend }; this.mapRouter = new MapRouter({ collaborators }); diff --git a/lib/cartodb/api/map/clustered-features-layergroup-controller.js b/lib/cartodb/api/map/clustered-features-layergroup-controller.js new file mode 100644 index 00000000..5c17c5b1 --- /dev/null +++ b/lib/cartodb/api/map/clustered-features-layergroup-controller.js @@ -0,0 +1,95 @@ +'use strict'; + +const layergroupToken = require('../middlewares/layergroup-token'); +const cleanUpQueryParams = require('../middlewares/clean-up-query-params'); +const credentials = require('../middlewares/credentials'); +const dbConnSetup = require('../middlewares/db-conn-setup'); +const authorize = require('../middlewares/authorize'); +const rateLimit = require('../middlewares/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; +const createMapStoreMapConfigProvider = require('../middlewares/map-store-map-config-provider'); +const cacheControlHeader = require('../middlewares/cache-control-header'); +const cacheChannelHeader = require('../middlewares/cache-channel-header'); +const surrogateKeyHeader = require('../middlewares/surrogate-key-header'); +const lastModifiedHeader = require('../middlewares/last-modified-header'); + +module.exports = class AggregatedFeaturesLayergroupController { + constructor ( + clusterBackend, + pgConnection, + mapStore, + userLimitsBackend, + layergroupAffectedTablesCache, + authBackend, + surrogateKeysCache + ) { + this.clusterBackend = clusterBackend; + this.pgConnection = pgConnection; + this.mapStore = mapStore; + this.userLimitsBackend = userLimitsBackend; + this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; + this.authBackend = authBackend; + this.surrogateKeysCache = surrogateKeysCache; + } + + register (mapRouter) { + mapRouter.get('/:token/:layer/:z/cluster/:clusterId', this.middlewares()); + } + + middlewares () { + return [ + layergroupToken(), + credentials(), + authorize(this.authBackend), + dbConnSetup(this.pgConnection), + // TODO: create its rate limit + rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.ATTRIBUTES), + cleanUpQueryParams([ 'aggregation' ]), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsBackend, + this.pgConnection, + this.layergroupAffectedTablesCache + ), + getClusteredFeatures(this.clusterBackend), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader() + ]; + } +}; + +function getClusteredFeatures (clusterBackend) { + return function getFeatureAttributesMiddleware (req, res, next) { + req.profiler.start('windshaft.maplayer_cluster_features'); + + const { mapConfigProvider } = res.locals; + const { user, token } = res.locals; + const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; + const { layer, z: zoom, clusterId } = req.params; + const { aggregation } = req.query; + + const params = { + user, token, + dbuser, dbname, dbpassword, dbhost, dbport, + layer, zoom, clusterId, + aggregation + }; + + clusterBackend.getClusterFeatures(mapConfigProvider, params, (err, features, stats = {}) => { + req.profiler.add(stats); + + if (err) { + err.label = 'GET CLUSTERED FEATURES'; + return next(err); + } + + res.statusCode = 200; + const { rows, fields } = features; + res.body = { rows, fields }; + + next(); + }); + }; +} diff --git a/lib/cartodb/api/map/map-router.js b/lib/cartodb/api/map/map-router.js index dac55670..8fb7b8b2 100644 --- a/lib/cartodb/api/map/map-router.js +++ b/lib/cartodb/api/map/map-router.js @@ -10,6 +10,7 @@ const TileLayergroupController = require('./tile-layergroup-controller'); const AnonymousMapController = require('./anonymous-map-controller'); const PreviewTemplateController = require('./preview-template-controller'); const AnalysesCatalogController = require('./analyses-catalog-controller'); +const ClusteredFeaturesLayergroupController = require('./clustered-features-layergroup-controller'); module.exports = class MapRouter { constructor ({ collaborators }) { @@ -32,7 +33,8 @@ module.exports = class MapRouter { statsBackend, layergroupMetadata, namedMapProviderCache, - tablesExtentBackend + tablesExtentBackend, + clusterBackend } = collaborators; this.analysisLayergroupController = new AnalysisLayergroupController( @@ -112,6 +114,16 @@ module.exports = class MapRouter { authBackend, userLimitsBackend ); + + this.clusteredFeaturesLayergroupController = new ClusteredFeaturesLayergroupController( + clusterBackend, + pgConnection, + mapStore, + userLimitsBackend, + layergroupAffectedTablesCache, + authBackend, + surrogateKeysCache + ); } register (apiRouter, mapPaths) { @@ -125,6 +137,7 @@ module.exports = class MapRouter { this.anonymousMapController.register(mapRouter); this.previewTemplateController.register(mapRouter); this.analysesController.register(mapRouter); + this.clusteredFeaturesLayergroupController.register(mapRouter); mapPaths.forEach(path => apiRouter.use(path, mapRouter)); } diff --git a/lib/cartodb/backends/cluster.js b/lib/cartodb/backends/cluster.js new file mode 100644 index 00000000..c9776eba --- /dev/null +++ b/lib/cartodb/backends/cluster.js @@ -0,0 +1,251 @@ +'use strict'; + +const PSQL = require('cartodb-psql'); +const dbParamsFromReqParams = require('../utils/database-params'); +const debug = require('debug')('backend:cluster'); +const AggregationMapConfig = require('../models/aggregation/aggregation-mapconfig'); + +module.exports = class ClusterBackend { + // TODO: reduce complexity + // jshint maxcomplexity: 17 + getClusterFeatures (mapConfigProvider, params, callback) { + mapConfigProvider.getMapConfig((err, _mapConfig) => { + if (err) { + return callback(err); + } + + let pg; + try { + pg = new PSQL(dbParamsFromReqParams(params)); + } catch (error) { + return callback(error); + } + + const { user, token, layer: layerIndex } = params; + const mapConfig = new AggregationMapConfig(user, _mapConfig.obj(), pg); + + const layer = mapConfig.getLayer(layerIndex); + + if (layer.options.aggregation === false || !mapConfig.isAggregationLayer(layerIndex)) { + const error = new Error(`Map ${token} has no aggregation defined for layer ${layerIndex}`); + error.http_status = 400; + error.type = 'layer'; + error.subtype = 'aggregation'; + error.layer = { + index: layerIndex, + type: layer.type + }; + + debug(error); + return callback(error); + } + + let { aggregation } = params; + + if ( aggregation !== undefined) { + try { + aggregation = JSON.parse(aggregation); + } catch (err) { + const error = new Error(`Invalid aggregation input, should be a a valid JSON`); + error.http_status = 400; + error.type = 'layer'; + error.subtype = 'aggregation'; + error.layer = { + index: layerIndex, + type: layer.type + }; + + return callback(error); + } + + const { columns, expressions } = aggregation; + + if (!Array.isArray(columns) || !columns.length) { + const error = new Error( + `Invalid aggregation input, columns should be and array of column names` + ); + error.http_status = 400; + error.type = 'layer'; + error.subtype = 'aggregation'; + error.layer = { + index: layerIndex, + type: layer.type + }; + + return callback(error); + } + + if (expressions !== undefined) { + if (expressions === null || + Array.isArray(expressions) || + ['string', 'number', 'boolean'].includes(typeof expressions)) { + const error = new Error( + `Invalid aggregation input, expressions should be and object with valid functions` + ); + error.http_status = 400; + error.type = 'layer'; + error.subtype = 'aggregation'; + error.layer = { + index: layerIndex, + type: layer.type + }; + + return callback(error); + } + + for (const { aggregate_function, aggregated_column } of Object.values(expressions)) { + if (typeof aggregated_column !== 'string') { + const error = new Error(`Invalid aggregation input, aggregated column should be an string`); + error.http_status = 400; + error.type = 'layer'; + error.subtype = 'aggregation'; + error.layer = { + index: layerIndex, + type: layer.type + }; + + return callback(error); + } + + if (typeof aggregate_function !== 'string') { + const error = new Error( + `Invalid aggregation input, aggregate function should be an string` + ); + error.http_status = 400; + error.type = 'layer'; + error.subtype = 'aggregation'; + error.layer = { + index: layerIndex, + type: layer.type + }; + + return callback(error); + } + } + } + } + + const query = layer.options.sql_raw; + const resolution = layer.options.aggregation.resolution || 1; + + getColumnsName(pg, query, (err, columns) => { + if (err) { + return callback(err); + } + + const { zoom, clusterId } = params; + + getClusterFeatures(pg, zoom, clusterId, columns, query, resolution, aggregation, (err, features) => { + if (err) { + return callback(err); + } + + return callback(null, features); + }); + }); + }); + } +}; + +const SKIP_COLUMNS = { + 'the_geom': true, + 'the_geom_webmercator': true +}; + +function getColumnsName (pg, query, callback) { + const sql = limitedQuery({ + query: query + }); + + debug('> getColumnsName:', sql); + + pg.query(sql, function (err, resultSet) { + if (err) { + return callback(err); + } + + const fields = resultSet.fields || []; + const columnNames = fields.map(field => field.name) + .filter(columnName => !SKIP_COLUMNS[columnName]); + + return callback(null, columnNames); + }, true); +} + +function getClusterFeatures (pg, zoom, clusterId, columns, query, resolution, aggregation, callback) { + let sql = clusterFeaturesQuery({ + zoom: zoom, + id: clusterId, + query: query, + res: 256/resolution, + columns: columns + }); + + if (aggregation !== undefined) { + let { columns = [], expressions = [] } = aggregation; + + if (expressions) { + expressions = Object.entries(expressions) + .map(([columnName, exp]) => `${exp.aggregate_function}(${exp.aggregated_column}) AS ${columnName}`); + } + + sql = aggregationQuery({ + columns, + expressions, + query: sql + }); + } + + debug('> getClusterFeatures:', sql); + + pg.query(sql, (err, data) => { + if (err) { + return callback(err); + } + + return callback(null, data); + } , true); // use read-only transaction +} + +const limitedQuery = ctx => `SELECT * FROM (${ctx.query}) __cdb_schema LIMIT 0`; +const clusterFeaturesQuery = ctx => ` + WITH + _cdb_params AS ( + SELECT + ${gridResolution(ctx)} AS res + ), + _cell AS ( + SELECT + ST_MakeEnvelope( + Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res)*_cdb_params.res, + Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)*_cdb_params.res, + Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res + 1)*_cdb_params.res, + Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res + 1)*_cdb_params.res, + 3857 + ) AS bbox + FROM (${ctx.query}) _cdb_query, _cdb_params + WHERE _cdb_query.cartodb_id = ${ctx.id} + ) + SELECT + ${ctx.columns.join(', ')} + FROM ( + SELECT _cdb_query.* + FROM _cell, (${ctx.query}) _cdb_query + WHERE ST_Intersects(_cdb_query.the_geom_webmercator, _cell.bbox) + ) __cdb_non_geoms_query +`; + +const gridResolution = ctx => { + const minimumResolution = 2*Math.PI*6378137/Math.pow(2,38); + const pixelSize = `CDB_XYZ_Resolution(${ctx.zoom})`; + return `GREATEST(${256/ctx.res}*${pixelSize}, ${minimumResolution})::double precision`; +}; + +const aggregationQuery = ctx => ` + SELECT + count(1) as _cdb_feature_count + ${ctx.columns.length ? `,${ctx.columns.join(', ')}` : ''} + ${ctx.expressions.length ? `,${ctx.expressions.join(', ')}` : ''} + FROM (${ctx.query}) __cdb_aggregation + ${ctx.columns.length ? `GROUP BY ${ctx.columns.join(', ')}` : ''} +`; diff --git a/lib/cartodb/models/aggregation/aggregation-query.js b/lib/cartodb/models/aggregation/aggregation-query.js index 1bddf02d..f74df93a 100644 --- a/lib/cartodb/models/aggregation/aggregation-query.js +++ b/lib/cartodb/models/aggregation/aggregation-query.js @@ -448,3 +448,31 @@ const aggregationQueryTemplates = { module.exports.SUPPORTED_PLACEMENTS = Object.keys(aggregationQueryTemplates); module.exports.GEOMETRY_COLUMN = 'the_geom_webmercator'; + +const clusterFeaturesQuery = ctx => ` + WITH + _cdb_params AS ( + SELECT + ${gridResolution(ctx)} AS res + ), + _cell AS ( + SELECT + ST_MakeEnvelope( + Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res)*_cdb_params.res, + Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)*_cdb_params.res, + Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res + 1)*_cdb_params.res, + Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res + 1)*_cdb_params.res, + 3857 + ) AS bbox + FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params + WHERE _cdb_query.cartodb_id = ${ctx.id} + ) + SELECT _cdb_query.* FROM _cell, (${ctx.sourceQuery}) _cdb_query + WHERE ST_Intersects(_cdb_query.the_geom_webmercator, _cell.bbox) +`; + +module.exports.featuresQuery = (id, options) => clusterFeaturesQuery({ + id, + sourceQuery: options.query, + res: 256/options.resolution +}); diff --git a/package-lock.json b/package-lock.json index fb770817..94641675 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "windshaft-cartodb", - "version": "7.0.1", + "version": "7.1.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index f2ce3a36..b62b2e6e 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "7.0.1", + "version": "7.1.0", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" diff --git a/test/acceptance/cluster.js b/test/acceptance/cluster.js new file mode 100644 index 00000000..632b6e8a --- /dev/null +++ b/test/acceptance/cluster.js @@ -0,0 +1,839 @@ +'use strict'; + +require('../support/test_helper'); + +const assert = require('../support/assert'); +const TestClient = require('../support/test-client'); + +const POINTS_SQL_1 = ` + select + x + 4 as cartodb_id, + st_setsrid(st_makepoint(x*10, x*10), 4326) as the_geom, + st_transform(st_setsrid(st_makepoint(x*10, x*10), 4326), 3857) as the_geom_webmercator, + x as value, + CASE + WHEN x % 2 = 0 THEN 'even' + ELSE 'odd' + END AS type + from generate_series(-3, 3) x +`; + +const defaultLayers = [{ + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + threshold: 1 + } + } +}]; + +function createVectorMapConfig (layers = defaultLayers) { + return { + version: '1.8.0', + layers: layers + }; +} + +describe('cluster', function () { + describe('w/o aggregation', function () { + it('should return error while fetching disaggregated features', function (done) { + const mapConfig = createVectorMapConfig([{ + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + cartocss: TestClient.CARTOCSS.POINTS, + cartocss_version: '2.3.0' + } + }]); + const testClient = new TestClient(mapConfig); + const zoom = 0; + const cartodb_id = 1; + const layerId = 0; + const params = { + response: { + status: 400 + } + }; + + testClient.getClusterFeatures(zoom, cartodb_id, layerId, params, (err, body) => { + if (err) { + return done(err); + } + + assert.deepStrictEqual(body, { + errors:[ 'Map c502fc8fc1cb0d5e412db3deabffeee5 has no aggregation defined for layer 0' ], + errors_with_context:[ + { + layer: { + index: '0', + type: 'cartodb' + }, + message: 'Map c502fc8fc1cb0d5e412db3deabffeee5 has no aggregation defined for layer 0', + subtype: 'aggregation', + type: 'layer' + } + ] + }); + + testClient.drain(done); + }); + }); + + it('with aggregation disabled should return error while fetching disaggregated features', function (done) { + const mapConfig = createVectorMapConfig([{ + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: false + } + }]); + const testClient = new TestClient(mapConfig); + const zoom = 0; + const cartodb_id = 1; + const layerId = 0; + const params = { + response: { + status: 400 + } + }; + + testClient.getClusterFeatures(zoom, cartodb_id, layerId, params, (err, body) => { + if (err) { + return done(err); + } + + assert.deepStrictEqual(body, { + errors:[ 'Map 18792467ae296929d04e32dfe7f81a80 has no aggregation defined for layer 0' ], + errors_with_context:[ + { + layer: { + index: '0', + type: 'cartodb' + }, + message: 'Map 18792467ae296929d04e32dfe7f81a80 has no aggregation defined for layer 0', + subtype: 'aggregation', + type: 'layer' + } + ] + }); + + testClient.drain(done); + }); + }); + }); + + describe('fetch features within a cluster grid', function () { + const suite = [ + { + zoom: 0, + cartodb_id: 1, + resolution: 0.5, + expected: [ { cartodb_id: 1, value: -3, type: 'odd' } ] + }, + { + zoom: 0, + cartodb_id: 2, + resolution: 0.5, + expected: [ { cartodb_id: 2, value: -2, type: 'even' } ] + }, + { + zoom: 0, + cartodb_id: 3, + resolution: 0.5, + expected: [ { cartodb_id: 3, value: -1, type: 'odd' } ] + }, + { + zoom: 0, + cartodb_id: 4, + resolution: 0.5, + expected: [ { cartodb_id: 4, value: 0, type: 'even' } ] + }, + { + zoom: 0, + cartodb_id: 5, + resolution: 0.5, + expected: [ { cartodb_id: 5, value: 1, type: 'odd' } ] + }, + { + zoom: 0, + cartodb_id: 6, + resolution: 0.5, + expected: [ { cartodb_id: 6, value: 2, type: 'even' } ] + }, + { + zoom: 0, + cartodb_id: 7, + resolution: 0.5, + expected: [ { cartodb_id: 7, value: 3, type: 'odd' } ] + }, + { + zoom: 0, + cartodb_id: 1, + resolution: 1, + expected: [ { cartodb_id: 1, value: -3, type: 'odd' } ] + }, + { + zoom: 0, + cartodb_id: 2, + resolution: 1, + expected: [ { cartodb_id: 2, value: -2, type: 'even' } ] + }, + { + zoom: 0, + cartodb_id: 3, + resolution: 1, + expected: [ { cartodb_id: 3, value: -1, type: 'odd' } ] + }, + { + zoom: 0, + cartodb_id: 4, + resolution: 1, + expected: [ { cartodb_id: 4, value: 0, type: 'even' } ] + }, + { + zoom: 0, + cartodb_id: 5, + resolution: 1, + expected: [ { cartodb_id: 5, value: 1, type: 'odd' } ] + }, + { + zoom: 0, + cartodb_id: 6, + resolution: 1, + expected: [ { cartodb_id: 6, value: 2, type: 'even' } ] + }, + { + zoom: 0, + cartodb_id: 7, + resolution: 1, + expected: [ { cartodb_id: 7, value: 3, type: 'odd' } ] + }, + { + zoom: 0, + cartodb_id: 1, + resolution: 50, + expected: [ + { cartodb_id: 1, value: -3, type: 'odd' }, + { cartodb_id: 2, value: -2, type: 'even' }, + { cartodb_id: 3, value: -1, type: 'odd' }, + { cartodb_id: 4, value: 0, type: 'even' }, + ] + }, + { + zoom: 0, + cartodb_id: 5, + resolution: 50, + expected: [ + { cartodb_id: 5, value: 1, type: 'odd' }, + { cartodb_id: 6, value: 2, type: 'even' }, + { cartodb_id: 7, value: 3, type: 'odd' } + ] + }, + { + zoom: 1, + cartodb_id: 1, + resolution: 1, + expected: [ { cartodb_id: 1, value: -3, type: 'odd' } ] + }, + { + zoom: 1, + cartodb_id: 2, + resolution: 1, + expected: [ { cartodb_id: 2, value: -2, type: 'even' } ] + }, + { + zoom: 1, + cartodb_id: 3, + resolution: 1, + expected: [ { cartodb_id: 3, value: -1, type: 'odd' } ] + }, + { + zoom: 1, + cartodb_id: 4, + resolution: 1, + expected: [ { cartodb_id: 4, value: 0, type: 'even' } ] + }, + { + zoom: 1, + cartodb_id: 5, + resolution: 1, + expected: [ { cartodb_id: 5, value: 1, type: 'odd' } ] + }, + { + zoom: 1, + cartodb_id: 6, + resolution: 1, + expected: [ { cartodb_id: 6, value: 2, type: 'even' } ] + }, + { + zoom: 1, + cartodb_id: 7, + resolution: 1, + expected: [ { cartodb_id: 7, value: 3, type: 'odd' } ] + }, + { + zoom: 1, + cartodb_id: 1, + resolution: 50, + expected: [ + { cartodb_id: 1, value: -3, type: 'odd' }, + { cartodb_id: 2, value: -2, type: 'even'}, + { cartodb_id: 3, value: -1, type: 'odd' }, + { cartodb_id: 4, value: 0, type: 'even' }, + ] + }, + { + zoom: 1, + cartodb_id: 5, + resolution: 50, + expected: [ + { cartodb_id: 5, value: 1, type: 'odd' }, + { cartodb_id: 6, value: 2, type: 'even' }, + { cartodb_id: 7, value: 3, type: 'odd' } + ] + } + ]; + + suite.forEach(({ zoom, cartodb_id, resolution, expected }) => { + const description = `should get features for z: ${zoom} cartodb_id: ${cartodb_id}, res: ${resolution}`; + it(description, function (done) { + const mapConfig = createVectorMapConfig([{ + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + threshold: 1, + resolution: resolution + } + } + }]); + const testClient = new TestClient(mapConfig); + const layerId = 0; + const params = {}; + + testClient.getClusterFeatures(zoom, cartodb_id, layerId, params, (err, body) => { + if (err) { + return done(err); + } + + assert.deepStrictEqual(body.rows, expected); + testClient.drain(done); + }); + }); + }); + }); + + describe('valid aggregation input', function () { + const suite = [ + { + zoom: 0, + cartodb_id: 1, + resolution: 1, + aggregation: { columns: ['type'] }, + expected: [ { _cdb_feature_count: 1, type: 'odd' } ] + }, + { + zoom: 0, + cartodb_id: 2, + resolution: 1, + aggregation: { columns: ['type'] }, + expected: [ { _cdb_feature_count: 1, type: 'even' } ] + }, + { + zoom: 0, + cartodb_id: 3, + resolution: 1, + aggregation: { columns: ['type'] }, + expected: [ { _cdb_feature_count: 1, type: 'odd' } ] + }, + { + zoom: 0, + cartodb_id: 4, + resolution: 1, + aggregation: { columns: ['type'] }, + expected: [ { _cdb_feature_count: 1, type: 'even' } ] + }, + { + zoom: 0, + cartodb_id: 5, + resolution: 1, + aggregation: { columns: ['type'] }, + expected: [ { _cdb_feature_count: 1, type: 'odd' } ] + }, + { + zoom: 0, + cartodb_id: 6, + resolution: 1, + aggregation: { columns: ['type'] }, + expected: [ { _cdb_feature_count: 1, type: 'even' } ] + }, + { + zoom: 0, + cartodb_id: 7, + resolution: 1, + aggregation: { columns: ['type'] }, + expected: [ { _cdb_feature_count: 1, type: 'odd' } ] + }, + { + zoom: 0, + cartodb_id: 1, + resolution: 50, + aggregation: { columns: ['type'] }, + expected: [ + { _cdb_feature_count: 2, type: 'even' }, + { _cdb_feature_count: 2, type: 'odd' } + ] + }, + { + zoom: 0, + cartodb_id: 5, + resolution: 50, + aggregation: { columns: ['type'] }, + expected: [ + { _cdb_feature_count: 1, type: 'even' }, + { _cdb_feature_count: 2, type: 'odd' } + ] + }, + { + zoom: 0, + cartodb_id: 1, + resolution: 1, + aggregation: { + columns: [ 'type' ], + expressions: { + max_value: { + aggregate_function: 'max', + aggregated_column: 'value', + } + } + }, + expected: [ { _cdb_feature_count: 1, type: 'odd', max_value: -3 } ] + }, + { + zoom: 0, + cartodb_id: 2, + resolution: 1, + aggregation: { + columns: [ 'type' ], + expressions: { + max_value: { + aggregate_function: 'max', + aggregated_column: 'value', + } + } + }, + expected: [ { _cdb_feature_count: 1, type: 'even', max_value: -2 } ] + }, + { + zoom: 0, + cartodb_id: 3, + resolution: 1, + aggregation: { + columns: [ 'type' ], + expressions: { + max_value: { + aggregate_function: 'max', + aggregated_column: 'value', + } + } + }, + expected: [ { _cdb_feature_count: 1, type: 'odd', max_value: -1 } ] + }, + { + zoom: 0, + cartodb_id: 4, + resolution: 1, + aggregation: { + columns: [ 'type' ], + expressions: { + max_value: { + aggregate_function: 'max', + aggregated_column: 'value', + } + } + }, + expected: [ { _cdb_feature_count: 1, type: 'even', max_value: 0 } ] + }, + { + zoom: 0, + cartodb_id: 5, + resolution: 1, + aggregation: { + columns: [ 'type' ], + expressions: { + max_value: { + aggregate_function: 'max', + aggregated_column: 'value', + } + } + }, + expected: [ { _cdb_feature_count: 1, type: 'odd', max_value: 1 } ] + }, + { + zoom: 0, + cartodb_id: 6, + resolution: 1, + aggregation: { + columns: [ 'type' ], + expressions: { + max_value: { + aggregate_function: 'max', + aggregated_column: 'value', + } + } + }, + expected: [ { _cdb_feature_count: 1, type: 'even', max_value: 2 } ] + }, + { + zoom: 0, + cartodb_id: 7, + resolution: 1, + aggregation: { + columns: [ 'type' ], + expressions: { + max_value: { + aggregate_function: 'max', + aggregated_column: 'value', + } + } + }, + expected: [ { _cdb_feature_count: 1, type: 'odd', max_value: 3 } ] + }, + { + zoom: 0, + cartodb_id: 1, + resolution: 50, + aggregation: { + columns: [ 'type' ], + expressions: { + max_value: { + aggregate_function: 'max', + aggregated_column: 'value', + } + } + }, + expected: [ + { _cdb_feature_count: 2, type: 'even', max_value: 0 }, + { _cdb_feature_count: 2, type: 'odd', max_value: -1 } + ] + }, + { + zoom: 0, + cartodb_id: 5, + resolution: 50, + aggregation: { + columns: [ 'type' ], + expressions: { + max_value: { + aggregate_function: 'max', + aggregated_column: 'value', + } + } + }, + expected: [ + { _cdb_feature_count: 1, type: 'even', max_value: 2 }, + { _cdb_feature_count: 2, type: 'odd', max_value: 3 } + ] + } + ]; + + suite.forEach(({ zoom, cartodb_id, resolution, aggregation, expected }) => { + it('should return features aggregated by type', function (done) { + const mapConfig = createVectorMapConfig([{ + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + threshold: 1, + resolution + } + } + }]); + const testClient = new TestClient(mapConfig); + const layerId = 0; + const params = { aggregation }; + + testClient.getClusterFeatures(zoom, cartodb_id, layerId, params, (err, body) => { + if (err) { + return done(err); + } + + assert.deepStrictEqual(body.rows, expected); + + testClient.drain(done); + }); + }); + }); + }); + + describe('invalid aggregation input', function () { + const expectedColumnsError = { + errors:[ 'Invalid aggregation input, columns should be and array of column names' ], + errors_with_context:[ + { + layer: { + index: '0', + type: 'cartodb' + }, + message: 'Invalid aggregation input, columns should be and array of column names', + subtype: 'aggregation', + type: 'layer' + } + ] + }; + + const expectedExpressionsError = { + errors:[ 'Invalid aggregation input, expressions should be and object with valid functions' ], + errors_with_context:[ + { + layer: { + index: '0', + type: 'cartodb' + }, + message: 'Invalid aggregation input, expressions should be and object with valid functions', + subtype: 'aggregation', + type: 'layer' + } + ] + }; + + const invalidFunctionExpressionsError = { + errors:[ 'function wadus(integer) does not exist' ], + errors_with_context:[ + { + message: 'function wadus(integer) does not exist', + type: 'unknown' + } + ] + }; + + const invalidColumnExpressionsError = { + errors:[ 'column \"wadus\" does not exist' ], + errors_with_context:[ + { + message: 'column \"wadus\" does not exist', + type: 'unknown' + } + ] + }; + + + const expectedAggregatedColumnError = { + errors:[ 'Invalid aggregation input, aggregated column should be an string' ], + errors_with_context:[ + { + layer: { + index: '0', + type: 'cartodb' + }, + message: 'Invalid aggregation input, aggregated column should be an string', + subtype: 'aggregation', + type: 'layer' + } + ] + }; + + const expectedAggregateFunctionError = { + errors:[ 'Invalid aggregation input, aggregate function should be an string' ], + errors_with_context:[ + { + layer: { + index: '0', + type: 'cartodb' + }, + message: 'Invalid aggregation input, aggregate function should be an string', + subtype: 'aggregation', + type: 'layer' + } + ] + }; + + const suite = [ + { + description: 'empty aggregation object should respond with error', + zoom: 0, + cartodb_id: 1, + resolution: 1, + aggregation: {}, + expected: expectedColumnsError + }, + { + description: 'empty aggregation array should respond with error', + zoom: 0, + cartodb_id: 1, + resolution: 1, + aggregation: [], + expected: expectedColumnsError + }, + { + description: 'aggregation as string should respond with error', + zoom: 0, + cartodb_id: 1, + resolution: 1, + aggregation: 'wadus', + expected: expectedColumnsError + }, + { + description: 'empty columns array should respond with error', + zoom: 0, + cartodb_id: 1, + resolution: 1, + aggregation: { columns: [] }, + expected: expectedColumnsError + }, + { + description: 'empty columns object should respond with error', + zoom: 0, + cartodb_id: 1, + resolution: 1, + aggregation: { columns: {} }, + expected: expectedColumnsError + }, + { + description: 'columns as string should respond with error', + zoom: 0, + cartodb_id: 1, + resolution: 1, + aggregation: { columns: 'wadus' }, + expected: expectedColumnsError + }, + { + description: 'columns as null should respond with error', + zoom: 0, + cartodb_id: 1, + resolution: 1, + aggregation: { columns: null }, + expected: expectedColumnsError + }, + { + description: 'empty expressions array should respond with error', + zoom: 0, + cartodb_id: 1, + resolution: 1, + aggregation: { columns: [ 'type' ], expressions: [] }, + expected: expectedExpressionsError + }, + { + description: 'empty expressions number should respond with error', + zoom: 0, + cartodb_id: 1, + resolution: 1, + aggregation: { columns: [ 'type' ], expressions: 1 }, + expected: expectedExpressionsError + }, + { + description: 'expressions as string should respond with error', + zoom: 0, + cartodb_id: 1, + resolution: 1, + aggregation: { columns: [ 'type' ], expressions: 'wadus' }, + expected: expectedExpressionsError + }, + { + description: 'expressions as null should respond with error', + zoom: 0, + cartodb_id: 1, + resolution: 1, + aggregation: { columns: [ 'type' ], expressions: null }, + expected: expectedExpressionsError + }, + { + description: 'invalid aggregation function should respond with error', + zoom: 0, + cartodb_id: 1, + resolution: 1, + aggregation: { + columns: [ 'type' ], + expressions: { + max_value: { + aggregate_function: 'wadus', + aggregated_column: 'value' + } + } + }, + expected: invalidFunctionExpressionsError + }, + { + description: 'invalid aggregation column should respond with error', + zoom: 0, + cartodb_id: 1, + resolution: 1, + aggregation: { + columns: [ 'type' ], + expressions: { + max_value: { + aggregate_function: 'max', + aggregated_column: 'wadus' + } + } + }, + status: 404, + expected: invalidColumnExpressionsError + }, + { + description: 'aggregated column as non string should respond with error', + zoom: 0, + cartodb_id: 1, + resolution: 1, + aggregation: { + columns: [ 'type' ], + expressions: { + max_value: { + aggregate_function: 'max', + aggregated_column: 1 + } + } + }, + expected: expectedAggregatedColumnError + }, + { + description: 'aggregate function as non string should respond with error', + zoom: 0, + cartodb_id: 1, + resolution: 1, + aggregation: { + columns: [ 'type' ], + expressions: { + max_value: { + aggregate_function: 1, + aggregated_column: 'value' + } + } + }, + expected: expectedAggregateFunctionError + } + ]; + + suite.forEach(({ description, zoom, cartodb_id, resolution, aggregation, expected, status = 400 }) => { + it(description, function (done) { + const mapConfig = createVectorMapConfig([{ + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + threshold: 1, + resolution + } + } + }]); + const testClient = new TestClient(mapConfig); + const layerId = 0; + const params = { + response: { status }, + aggregation + }; + + testClient.getClusterFeatures(zoom, cartodb_id, layerId, params, (err, body) => { + if (err) { + return done(err); + } + + assert.deepStrictEqual(body, expected); + + testClient.drain(done); + }); + }); + }); + }); +}); diff --git a/test/support/test-client.js b/test/support/test-client.js index 77363b01..5d88f6e0 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -620,6 +620,111 @@ TestClient.prototype.getFeatureAttributes = function(featureId, layerId, params, ); }; +TestClient.prototype.getClusterFeatures = function (zoom, clusterId, layerId, params, callback) { + var self = this; + + if (!callback) { + callback = params; + params = {}; + } + + var extraParams = {}; + + if (this.apiKey) { + extraParams.api_key = this.apiKey; + } + + // if (params && params.filters) { + // extraParams.filters = JSON.stringify(params.filters); + // } + + var url = '/api/v1/map'; + if (Object.keys(extraParams).length > 0) { + url += '?' + qs.stringify(extraParams); + } + + var expectedResponse = params.response || { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + step( + function createLayergroup() { + var next = this; + assert.response(self.server, + { + url: url, + method: 'POST', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, + data: JSON.stringify(self.mapConfig) + }, + { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }, + function(res, err) { + if (err) { + return next(err); + } + + var parsedBody = JSON.parse(res.body); + + if (parsedBody.layergroupid) { + self.keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0; + self.keysToDelete['user:localhost:mapviews:global'] = 5; + } + + return next(null, parsedBody.layergroupid); + } + ); + }, + function getCLusterFeatures (err, layergroupId) { + assert.ifError(err); + + var next = this; + + let queryParams = ''; + if (params.aggregation) { + queryParams = qs.stringify({ aggregation: JSON.stringify(params.aggregation) }); + } + + url = `/api/v1/map/${layergroupId}/${layerId}/${zoom}/cluster/${clusterId}?${queryParams}`; + + assert.response(self.server, + { + url: url, + method: 'GET', + headers: { + host: 'localhost' + } + }, + expectedResponse, + function(res, err) { + if (err) { + return next(err); + } + + next(null, JSON.parse(res.body)); + } + ); + }, + function finish(err, attributes) { + if (err) { + return callback(err); + } + + return callback(null, attributes); + } + ); +}; + TestClient.prototype.getTile = function(z, x, y, params, callback) { var self = this;