Merge pull request #1077 from CartoDB/mvt-list-aggregated-features

Experimental support for listing features in a grid
This commit is contained in:
Daniel G. Aubert
2019-03-01 18:06:38 +01:00
committed by GitHub
10 changed files with 1343 additions and 5 deletions
+4 -1
View File
@@ -1,8 +1,11 @@
# Changelog
## 7.0.1
## 7.1.0
Released 2019-mm-dd
Announcements:
- Experimental support for listing features in a grid when the map uses the dynamic agregation.
## 7.0.0
Released 2019-02-22
+5 -1
View File
@@ -21,6 +21,8 @@ const OverviewsMetadataBackend = require('../backends/overviews-metadata');
const FilterStatsApi = require('../backends/filter-stats');
const TablesExtentBackend = require('../backends/tables-extent');
const ClusterBackend = require('../backends/cluster');
const LayergroupAffectedTablesCache = require('../cache/layergroup_affected_tables');
const SurrogateKeysCache = require('../cache/surrogate_keys_cache');
const VarnishHttpCacheBackend = require('../cache/backend/varnish_http');
@@ -110,6 +112,7 @@ module.exports = class ApiRouter {
const analysisBackend = new AnalysisBackend(metadataBackend, serverOptions.analysis);
const dataviewBackend = new DataviewBackend(analysisBackend);
const statsBackend = new StatsBackend();
const clusterBackend = new ClusterBackend();
const userLimitsBackend = new UserLimitsBackend(metadataBackend, {
limits: {
@@ -179,7 +182,8 @@ module.exports = class ApiRouter {
statsBackend,
layergroupMetadata,
namedMapProviderCache,
tablesExtentBackend
tablesExtentBackend,
clusterBackend
};
this.mapRouter = new MapRouter({ collaborators });
@@ -0,0 +1,95 @@
'use strict';
const layergroupToken = require('../middlewares/layergroup-token');
const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
const credentials = require('../middlewares/credentials');
const dbConnSetup = require('../middlewares/db-conn-setup');
const authorize = require('../middlewares/authorize');
const rateLimit = require('../middlewares/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
const createMapStoreMapConfigProvider = require('../middlewares/map-store-map-config-provider');
const cacheControlHeader = require('../middlewares/cache-control-header');
const cacheChannelHeader = require('../middlewares/cache-channel-header');
const surrogateKeyHeader = require('../middlewares/surrogate-key-header');
const lastModifiedHeader = require('../middlewares/last-modified-header');
module.exports = class AggregatedFeaturesLayergroupController {
constructor (
clusterBackend,
pgConnection,
mapStore,
userLimitsBackend,
layergroupAffectedTablesCache,
authBackend,
surrogateKeysCache
) {
this.clusterBackend = clusterBackend;
this.pgConnection = pgConnection;
this.mapStore = mapStore;
this.userLimitsBackend = userLimitsBackend;
this.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
this.authBackend = authBackend;
this.surrogateKeysCache = surrogateKeysCache;
}
register (mapRouter) {
mapRouter.get('/:token/:layer/:z/cluster/:clusterId', this.middlewares());
}
middlewares () {
return [
layergroupToken(),
credentials(),
authorize(this.authBackend),
dbConnSetup(this.pgConnection),
// TODO: create its rate limit
rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.ATTRIBUTES),
cleanUpQueryParams([ 'aggregation' ]),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsBackend,
this.pgConnection,
this.layergroupAffectedTablesCache
),
getClusteredFeatures(this.clusterBackend),
cacheControlHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader()
];
}
};
function getClusteredFeatures (clusterBackend) {
return function getFeatureAttributesMiddleware (req, res, next) {
req.profiler.start('windshaft.maplayer_cluster_features');
const { mapConfigProvider } = res.locals;
const { user, token } = res.locals;
const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals;
const { layer, z: zoom, clusterId } = req.params;
const { aggregation } = req.query;
const params = {
user, token,
dbuser, dbname, dbpassword, dbhost, dbport,
layer, zoom, clusterId,
aggregation
};
clusterBackend.getClusterFeatures(mapConfigProvider, params, (err, features, stats = {}) => {
req.profiler.add(stats);
if (err) {
err.label = 'GET CLUSTERED FEATURES';
return next(err);
}
res.statusCode = 200;
const { rows, fields } = features;
res.body = { rows, fields };
next();
});
};
}
+14 -1
View File
@@ -10,6 +10,7 @@ const TileLayergroupController = require('./tile-layergroup-controller');
const AnonymousMapController = require('./anonymous-map-controller');
const PreviewTemplateController = require('./preview-template-controller');
const AnalysesCatalogController = require('./analyses-catalog-controller');
const ClusteredFeaturesLayergroupController = require('./clustered-features-layergroup-controller');
module.exports = class MapRouter {
constructor ({ collaborators }) {
@@ -32,7 +33,8 @@ module.exports = class MapRouter {
statsBackend,
layergroupMetadata,
namedMapProviderCache,
tablesExtentBackend
tablesExtentBackend,
clusterBackend
} = collaborators;
this.analysisLayergroupController = new AnalysisLayergroupController(
@@ -112,6 +114,16 @@ module.exports = class MapRouter {
authBackend,
userLimitsBackend
);
this.clusteredFeaturesLayergroupController = new ClusteredFeaturesLayergroupController(
clusterBackend,
pgConnection,
mapStore,
userLimitsBackend,
layergroupAffectedTablesCache,
authBackend,
surrogateKeysCache
);
}
register (apiRouter, mapPaths) {
@@ -125,6 +137,7 @@ module.exports = class MapRouter {
this.anonymousMapController.register(mapRouter);
this.previewTemplateController.register(mapRouter);
this.analysesController.register(mapRouter);
this.clusteredFeaturesLayergroupController.register(mapRouter);
mapPaths.forEach(path => apiRouter.use(path, mapRouter));
}
+251
View File
@@ -0,0 +1,251 @@
'use strict';
const PSQL = require('cartodb-psql');
const dbParamsFromReqParams = require('../utils/database-params');
const debug = require('debug')('backend:cluster');
const AggregationMapConfig = require('../models/aggregation/aggregation-mapconfig');
module.exports = class ClusterBackend {
// TODO: reduce complexity
// jshint maxcomplexity: 17
getClusterFeatures (mapConfigProvider, params, callback) {
mapConfigProvider.getMapConfig((err, _mapConfig) => {
if (err) {
return callback(err);
}
let pg;
try {
pg = new PSQL(dbParamsFromReqParams(params));
} catch (error) {
return callback(error);
}
const { user, token, layer: layerIndex } = params;
const mapConfig = new AggregationMapConfig(user, _mapConfig.obj(), pg);
const layer = mapConfig.getLayer(layerIndex);
if (layer.options.aggregation === false || !mapConfig.isAggregationLayer(layerIndex)) {
const error = new Error(`Map ${token} has no aggregation defined for layer ${layerIndex}`);
error.http_status = 400;
error.type = 'layer';
error.subtype = 'aggregation';
error.layer = {
index: layerIndex,
type: layer.type
};
debug(error);
return callback(error);
}
let { aggregation } = params;
if ( aggregation !== undefined) {
try {
aggregation = JSON.parse(aggregation);
} catch (err) {
const error = new Error(`Invalid aggregation input, should be a a valid JSON`);
error.http_status = 400;
error.type = 'layer';
error.subtype = 'aggregation';
error.layer = {
index: layerIndex,
type: layer.type
};
return callback(error);
}
const { columns, expressions } = aggregation;
if (!Array.isArray(columns) || !columns.length) {
const error = new Error(
`Invalid aggregation input, columns should be and array of column names`
);
error.http_status = 400;
error.type = 'layer';
error.subtype = 'aggregation';
error.layer = {
index: layerIndex,
type: layer.type
};
return callback(error);
}
if (expressions !== undefined) {
if (expressions === null ||
Array.isArray(expressions) ||
['string', 'number', 'boolean'].includes(typeof expressions)) {
const error = new Error(
`Invalid aggregation input, expressions should be and object with valid functions`
);
error.http_status = 400;
error.type = 'layer';
error.subtype = 'aggregation';
error.layer = {
index: layerIndex,
type: layer.type
};
return callback(error);
}
for (const { aggregate_function, aggregated_column } of Object.values(expressions)) {
if (typeof aggregated_column !== 'string') {
const error = new Error(`Invalid aggregation input, aggregated column should be an string`);
error.http_status = 400;
error.type = 'layer';
error.subtype = 'aggregation';
error.layer = {
index: layerIndex,
type: layer.type
};
return callback(error);
}
if (typeof aggregate_function !== 'string') {
const error = new Error(
`Invalid aggregation input, aggregate function should be an string`
);
error.http_status = 400;
error.type = 'layer';
error.subtype = 'aggregation';
error.layer = {
index: layerIndex,
type: layer.type
};
return callback(error);
}
}
}
}
const query = layer.options.sql_raw;
const resolution = layer.options.aggregation.resolution || 1;
getColumnsName(pg, query, (err, columns) => {
if (err) {
return callback(err);
}
const { zoom, clusterId } = params;
getClusterFeatures(pg, zoom, clusterId, columns, query, resolution, aggregation, (err, features) => {
if (err) {
return callback(err);
}
return callback(null, features);
});
});
});
}
};
const SKIP_COLUMNS = {
'the_geom': true,
'the_geom_webmercator': true
};
function getColumnsName (pg, query, callback) {
const sql = limitedQuery({
query: query
});
debug('> getColumnsName:', sql);
pg.query(sql, function (err, resultSet) {
if (err) {
return callback(err);
}
const fields = resultSet.fields || [];
const columnNames = fields.map(field => field.name)
.filter(columnName => !SKIP_COLUMNS[columnName]);
return callback(null, columnNames);
}, true);
}
function getClusterFeatures (pg, zoom, clusterId, columns, query, resolution, aggregation, callback) {
let sql = clusterFeaturesQuery({
zoom: zoom,
id: clusterId,
query: query,
res: 256/resolution,
columns: columns
});
if (aggregation !== undefined) {
let { columns = [], expressions = [] } = aggregation;
if (expressions) {
expressions = Object.entries(expressions)
.map(([columnName, exp]) => `${exp.aggregate_function}(${exp.aggregated_column}) AS ${columnName}`);
}
sql = aggregationQuery({
columns,
expressions,
query: sql
});
}
debug('> getClusterFeatures:', sql);
pg.query(sql, (err, data) => {
if (err) {
return callback(err);
}
return callback(null, data);
} , true); // use read-only transaction
}
const limitedQuery = ctx => `SELECT * FROM (${ctx.query}) __cdb_schema LIMIT 0`;
const clusterFeaturesQuery = ctx => `
WITH
_cdb_params AS (
SELECT
${gridResolution(ctx)} AS res
),
_cell AS (
SELECT
ST_MakeEnvelope(
Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res)*_cdb_params.res,
Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)*_cdb_params.res,
Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res + 1)*_cdb_params.res,
Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res + 1)*_cdb_params.res,
3857
) AS bbox
FROM (${ctx.query}) _cdb_query, _cdb_params
WHERE _cdb_query.cartodb_id = ${ctx.id}
)
SELECT
${ctx.columns.join(', ')}
FROM (
SELECT _cdb_query.*
FROM _cell, (${ctx.query}) _cdb_query
WHERE ST_Intersects(_cdb_query.the_geom_webmercator, _cell.bbox)
) __cdb_non_geoms_query
`;
const gridResolution = ctx => {
const minimumResolution = 2*Math.PI*6378137/Math.pow(2,38);
const pixelSize = `CDB_XYZ_Resolution(${ctx.zoom})`;
return `GREATEST(${256/ctx.res}*${pixelSize}, ${minimumResolution})::double precision`;
};
const aggregationQuery = ctx => `
SELECT
count(1) as _cdb_feature_count
${ctx.columns.length ? `,${ctx.columns.join(', ')}` : ''}
${ctx.expressions.length ? `,${ctx.expressions.join(', ')}` : ''}
FROM (${ctx.query}) __cdb_aggregation
${ctx.columns.length ? `GROUP BY ${ctx.columns.join(', ')}` : ''}
`;
@@ -448,3 +448,31 @@ const aggregationQueryTemplates = {
module.exports.SUPPORTED_PLACEMENTS = Object.keys(aggregationQueryTemplates);
module.exports.GEOMETRY_COLUMN = 'the_geom_webmercator';
const clusterFeaturesQuery = ctx => `
WITH
_cdb_params AS (
SELECT
${gridResolution(ctx)} AS res
),
_cell AS (
SELECT
ST_MakeEnvelope(
Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res)*_cdb_params.res,
Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)*_cdb_params.res,
Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res + 1)*_cdb_params.res,
Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res + 1)*_cdb_params.res,
3857
) AS bbox
FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params
WHERE _cdb_query.cartodb_id = ${ctx.id}
)
SELECT _cdb_query.* FROM _cell, (${ctx.sourceQuery}) _cdb_query
WHERE ST_Intersects(_cdb_query.the_geom_webmercator, _cell.bbox)
`;
module.exports.featuresQuery = (id, options) => clusterFeaturesQuery({
id,
sourceQuery: options.query,
res: 256/options.resolution
});
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "windshaft-cartodb",
"version": "7.0.1",
"version": "7.1.0",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
+1 -1
View File
@@ -1,7 +1,7 @@
{
"private": true,
"name": "windshaft-cartodb",
"version": "7.0.1",
"version": "7.1.0",
"description": "A map tile server for CartoDB",
"keywords": [
"cartodb"
+839
View File
@@ -0,0 +1,839 @@
'use strict';
require('../support/test_helper');
const assert = require('../support/assert');
const TestClient = require('../support/test-client');
const POINTS_SQL_1 = `
select
x + 4 as cartodb_id,
st_setsrid(st_makepoint(x*10, x*10), 4326) as the_geom,
st_transform(st_setsrid(st_makepoint(x*10, x*10), 4326), 3857) as the_geom_webmercator,
x as value,
CASE
WHEN x % 2 = 0 THEN 'even'
ELSE 'odd'
END AS type
from generate_series(-3, 3) x
`;
const defaultLayers = [{
type: 'cartodb',
options: {
sql: POINTS_SQL_1,
aggregation: {
threshold: 1
}
}
}];
function createVectorMapConfig (layers = defaultLayers) {
return {
version: '1.8.0',
layers: layers
};
}
describe('cluster', function () {
describe('w/o aggregation', function () {
it('should return error while fetching disaggregated features', function (done) {
const mapConfig = createVectorMapConfig([{
type: 'cartodb',
options: {
sql: POINTS_SQL_1,
cartocss: TestClient.CARTOCSS.POINTS,
cartocss_version: '2.3.0'
}
}]);
const testClient = new TestClient(mapConfig);
const zoom = 0;
const cartodb_id = 1;
const layerId = 0;
const params = {
response: {
status: 400
}
};
testClient.getClusterFeatures(zoom, cartodb_id, layerId, params, (err, body) => {
if (err) {
return done(err);
}
assert.deepStrictEqual(body, {
errors:[ 'Map c502fc8fc1cb0d5e412db3deabffeee5 has no aggregation defined for layer 0' ],
errors_with_context:[
{
layer: {
index: '0',
type: 'cartodb'
},
message: 'Map c502fc8fc1cb0d5e412db3deabffeee5 has no aggregation defined for layer 0',
subtype: 'aggregation',
type: 'layer'
}
]
});
testClient.drain(done);
});
});
it('with aggregation disabled should return error while fetching disaggregated features', function (done) {
const mapConfig = createVectorMapConfig([{
type: 'cartodb',
options: {
sql: POINTS_SQL_1,
aggregation: false
}
}]);
const testClient = new TestClient(mapConfig);
const zoom = 0;
const cartodb_id = 1;
const layerId = 0;
const params = {
response: {
status: 400
}
};
testClient.getClusterFeatures(zoom, cartodb_id, layerId, params, (err, body) => {
if (err) {
return done(err);
}
assert.deepStrictEqual(body, {
errors:[ 'Map 18792467ae296929d04e32dfe7f81a80 has no aggregation defined for layer 0' ],
errors_with_context:[
{
layer: {
index: '0',
type: 'cartodb'
},
message: 'Map 18792467ae296929d04e32dfe7f81a80 has no aggregation defined for layer 0',
subtype: 'aggregation',
type: 'layer'
}
]
});
testClient.drain(done);
});
});
});
describe('fetch features within a cluster grid', function () {
const suite = [
{
zoom: 0,
cartodb_id: 1,
resolution: 0.5,
expected: [ { cartodb_id: 1, value: -3, type: 'odd' } ]
},
{
zoom: 0,
cartodb_id: 2,
resolution: 0.5,
expected: [ { cartodb_id: 2, value: -2, type: 'even' } ]
},
{
zoom: 0,
cartodb_id: 3,
resolution: 0.5,
expected: [ { cartodb_id: 3, value: -1, type: 'odd' } ]
},
{
zoom: 0,
cartodb_id: 4,
resolution: 0.5,
expected: [ { cartodb_id: 4, value: 0, type: 'even' } ]
},
{
zoom: 0,
cartodb_id: 5,
resolution: 0.5,
expected: [ { cartodb_id: 5, value: 1, type: 'odd' } ]
},
{
zoom: 0,
cartodb_id: 6,
resolution: 0.5,
expected: [ { cartodb_id: 6, value: 2, type: 'even' } ]
},
{
zoom: 0,
cartodb_id: 7,
resolution: 0.5,
expected: [ { cartodb_id: 7, value: 3, type: 'odd' } ]
},
{
zoom: 0,
cartodb_id: 1,
resolution: 1,
expected: [ { cartodb_id: 1, value: -3, type: 'odd' } ]
},
{
zoom: 0,
cartodb_id: 2,
resolution: 1,
expected: [ { cartodb_id: 2, value: -2, type: 'even' } ]
},
{
zoom: 0,
cartodb_id: 3,
resolution: 1,
expected: [ { cartodb_id: 3, value: -1, type: 'odd' } ]
},
{
zoom: 0,
cartodb_id: 4,
resolution: 1,
expected: [ { cartodb_id: 4, value: 0, type: 'even' } ]
},
{
zoom: 0,
cartodb_id: 5,
resolution: 1,
expected: [ { cartodb_id: 5, value: 1, type: 'odd' } ]
},
{
zoom: 0,
cartodb_id: 6,
resolution: 1,
expected: [ { cartodb_id: 6, value: 2, type: 'even' } ]
},
{
zoom: 0,
cartodb_id: 7,
resolution: 1,
expected: [ { cartodb_id: 7, value: 3, type: 'odd' } ]
},
{
zoom: 0,
cartodb_id: 1,
resolution: 50,
expected: [
{ cartodb_id: 1, value: -3, type: 'odd' },
{ cartodb_id: 2, value: -2, type: 'even' },
{ cartodb_id: 3, value: -1, type: 'odd' },
{ cartodb_id: 4, value: 0, type: 'even' },
]
},
{
zoom: 0,
cartodb_id: 5,
resolution: 50,
expected: [
{ cartodb_id: 5, value: 1, type: 'odd' },
{ cartodb_id: 6, value: 2, type: 'even' },
{ cartodb_id: 7, value: 3, type: 'odd' }
]
},
{
zoom: 1,
cartodb_id: 1,
resolution: 1,
expected: [ { cartodb_id: 1, value: -3, type: 'odd' } ]
},
{
zoom: 1,
cartodb_id: 2,
resolution: 1,
expected: [ { cartodb_id: 2, value: -2, type: 'even' } ]
},
{
zoom: 1,
cartodb_id: 3,
resolution: 1,
expected: [ { cartodb_id: 3, value: -1, type: 'odd' } ]
},
{
zoom: 1,
cartodb_id: 4,
resolution: 1,
expected: [ { cartodb_id: 4, value: 0, type: 'even' } ]
},
{
zoom: 1,
cartodb_id: 5,
resolution: 1,
expected: [ { cartodb_id: 5, value: 1, type: 'odd' } ]
},
{
zoom: 1,
cartodb_id: 6,
resolution: 1,
expected: [ { cartodb_id: 6, value: 2, type: 'even' } ]
},
{
zoom: 1,
cartodb_id: 7,
resolution: 1,
expected: [ { cartodb_id: 7, value: 3, type: 'odd' } ]
},
{
zoom: 1,
cartodb_id: 1,
resolution: 50,
expected: [
{ cartodb_id: 1, value: -3, type: 'odd' },
{ cartodb_id: 2, value: -2, type: 'even'},
{ cartodb_id: 3, value: -1, type: 'odd' },
{ cartodb_id: 4, value: 0, type: 'even' },
]
},
{
zoom: 1,
cartodb_id: 5,
resolution: 50,
expected: [
{ cartodb_id: 5, value: 1, type: 'odd' },
{ cartodb_id: 6, value: 2, type: 'even' },
{ cartodb_id: 7, value: 3, type: 'odd' }
]
}
];
suite.forEach(({ zoom, cartodb_id, resolution, expected }) => {
const description = `should get features for z: ${zoom} cartodb_id: ${cartodb_id}, res: ${resolution}`;
it(description, function (done) {
const mapConfig = createVectorMapConfig([{
type: 'cartodb',
options: {
sql: POINTS_SQL_1,
aggregation: {
threshold: 1,
resolution: resolution
}
}
}]);
const testClient = new TestClient(mapConfig);
const layerId = 0;
const params = {};
testClient.getClusterFeatures(zoom, cartodb_id, layerId, params, (err, body) => {
if (err) {
return done(err);
}
assert.deepStrictEqual(body.rows, expected);
testClient.drain(done);
});
});
});
});
describe('valid aggregation input', function () {
const suite = [
{
zoom: 0,
cartodb_id: 1,
resolution: 1,
aggregation: { columns: ['type'] },
expected: [ { _cdb_feature_count: 1, type: 'odd' } ]
},
{
zoom: 0,
cartodb_id: 2,
resolution: 1,
aggregation: { columns: ['type'] },
expected: [ { _cdb_feature_count: 1, type: 'even' } ]
},
{
zoom: 0,
cartodb_id: 3,
resolution: 1,
aggregation: { columns: ['type'] },
expected: [ { _cdb_feature_count: 1, type: 'odd' } ]
},
{
zoom: 0,
cartodb_id: 4,
resolution: 1,
aggregation: { columns: ['type'] },
expected: [ { _cdb_feature_count: 1, type: 'even' } ]
},
{
zoom: 0,
cartodb_id: 5,
resolution: 1,
aggregation: { columns: ['type'] },
expected: [ { _cdb_feature_count: 1, type: 'odd' } ]
},
{
zoom: 0,
cartodb_id: 6,
resolution: 1,
aggregation: { columns: ['type'] },
expected: [ { _cdb_feature_count: 1, type: 'even' } ]
},
{
zoom: 0,
cartodb_id: 7,
resolution: 1,
aggregation: { columns: ['type'] },
expected: [ { _cdb_feature_count: 1, type: 'odd' } ]
},
{
zoom: 0,
cartodb_id: 1,
resolution: 50,
aggregation: { columns: ['type'] },
expected: [
{ _cdb_feature_count: 2, type: 'even' },
{ _cdb_feature_count: 2, type: 'odd' }
]
},
{
zoom: 0,
cartodb_id: 5,
resolution: 50,
aggregation: { columns: ['type'] },
expected: [
{ _cdb_feature_count: 1, type: 'even' },
{ _cdb_feature_count: 2, type: 'odd' }
]
},
{
zoom: 0,
cartodb_id: 1,
resolution: 1,
aggregation: {
columns: [ 'type' ],
expressions: {
max_value: {
aggregate_function: 'max',
aggregated_column: 'value',
}
}
},
expected: [ { _cdb_feature_count: 1, type: 'odd', max_value: -3 } ]
},
{
zoom: 0,
cartodb_id: 2,
resolution: 1,
aggregation: {
columns: [ 'type' ],
expressions: {
max_value: {
aggregate_function: 'max',
aggregated_column: 'value',
}
}
},
expected: [ { _cdb_feature_count: 1, type: 'even', max_value: -2 } ]
},
{
zoom: 0,
cartodb_id: 3,
resolution: 1,
aggregation: {
columns: [ 'type' ],
expressions: {
max_value: {
aggregate_function: 'max',
aggregated_column: 'value',
}
}
},
expected: [ { _cdb_feature_count: 1, type: 'odd', max_value: -1 } ]
},
{
zoom: 0,
cartodb_id: 4,
resolution: 1,
aggregation: {
columns: [ 'type' ],
expressions: {
max_value: {
aggregate_function: 'max',
aggregated_column: 'value',
}
}
},
expected: [ { _cdb_feature_count: 1, type: 'even', max_value: 0 } ]
},
{
zoom: 0,
cartodb_id: 5,
resolution: 1,
aggregation: {
columns: [ 'type' ],
expressions: {
max_value: {
aggregate_function: 'max',
aggregated_column: 'value',
}
}
},
expected: [ { _cdb_feature_count: 1, type: 'odd', max_value: 1 } ]
},
{
zoom: 0,
cartodb_id: 6,
resolution: 1,
aggregation: {
columns: [ 'type' ],
expressions: {
max_value: {
aggregate_function: 'max',
aggregated_column: 'value',
}
}
},
expected: [ { _cdb_feature_count: 1, type: 'even', max_value: 2 } ]
},
{
zoom: 0,
cartodb_id: 7,
resolution: 1,
aggregation: {
columns: [ 'type' ],
expressions: {
max_value: {
aggregate_function: 'max',
aggregated_column: 'value',
}
}
},
expected: [ { _cdb_feature_count: 1, type: 'odd', max_value: 3 } ]
},
{
zoom: 0,
cartodb_id: 1,
resolution: 50,
aggregation: {
columns: [ 'type' ],
expressions: {
max_value: {
aggregate_function: 'max',
aggregated_column: 'value',
}
}
},
expected: [
{ _cdb_feature_count: 2, type: 'even', max_value: 0 },
{ _cdb_feature_count: 2, type: 'odd', max_value: -1 }
]
},
{
zoom: 0,
cartodb_id: 5,
resolution: 50,
aggregation: {
columns: [ 'type' ],
expressions: {
max_value: {
aggregate_function: 'max',
aggregated_column: 'value',
}
}
},
expected: [
{ _cdb_feature_count: 1, type: 'even', max_value: 2 },
{ _cdb_feature_count: 2, type: 'odd', max_value: 3 }
]
}
];
suite.forEach(({ zoom, cartodb_id, resolution, aggregation, expected }) => {
it('should return features aggregated by type', function (done) {
const mapConfig = createVectorMapConfig([{
type: 'cartodb',
options: {
sql: POINTS_SQL_1,
aggregation: {
threshold: 1,
resolution
}
}
}]);
const testClient = new TestClient(mapConfig);
const layerId = 0;
const params = { aggregation };
testClient.getClusterFeatures(zoom, cartodb_id, layerId, params, (err, body) => {
if (err) {
return done(err);
}
assert.deepStrictEqual(body.rows, expected);
testClient.drain(done);
});
});
});
});
describe('invalid aggregation input', function () {
const expectedColumnsError = {
errors:[ 'Invalid aggregation input, columns should be and array of column names' ],
errors_with_context:[
{
layer: {
index: '0',
type: 'cartodb'
},
message: 'Invalid aggregation input, columns should be and array of column names',
subtype: 'aggregation',
type: 'layer'
}
]
};
const expectedExpressionsError = {
errors:[ 'Invalid aggregation input, expressions should be and object with valid functions' ],
errors_with_context:[
{
layer: {
index: '0',
type: 'cartodb'
},
message: 'Invalid aggregation input, expressions should be and object with valid functions',
subtype: 'aggregation',
type: 'layer'
}
]
};
const invalidFunctionExpressionsError = {
errors:[ 'function wadus(integer) does not exist' ],
errors_with_context:[
{
message: 'function wadus(integer) does not exist',
type: 'unknown'
}
]
};
const invalidColumnExpressionsError = {
errors:[ 'column \"wadus\" does not exist' ],
errors_with_context:[
{
message: 'column \"wadus\" does not exist',
type: 'unknown'
}
]
};
const expectedAggregatedColumnError = {
errors:[ 'Invalid aggregation input, aggregated column should be an string' ],
errors_with_context:[
{
layer: {
index: '0',
type: 'cartodb'
},
message: 'Invalid aggregation input, aggregated column should be an string',
subtype: 'aggregation',
type: 'layer'
}
]
};
const expectedAggregateFunctionError = {
errors:[ 'Invalid aggregation input, aggregate function should be an string' ],
errors_with_context:[
{
layer: {
index: '0',
type: 'cartodb'
},
message: 'Invalid aggregation input, aggregate function should be an string',
subtype: 'aggregation',
type: 'layer'
}
]
};
const suite = [
{
description: 'empty aggregation object should respond with error',
zoom: 0,
cartodb_id: 1,
resolution: 1,
aggregation: {},
expected: expectedColumnsError
},
{
description: 'empty aggregation array should respond with error',
zoom: 0,
cartodb_id: 1,
resolution: 1,
aggregation: [],
expected: expectedColumnsError
},
{
description: 'aggregation as string should respond with error',
zoom: 0,
cartodb_id: 1,
resolution: 1,
aggregation: 'wadus',
expected: expectedColumnsError
},
{
description: 'empty columns array should respond with error',
zoom: 0,
cartodb_id: 1,
resolution: 1,
aggregation: { columns: [] },
expected: expectedColumnsError
},
{
description: 'empty columns object should respond with error',
zoom: 0,
cartodb_id: 1,
resolution: 1,
aggregation: { columns: {} },
expected: expectedColumnsError
},
{
description: 'columns as string should respond with error',
zoom: 0,
cartodb_id: 1,
resolution: 1,
aggregation: { columns: 'wadus' },
expected: expectedColumnsError
},
{
description: 'columns as null should respond with error',
zoom: 0,
cartodb_id: 1,
resolution: 1,
aggregation: { columns: null },
expected: expectedColumnsError
},
{
description: 'empty expressions array should respond with error',
zoom: 0,
cartodb_id: 1,
resolution: 1,
aggregation: { columns: [ 'type' ], expressions: [] },
expected: expectedExpressionsError
},
{
description: 'empty expressions number should respond with error',
zoom: 0,
cartodb_id: 1,
resolution: 1,
aggregation: { columns: [ 'type' ], expressions: 1 },
expected: expectedExpressionsError
},
{
description: 'expressions as string should respond with error',
zoom: 0,
cartodb_id: 1,
resolution: 1,
aggregation: { columns: [ 'type' ], expressions: 'wadus' },
expected: expectedExpressionsError
},
{
description: 'expressions as null should respond with error',
zoom: 0,
cartodb_id: 1,
resolution: 1,
aggregation: { columns: [ 'type' ], expressions: null },
expected: expectedExpressionsError
},
{
description: 'invalid aggregation function should respond with error',
zoom: 0,
cartodb_id: 1,
resolution: 1,
aggregation: {
columns: [ 'type' ],
expressions: {
max_value: {
aggregate_function: 'wadus',
aggregated_column: 'value'
}
}
},
expected: invalidFunctionExpressionsError
},
{
description: 'invalid aggregation column should respond with error',
zoom: 0,
cartodb_id: 1,
resolution: 1,
aggregation: {
columns: [ 'type' ],
expressions: {
max_value: {
aggregate_function: 'max',
aggregated_column: 'wadus'
}
}
},
status: 404,
expected: invalidColumnExpressionsError
},
{
description: 'aggregated column as non string should respond with error',
zoom: 0,
cartodb_id: 1,
resolution: 1,
aggregation: {
columns: [ 'type' ],
expressions: {
max_value: {
aggregate_function: 'max',
aggregated_column: 1
}
}
},
expected: expectedAggregatedColumnError
},
{
description: 'aggregate function as non string should respond with error',
zoom: 0,
cartodb_id: 1,
resolution: 1,
aggregation: {
columns: [ 'type' ],
expressions: {
max_value: {
aggregate_function: 1,
aggregated_column: 'value'
}
}
},
expected: expectedAggregateFunctionError
}
];
suite.forEach(({ description, zoom, cartodb_id, resolution, aggregation, expected, status = 400 }) => {
it(description, function (done) {
const mapConfig = createVectorMapConfig([{
type: 'cartodb',
options: {
sql: POINTS_SQL_1,
aggregation: {
threshold: 1,
resolution
}
}
}]);
const testClient = new TestClient(mapConfig);
const layerId = 0;
const params = {
response: { status },
aggregation
};
testClient.getClusterFeatures(zoom, cartodb_id, layerId, params, (err, body) => {
if (err) {
return done(err);
}
assert.deepStrictEqual(body, expected);
testClient.drain(done);
});
});
});
});
});
+105
View File
@@ -620,6 +620,111 @@ TestClient.prototype.getFeatureAttributes = function(featureId, layerId, params,
);
};
TestClient.prototype.getClusterFeatures = function (zoom, clusterId, layerId, params, callback) {
var self = this;
if (!callback) {
callback = params;
params = {};
}
var extraParams = {};
if (this.apiKey) {
extraParams.api_key = this.apiKey;
}
// if (params && params.filters) {
// extraParams.filters = JSON.stringify(params.filters);
// }
var url = '/api/v1/map';
if (Object.keys(extraParams).length > 0) {
url += '?' + qs.stringify(extraParams);
}
var expectedResponse = params.response || {
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
step(
function createLayergroup() {
var next = this;
assert.response(self.server,
{
url: url,
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(self.mapConfig)
},
{
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
},
function(res, err) {
if (err) {
return next(err);
}
var parsedBody = JSON.parse(res.body);
if (parsedBody.layergroupid) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
}
return next(null, parsedBody.layergroupid);
}
);
},
function getCLusterFeatures (err, layergroupId) {
assert.ifError(err);
var next = this;
let queryParams = '';
if (params.aggregation) {
queryParams = qs.stringify({ aggregation: JSON.stringify(params.aggregation) });
}
url = `/api/v1/map/${layergroupId}/${layerId}/${zoom}/cluster/${clusterId}?${queryParams}`;
assert.response(self.server,
{
url: url,
method: 'GET',
headers: {
host: 'localhost'
}
},
expectedResponse,
function(res, err) {
if (err) {
return next(err);
}
next(null, JSON.parse(res.body));
}
);
},
function finish(err, attributes) {
if (err) {
return callback(err);
}
return callback(null, attributes);
}
);
};
TestClient.prototype.getTile = function(z, x, y, params, callback) {
var self = this;