Be able to aggregate by a field
This commit is contained in:
Daniel García Aubert
@@ -43,7 +43,7 @@ module.exports = class AggregatedFeaturesLayergroupController {
|
||||
authorize(this.authBackend),
|
||||
dbConnSetup(this.pgConnection),
|
||||
rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.ATTRIBUTES),
|
||||
cleanUpQueryParams(),
|
||||
cleanUpQueryParams([ 'aggregation' ]),
|
||||
createMapStoreMapConfigProvider(
|
||||
this.mapStore,
|
||||
this.userLimitsBackend,
|
||||
@@ -67,11 +67,13 @@ function getClusteredFeatures (clusterBackend) {
|
||||
const { user, token } = res.locals;
|
||||
const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals;
|
||||
const { layer, z: zoom, clusterId } = req.params;
|
||||
const { aggregation } = req.query;
|
||||
|
||||
const params = {
|
||||
user, token,
|
||||
dbuser, dbname, dbpassword, dbhost, dbport,
|
||||
layer, zoom, clusterId
|
||||
layer, zoom, clusterId,
|
||||
aggregation
|
||||
};
|
||||
|
||||
clusterBackend.getClusterFeatures(mapConfigProvider, params, (err, features, stats = {}) => {
|
||||
|
||||
@@ -38,6 +38,7 @@ module.exports = class ClusterBackend {
|
||||
return callback(error);
|
||||
}
|
||||
|
||||
const { aggregation } = params;
|
||||
const query = layer.options.sql_raw;
|
||||
const resolution = layer.options.aggregation.resolution || 1;
|
||||
|
||||
@@ -48,7 +49,7 @@ module.exports = class ClusterBackend {
|
||||
|
||||
const { zoom, clusterId } = params;
|
||||
|
||||
getClusterFeatures(pg, zoom, clusterId, columns, query, resolution, (err, features) => {
|
||||
getClusterFeatures(pg, zoom, clusterId, columns, query, resolution, aggregation, (err, features) => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
@@ -85,8 +86,8 @@ function getColumnsName (pg, query, callback) {
|
||||
}, true);
|
||||
}
|
||||
|
||||
function getClusterFeatures (pg, zoom, clusterId, columns, query, resolution, callback) {
|
||||
const sql = clusterFeaturesQuery({
|
||||
function getClusterFeatures (pg, zoom, clusterId, columns, query, resolution, aggregation, callback) {
|
||||
let sql = clusterFeaturesQuery({
|
||||
zoom: zoom,
|
||||
id: clusterId,
|
||||
query: query,
|
||||
@@ -94,6 +95,17 @@ function getClusterFeatures (pg, zoom, clusterId, columns, query, resolution, ca
|
||||
columns: columns
|
||||
});
|
||||
|
||||
if (aggregation !== undefined) {
|
||||
aggregation = JSON.parse(aggregation);
|
||||
const { columns = [], expresions = [] } = aggregation;
|
||||
|
||||
sql = aggregationQuery({
|
||||
columns,
|
||||
query: sql,
|
||||
expresions
|
||||
});
|
||||
}
|
||||
|
||||
debug('> getClusterFeatures:', sql);
|
||||
|
||||
pg.query(sql, (err, data) => {
|
||||
@@ -133,15 +145,17 @@ const clusterFeaturesQuery = ctx => `
|
||||
) __cdb_non_geoms_query
|
||||
`;
|
||||
|
||||
// SQL expression to compute the aggregation resolution (grid cell size).
|
||||
// This is defined by the ctx.res parameter, which is the number of grid cells per tile linear dimension
|
||||
// (i.e. each tile is divided into ctx.res*ctx.res cells).
|
||||
// We limit the the minimum resolution to avoid division by zero problems. The limit used is
|
||||
// the pixel size of zoom level 30 (i.e. 1/2*(30+8) of the full earth web-mercator extent), which is about 0.15 mm.
|
||||
// Computing this using !scale_denominator!, !pixel_width! or !pixel_height! produces
|
||||
// inaccurate results due to rounding present in those values.
|
||||
const gridResolution = ctx => {
|
||||
const minimumResolution = 2*Math.PI*6378137/Math.pow(2,38);
|
||||
const pixelSize = `CDB_XYZ_Resolution(${ctx.zoom})`;
|
||||
return `GREATEST(${256/ctx.res}*${pixelSize}, ${minimumResolution})::double precision`;
|
||||
};
|
||||
|
||||
const aggregationQuery = ctx => `
|
||||
SELECT
|
||||
count(1) as _cdb_feature_count
|
||||
${ctx.columns.length ? `,${ctx.columns.join(', ')}` : ''}
|
||||
${ctx.expresions.length ? `,${ctx.expresions.join(', ')}` : ''}
|
||||
FROM (${ctx.query}) __cdb_aggregation
|
||||
${ctx.columns.length ? `GROUP BY ${ctx.columns.join(', ')}` : ''}
|
||||
`;
|
||||
|
||||
+147
-40
@@ -10,7 +10,11 @@ const POINTS_SQL_1 = `
|
||||
x + 4 as cartodb_id,
|
||||
st_setsrid(st_makepoint(x*10, x*10), 4326) as the_geom,
|
||||
st_transform(st_setsrid(st_makepoint(x*10, x*10), 4326), 3857) as the_geom_webmercator,
|
||||
x as value
|
||||
x as value,
|
||||
CASE
|
||||
WHEN x % 2 = 0 THEN 'even'
|
||||
ELSE 'odd'
|
||||
END AS type
|
||||
from generate_series(-3, 3) x
|
||||
`;
|
||||
|
||||
@@ -58,14 +62,14 @@ describe('cluster', function () {
|
||||
}
|
||||
|
||||
assert.deepStrictEqual(body, {
|
||||
errors:[ 'Map d725a568ab961af8197d311eececb83a has no aggregation defined for layer 0' ],
|
||||
errors:[ 'Map c502fc8fc1cb0d5e412db3deabffeee5 has no aggregation defined for layer 0' ],
|
||||
errors_with_context:[
|
||||
{
|
||||
layer: {
|
||||
index: '0',
|
||||
type: 'cartodb'
|
||||
},
|
||||
message: 'Map d725a568ab961af8197d311eececb83a has no aggregation defined for layer 0',
|
||||
message: 'Map c502fc8fc1cb0d5e412db3deabffeee5 has no aggregation defined for layer 0',
|
||||
subtype: 'aggregation',
|
||||
type: 'layer'
|
||||
}
|
||||
@@ -100,14 +104,14 @@ describe('cluster', function () {
|
||||
}
|
||||
|
||||
assert.deepStrictEqual(body, {
|
||||
errors:[ 'Map 3a09728f8c08444820336ea9983ce92b has no aggregation defined for layer 0' ],
|
||||
errors:[ 'Map 18792467ae296929d04e32dfe7f81a80 has no aggregation defined for layer 0' ],
|
||||
errors_with_context:[
|
||||
{
|
||||
layer: {
|
||||
index: '0',
|
||||
type: 'cartodb'
|
||||
},
|
||||
message: 'Map 3a09728f8c08444820336ea9983ce92b has no aggregation defined for layer 0',
|
||||
message: 'Map 18792467ae296929d04e32dfe7f81a80 has no aggregation defined for layer 0',
|
||||
subtype: 'aggregation',
|
||||
type: 'layer'
|
||||
}
|
||||
@@ -125,95 +129,95 @@ describe('cluster', function () {
|
||||
zoom: 0,
|
||||
cartodb_id: 1,
|
||||
resolution: 0.5,
|
||||
expected: [ { cartodb_id: 1, value: -3 } ]
|
||||
expected: [ { cartodb_id: 1, value: -3, type: 'odd' } ]
|
||||
},
|
||||
{
|
||||
zoom: 0,
|
||||
cartodb_id: 2,
|
||||
resolution: 0.5,
|
||||
expected: [ { cartodb_id: 2, value: -2 } ]
|
||||
expected: [ { cartodb_id: 2, value: -2, type: 'even' } ]
|
||||
},
|
||||
{
|
||||
zoom: 0,
|
||||
cartodb_id: 3,
|
||||
resolution: 0.5,
|
||||
expected: [ { cartodb_id: 3, value: -1 } ]
|
||||
expected: [ { cartodb_id: 3, value: -1, type: 'odd' } ]
|
||||
},
|
||||
{
|
||||
zoom: 0,
|
||||
cartodb_id: 4,
|
||||
resolution: 0.5,
|
||||
expected: [ { cartodb_id: 4, value: 0 } ]
|
||||
expected: [ { cartodb_id: 4, value: 0, type: 'even' } ]
|
||||
},
|
||||
{
|
||||
zoom: 0,
|
||||
cartodb_id: 5,
|
||||
resolution: 0.5,
|
||||
expected: [ { cartodb_id: 5, value: 1 } ]
|
||||
expected: [ { cartodb_id: 5, value: 1, type: 'odd' } ]
|
||||
},
|
||||
{
|
||||
zoom: 0,
|
||||
cartodb_id: 6,
|
||||
resolution: 0.5,
|
||||
expected: [ { cartodb_id: 6, value: 2 } ]
|
||||
expected: [ { cartodb_id: 6, value: 2, type: 'even' } ]
|
||||
},
|
||||
{
|
||||
zoom: 0,
|
||||
cartodb_id: 7,
|
||||
resolution: 0.5,
|
||||
expected: [ { cartodb_id: 7, value: 3 } ]
|
||||
expected: [ { cartodb_id: 7, value: 3, type: 'odd' } ]
|
||||
},
|
||||
{
|
||||
zoom: 0,
|
||||
cartodb_id: 1,
|
||||
resolution: 1,
|
||||
expected: [ { cartodb_id: 1, value: -3 } ]
|
||||
expected: [ { cartodb_id: 1, value: -3, type: 'odd' } ]
|
||||
},
|
||||
{
|
||||
zoom: 0,
|
||||
cartodb_id: 2,
|
||||
resolution: 1,
|
||||
expected: [ { cartodb_id: 2, value: -2 } ]
|
||||
expected: [ { cartodb_id: 2, value: -2, type: 'even' } ]
|
||||
},
|
||||
{
|
||||
zoom: 0,
|
||||
cartodb_id: 3,
|
||||
resolution: 1,
|
||||
expected: [ { cartodb_id: 3, value: -1 } ]
|
||||
expected: [ { cartodb_id: 3, value: -1, type: 'odd' } ]
|
||||
},
|
||||
{
|
||||
zoom: 0,
|
||||
cartodb_id: 4,
|
||||
resolution: 1,
|
||||
expected: [ { cartodb_id: 4, value: 0 } ]
|
||||
expected: [ { cartodb_id: 4, value: 0, type: 'even' } ]
|
||||
},
|
||||
{
|
||||
zoom: 0,
|
||||
cartodb_id: 5,
|
||||
resolution: 1,
|
||||
expected: [ { cartodb_id: 5, value: 1 } ]
|
||||
expected: [ { cartodb_id: 5, value: 1, type: 'odd' } ]
|
||||
},
|
||||
{
|
||||
zoom: 0,
|
||||
cartodb_id: 6,
|
||||
resolution: 1,
|
||||
expected: [ { cartodb_id: 6, value: 2 } ]
|
||||
expected: [ { cartodb_id: 6, value: 2, type: 'even' } ]
|
||||
},
|
||||
{
|
||||
zoom: 0,
|
||||
cartodb_id: 7,
|
||||
resolution: 1,
|
||||
expected: [ { cartodb_id: 7, value: 3 } ]
|
||||
expected: [ { cartodb_id: 7, value: 3, type: 'odd' } ]
|
||||
},
|
||||
{
|
||||
zoom: 0,
|
||||
cartodb_id: 1,
|
||||
resolution: 50,
|
||||
expected: [
|
||||
{ cartodb_id: 1, value: -3 },
|
||||
{ cartodb_id: 2, value: -2 },
|
||||
{ cartodb_id: 3, value: -1 },
|
||||
{ cartodb_id: 4, value: 0 },
|
||||
{ cartodb_id: 1, value: -3, type: 'odd' },
|
||||
{ cartodb_id: 2, value: -2, type: 'even' },
|
||||
{ cartodb_id: 3, value: -1, type: 'odd' },
|
||||
{ cartodb_id: 4, value: 0, type: 'even' },
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -221,62 +225,62 @@ describe('cluster', function () {
|
||||
cartodb_id: 5,
|
||||
resolution: 50,
|
||||
expected: [
|
||||
{ cartodb_id: 5, value: 1 },
|
||||
{ cartodb_id: 6, value: 2 },
|
||||
{ cartodb_id: 7, value: 3 }
|
||||
{ cartodb_id: 5, value: 1, type: 'odd' },
|
||||
{ cartodb_id: 6, value: 2, type: 'even' },
|
||||
{ cartodb_id: 7, value: 3, type: 'odd' }
|
||||
]
|
||||
},
|
||||
{
|
||||
zoom: 1,
|
||||
cartodb_id: 1,
|
||||
resolution: 1,
|
||||
expected: [ { cartodb_id: 1, value: -3 } ]
|
||||
expected: [ { cartodb_id: 1, value: -3, type: 'odd' } ]
|
||||
},
|
||||
{
|
||||
zoom: 1,
|
||||
cartodb_id: 2,
|
||||
resolution: 1,
|
||||
expected: [ { cartodb_id: 2, value: -2 } ]
|
||||
expected: [ { cartodb_id: 2, value: -2, type: 'even' } ]
|
||||
},
|
||||
{
|
||||
zoom: 1,
|
||||
cartodb_id: 3,
|
||||
resolution: 1,
|
||||
expected: [ { cartodb_id: 3, value: -1 } ]
|
||||
expected: [ { cartodb_id: 3, value: -1, type: 'odd' } ]
|
||||
},
|
||||
{
|
||||
zoom: 1,
|
||||
cartodb_id: 4,
|
||||
resolution: 1,
|
||||
expected: [ { cartodb_id: 4, value: 0 } ]
|
||||
expected: [ { cartodb_id: 4, value: 0, type: 'even' } ]
|
||||
},
|
||||
{
|
||||
zoom: 1,
|
||||
cartodb_id: 5,
|
||||
resolution: 1,
|
||||
expected: [ { cartodb_id: 5, value: 1 } ]
|
||||
expected: [ { cartodb_id: 5, value: 1, type: 'odd' } ]
|
||||
},
|
||||
{
|
||||
zoom: 1,
|
||||
cartodb_id: 6,
|
||||
resolution: 1,
|
||||
expected: [ { cartodb_id: 6, value: 2 } ]
|
||||
expected: [ { cartodb_id: 6, value: 2, type: 'even' } ]
|
||||
},
|
||||
{
|
||||
zoom: 1,
|
||||
cartodb_id: 7,
|
||||
resolution: 1,
|
||||
expected: [ { cartodb_id: 7, value: 3 } ]
|
||||
expected: [ { cartodb_id: 7, value: 3, type: 'odd' } ]
|
||||
},
|
||||
{
|
||||
zoom: 1,
|
||||
cartodb_id: 1,
|
||||
resolution: 50,
|
||||
expected: [
|
||||
{ cartodb_id: 1, value: -3 },
|
||||
{ cartodb_id: 2, value: -2 },
|
||||
{ cartodb_id: 3, value: -1 },
|
||||
{ cartodb_id: 4, value: 0 },
|
||||
{ cartodb_id: 1, value: -3, type: 'odd' },
|
||||
{ cartodb_id: 2, value: -2, type: 'even'},
|
||||
{ cartodb_id: 3, value: -1, type: 'odd' },
|
||||
{ cartodb_id: 4, value: 0, type: 'even' },
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -284,9 +288,9 @@ describe('cluster', function () {
|
||||
cartodb_id: 5,
|
||||
resolution: 50,
|
||||
expected: [
|
||||
{ cartodb_id: 5, value: 1 },
|
||||
{ cartodb_id: 6, value: 2 },
|
||||
{ cartodb_id: 7, value: 3 }
|
||||
{ cartodb_id: 5, value: 1, type: 'odd' },
|
||||
{ cartodb_id: 6, value: 2, type: 'even' },
|
||||
{ cartodb_id: 7, value: 3, type: 'odd' }
|
||||
]
|
||||
}
|
||||
];
|
||||
@@ -319,4 +323,107 @@ describe('cluster', function () {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('map-config w/o aggregation', function () {
|
||||
const suite = [
|
||||
{
|
||||
zoom: 0,
|
||||
cartodb_id: 1,
|
||||
resolution: 1,
|
||||
aggregation: { columns: ['type'] },
|
||||
expected: [ { _cdb_feature_count: 1, type: 'odd' } ]
|
||||
},
|
||||
{
|
||||
zoom: 0,
|
||||
cartodb_id: 2,
|
||||
resolution: 1,
|
||||
aggregation: { columns: ['type'] },
|
||||
expected: [ { _cdb_feature_count: 1, type: 'even' } ]
|
||||
},
|
||||
{
|
||||
zoom: 0,
|
||||
cartodb_id: 3,
|
||||
resolution: 1,
|
||||
aggregation: { columns: ['type'] },
|
||||
expected: [ { _cdb_feature_count: 1, type: 'odd' } ]
|
||||
},
|
||||
{
|
||||
zoom: 0,
|
||||
cartodb_id: 4,
|
||||
resolution: 1,
|
||||
aggregation: { columns: ['type'] },
|
||||
expected: [ { _cdb_feature_count: 1, type: 'even' } ]
|
||||
},
|
||||
{
|
||||
zoom: 0,
|
||||
cartodb_id: 5,
|
||||
resolution: 1,
|
||||
aggregation: { columns: ['type'] },
|
||||
expected: [ { _cdb_feature_count: 1, type: 'odd' } ]
|
||||
},
|
||||
{
|
||||
zoom: 0,
|
||||
cartodb_id: 6,
|
||||
resolution: 1,
|
||||
aggregation: { columns: ['type'] },
|
||||
expected: [ { _cdb_feature_count: 1, type: 'even' } ]
|
||||
},
|
||||
{
|
||||
zoom: 0,
|
||||
cartodb_id: 7,
|
||||
resolution: 1,
|
||||
aggregation: { columns: ['type'] },
|
||||
expected: [ { _cdb_feature_count: 1, type: 'odd' } ]
|
||||
},
|
||||
{
|
||||
zoom: 0,
|
||||
cartodb_id: 1,
|
||||
resolution: 50,
|
||||
aggregation: { columns: ['type'] },
|
||||
expected: [
|
||||
{ _cdb_feature_count: 2, type: 'even' },
|
||||
{ _cdb_feature_count: 2, type: 'odd' }
|
||||
]
|
||||
},
|
||||
{
|
||||
zoom: 0,
|
||||
cartodb_id: 5,
|
||||
resolution: 50,
|
||||
aggregation: { columns: ['type'] },
|
||||
expected: [
|
||||
{ _cdb_feature_count: 1, type: 'even' },
|
||||
{ _cdb_feature_count: 2, type: 'odd' }
|
||||
]
|
||||
}
|
||||
];
|
||||
|
||||
|
||||
suite.forEach(({ zoom, cartodb_id, resolution, aggregation, expected }) => {
|
||||
it('should return features aggregated by type', function (done) {
|
||||
const mapConfig = createVectorMapConfig([{
|
||||
type: 'cartodb',
|
||||
options: {
|
||||
sql: POINTS_SQL_1,
|
||||
aggregation: {
|
||||
threshold: 1,
|
||||
resolution
|
||||
}
|
||||
}
|
||||
}]);
|
||||
const testClient = new TestClient(mapConfig);
|
||||
const layerId = 0;
|
||||
const params = { aggregation };
|
||||
|
||||
testClient.getClusterFeatures(zoom, cartodb_id, layerId, params, (err, body) => {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
|
||||
assert.deepStrictEqual(body.rows, expected);
|
||||
|
||||
testClient.drain(done);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -690,7 +690,12 @@ TestClient.prototype.getClusterFeatures = function (zoom, clusterId, layerId, pa
|
||||
|
||||
var next = this;
|
||||
|
||||
url = '/api/v1/map/' + layergroupId + '/' + layerId + '/' + zoom + '/cluster/' + clusterId;
|
||||
let queryParams = '';
|
||||
if (params.aggregation) {
|
||||
queryParams = qs.stringify({ aggregation: JSON.stringify(params.aggregation) });
|
||||
}
|
||||
|
||||
url = `/api/v1/map/${layergroupId}/${layerId}/${zoom}/cluster/${clusterId}?${queryParams}`;
|
||||
|
||||
assert.response(self.server,
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user