cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implement aggregation queries.

Browse Source 
Implmented for placements: centroid, point-gird, point-sample.
Aggregated columns not yet implemented (only count).
Aggregation could be made more efficient by using quadkeys
This commit is contained in:
Javier Goizueta
2017-12-11 18:33:06 +01:00
parent 006dd86614
commit 2edcbb4724
6 changed files with 114 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
lib/cartodb/models/aggregation/aggregation-proxy.js
View File

@@ -4,8 +4,9 @@ const RASTER_AGGREGATION = 'RasterAggregation';
const VECTOR_AGGREGATION = 'VectorAggregation';
module.exports = class AggregationProxy {
constructor (mapconfig, { resolution = 256, threshold = 10e5, placement = 'centroid', columns = {}} = {}) {
constructor (mapconfig, query, { resolution = 256, threshold = 1e5, placement = 'centroid', columns = {}} = {}) {
this.mapconfig = mapconfig;
this.query = query;
this.resolution = resolution;
this.threshold = threshold;
this.placement = placement;
@@ -18,10 +19,10 @@ module.exports = class AggregationProxy {
switch (this._getAggregationType()) {
case VECTOR_AGGREGATION:
implementation = new VectorAggregation(this.resolution, this.threshold, this.placement, this.columns);
implementation = new VectorAggregation(this.query, this.resolution, this.threshold, this.placement, this.columns);
break;
case RASTER_AGGREGATION:
implementation = new RasterAggregation(this.resolution, this.threshold, this.placement, this.columns);
implementation = new RasterAggregation(this.query, this.resolution, this.threshold, this.placement, this.columns);
break;
default:
throw new Error('Unsupported aggregation type');
@@ -38,7 +39,7 @@ module.exports = class AggregationProxy {
return RASTER_AGGREGATION;
}
sql (options) {
return this.implementation.sql(options);
sql () {
return this.implementation.sql();
}
};

82
lib/cartodb/models/aggregation/aggregation-templates.js Normal file
View File

@@ -0,0 +1,82 @@
/**
* Returns template function (function that accepts template parameters and returns a string)
*/
module.exports = (options) => {
let templateFn = aggregationQueryTemplates[options.placement];
console.log(options);
if (!templateFn) {
throw new Error("Invalid Aggregation placement: '" + options.placement + "'");
}
return templateFn;
};
// Notes:
// * ${ctx.res*0.00028/256}*!scale_denominator! is equivalent to ${ctx.res/256}*CDB_XYZ_Resolution(CDB_ZoomFromScale(!scale_denominator!))
// * We need to filter spatially using !bbox! to make the queries efficient because the filter added by Mapnik (wrapping the query)
// is only applied after the aggregation.
// * This queries are used for rendering and the_geom is omitted in the results for better performance
const aggregationQueryTemplates = {
'centroid': ctx => `
WITH _cdb_params AS (
SELECT
(${ctx.res*0.00028/256}*!scale_denominator!)::double precision AS res,
!bbox! AS bbox
)
SELECT
row_number() over() AS cartodb_id,
ST_SetSRID(
ST_MakePoint(
AVG(ST_X(_cdb_query.the_geom_webmercator)),
AVG(ST_Y(_cdb_query.the_geom_webmercator))
), 3857
) AS the_geom_webmercator,
count(*) AS _cdb_feature_count
FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params
WHERE _cdb_query.the_geom_webmercator && _cdb_params.bbox
GROUP BY Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res), Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)
`,
'point-grid': ctx => `
WITH _cdb_params AS (
SELECT
(${ctx.res*0.00028/256}*!scale_denominator!)::double precision AS res,
!bbox! AS bbox
),
_cdb_clusters AS (
SELECT
ST_SetSRID(ST_MakePoint(AVG(ST_X(_cdb_query.the_geom_webmercator)), AVG(ST_Y(_cdb_query.the_geom_webmercator))), 3857) AS the_geom_webmercator,
Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res)::int AS _cdb_gx,
Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)::int AS _cdb_gy,
count(*) AS _cdb_feature_count
FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params
WHERE the_geom_webmercator && _cdb_params.bbox
GROUP BY _cdb_gx, _cdb_gy
)
SELECT
ST_SetSRID(ST_MakePoint(_cdb_gx*(res+0.5), _cdb_gy*(res*0.5)), 3857) AS the_geom_webmercator,
_cdb_feature_count
FROM _cdb_clusters, _cdb_params
`,
'point-sample-': ctx => `
WITH _cdb_params AS (
SELECT
(${ctx.res*0.00028/256}*!scale_denominator!)::double precision AS res,
!bbox! AS bbox
), _cdb_clusters AS (
SELECT
MIN(cartodb_id) AS cartodb_id,
count(*) AS _cdb_feature_count
FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params
WHERE _cdb_query.the_geom_webmercator && _cdb_params.bbox
GROUP BY Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res), Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)
) SELECT
_cdb_clusters.cartodb_id,
the_geom, the_geom_webmercator,
_cdb_feature_count
FROM _cdb_clusters INNER JOIN (${ctx.sourceQuery}) _cdb_query on (_cdb_clusters.cartodb_id = _cdb_query.cartodb_id)
`
};

7
lib/cartodb/models/aggregation/base-aggregation.js
View File

@@ -1,4 +1,11 @@
module.exports = class BaseAggregation {
constructor(query, resolution, threshold, placement, columns) {
this.query = query;
this.resolution = resolution;
this.threshold = threshold;
this.placement = placement;
this.columns = columns;
}
sql () {
throw new Error('Unimplemented method');
}

16
lib/cartodb/models/aggregation/raster-aggregation.js
View File

@@ -1,13 +1,15 @@
const BaseAggregation = require('./base-aggregation');
const aggregationTemplate = require('./aggregation-templates');
module.exports = class RasterAggregation extends BaseAggregation {
sql (options) {
return rasterAggregationQueryTemplate({
sourceQuery: options.sql,
res: options.resolution,
columns: options.columns
constructor () {
super(...arguments);
}
sql () {
return aggregationTemplate(this)({
sourceQuery: this.query,
res: this.resolution,
columns: this.columns
});
}
};
const rasterAggregationQueryTemplate = ctx => `/** aggregated query (raster) **/ ${ctx.sourceQuery}`;

16
lib/cartodb/models/aggregation/vector-aggregation.js
View File

@@ -1,13 +1,15 @@
const BaseAggregation = require('./base-aggregation');
const aggregationTemplate = require('./aggregation-templates');
module.exports = class VectorAggregation extends BaseAggregation {
sql (options) {
return vectorAggregationQueryTemplate({
sourceQuery: options.sql,
res: options.resolution,
columns: options.columns
constructor () {
super(...arguments);
}
sql () {
return aggregationTemplate(this)({
sourceQuery: this.query,
res: this.resolution,
columns: this.columns
});
}
};
const vectorAggregationQueryTemplate = ctx => `/** aggregated query (vector) **/ ${ctx.sourceQuery}`;

2
lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js
View File

@@ -145,7 +145,7 @@ module.exports = class AggregationMapConfigAdapter {
return reject(new Error(unsupportedGeometryTypeErrorMessage({ geometryType })));
}
const aggregation = new AggregationProxy(mapConfig, layer.options.aggregation);
const aggregation = new AggregationProxy(mapConfig, layer.options.sql, layer.options.aggregation);
const sqlQueryWrap = layer.options.sql_wrap;
let aggregationSql = aggregation.sql(layer.options);