cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'separate-app-and-controllers-creation' into separate-routers

Browse Source
This commit is contained in:
Daniel García Aubert
2018-04-10 15:59:05 +02:00
parent 61fc15cec0 aa3fb4807b
commit bc45b50290
16 changed files with 1110 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/cartodb/api/auth_api.js
+1 -1
View File
@@ -109,7 +109,7 @@ AuthApi.prototype.authorizedByAPIKey = function(user, res, callback) {
return callback(error);
}
return callback(null, true);
return callback(null, true, apikey);
});
};
lib/cartodb/controllers/map/middlewares/last-updated-time-layergroup.js
+1 -1
View File
@@ -3,7 +3,7 @@ module.exports = function setLastUpdatedTimeToLayergroup () {
const { mapConfigProvider, analysesResults } = res.locals;
const layergroup = res.body;
mapConfigProvider.getAffectedTables((err, affectedTables) => {
mapConfigProvider.createAffectedTables((err, affectedTables) => {
if (err) {
return next(err);
}
lib/cartodb/controllers/template/admin.js
+10 -1
View File
@@ -79,7 +79,7 @@ function authorizedByAPIKey ({ authApi, action, label }) {
return function authorizedByAPIKeyMiddleware (req, res, next) {
const { user } = res.locals;
authApi.authorizedByAPIKey(user, res, (err, authenticated) => {
authApi.authorizedByAPIKey(user, res, (err, authenticated, apikey) => {
if (err) {
return next(err);
}
@@ -91,6 +91,15 @@ function authorizedByAPIKey ({ authApi, action, label }) {
return next(error);
}
if (apikey.type !== 'master') {
const error = new Error('Forbidden');
error.type = 'auth';
error.subtype = 'api-key-does-not-grant-access';
error.http_status = 403;
return next(error);
}
next();
});
};
lib/cartodb/controllers/template/preview.js
+1 -1
View File
@@ -153,7 +153,7 @@ function getStaticImageOptions ({ tablesExtentApi }) {
res.locals.imageOpts = DEFAULT_ZOOM_CENTER;
mapConfigProvider.getAffectedTables((err, affectedTables) => {
mapConfigProvider.createAffectedTables((err, affectedTables) => {
if (err) {
return next();
}
lib/cartodb/models/aggregation/aggregation-mapconfig.js
+28 -10
View File
@@ -4,7 +4,8 @@ const aggregationValidator = require('./aggregation-validator');
const {
createPositiveNumberValidator,
createIncludesValueValidator,
createAggregationColumnsValidator
createAggregationColumnsValidator,
createAggregationFiltersValidator
} = aggregationValidator;
const SubstitutionTokens = require('../../utils/substitution-tokens');
@@ -43,6 +44,18 @@ module.exports = class AggregationMapConfig extends MapConfig {
];
}
static get FILTER_PARAMETERS () {
return [
// TODO: valid combinations of parameters:
// * Except for less/greater params, only one parameter allowed per filter.
// * Any less parameter can be combined with one of the greater paramters. (to define a range)
'less_than', 'less_than_or_equal_to',
'greater_than', 'greater_than_or_equal_to',
'equal', 'not_equal',
'between', 'in', 'not_in'
];
}
static supportsGeometryType(geometryType) {
return AggregationMapConfig.SUPPORTED_GEOMETRY_TYPES.includes(geometryType);
}
@@ -58,11 +71,15 @@ module.exports = class AggregationMapConfig extends MapConfig {
const positiveNumberValidator = createPositiveNumberValidator(this);
const includesValidPlacementsValidator = createIncludesValueValidator(this, AggregationMapConfig.PLACEMENTS);
const aggregationColumnsValidator = createAggregationColumnsValidator(this, AggregationMapConfig.AGGREGATIONS);
const aggregationFiltersValidator = createAggregationFiltersValidator(
this, AggregationMapConfig.FILTER_PARAMETERS
);
validate('resolution', positiveNumberValidator);
validate('placement', includesValidPlacementsValidator);
validate('threshold', positiveNumberValidator);
validate('columns', aggregationColumnsValidator);
validate('filters', aggregationFiltersValidator);
this.user = user;
this.pgConnection = connection;
@@ -77,7 +94,8 @@ module.exports = class AggregationMapConfig extends MapConfig {
threshold = AggregationMapConfig.THRESHOLD,
placement,
columns = {},
dimensions = {}
dimensions = {},
filters = {}
} = this.getAggregation(index);
return aggregationQuery({
@@ -87,6 +105,7 @@ module.exports = class AggregationMapConfig extends MapConfig {
placement,
columns,
dimensions,
filters,
isDefaultAggregation: this._isDefaultLayerAggregation(index)
});
}
@@ -152,21 +171,19 @@ module.exports = class AggregationMapConfig extends MapConfig {
_getLayerAggregationRequiredColumns (index) {
const { columns, dimensions } = this.getAggregation(index);
let finalColumns = ['cartodb_id', '_cdb_feature_count'];
let aggregatedColumns = [];
if (columns) {
aggregatedColumns = Object.keys(columns)
.map(key => columns[key].aggregated_column)
.filter(aggregatedColumn => typeof aggregatedColumn === 'string');
aggregatedColumns = Object.keys(columns);
}
let dimensionsColumns = [];
if (dimensions) {
dimensionsColumns = Object.keys(dimensions)
.map(key => dimensions[key])
.filter(dimension => typeof dimension === 'string');
dimensionsColumns = Object.keys(dimensions);
}
return removeDuplicates(aggregatedColumns.concat(dimensionsColumns));
return removeDuplicates(finalColumns.concat(aggregatedColumns).concat(dimensionsColumns));
}
doesLayerReachThreshold(index, featureCount) {
@@ -220,7 +237,8 @@ module.exports = class AggregationMapConfig extends MapConfig {
_isDefaultAggregation (aggregation) {
return aggregation.placement === undefined &&
aggregation.columns === undefined &&
this._isEmptyParameter(aggregation.dimensions);
this._isEmptyParameter(aggregation.dimensions) &&
this._isEmptyParameter(aggregation.filters);
}
_isEmptyParameter(parameter) {
lib/cartodb/models/aggregation/aggregation-query.js
+124 -11
View File
@@ -42,7 +42,8 @@ const queryForOptions = (options) => templateForOptions(options)({
sourceQuery: options.query,
res: 256/options.resolution,
columns: options.columns,
dimensions: options.dimensions
dimensions: options.dimensions,
filters: options.filters
});
module.exports = queryForOptions;
@@ -93,20 +94,23 @@ const aggregateColumnNames = (ctx, table) => {
return sep(Object.keys(columns));
};
const aggregateExpression = (column_name, column_parameters) => {
const aggregate_function = column_parameters.aggregate_function || 'count';
const aggregate_definition = SUPPORTED_AGGREGATE_FUNCTIONS[aggregate_function];
if (!aggregate_definition) {
throw new Error("Invalid Aggregate function: '" + aggregate_function + "'");
}
return aggregate_definition.sql(column_name, column_parameters);
};
const aggregateColumnDefs = ctx => {
let columns = aggregateColumns(ctx);
return sep(Object.keys(columns).map(column_name => {
const aggregate_function = columns[column_name].aggregate_function || 'count';
const aggregate_definition = SUPPORTED_AGGREGATE_FUNCTIONS[aggregate_function];
if (!aggregate_definition) {
throw new Error("Invalid Aggregate function: '" + aggregate_function + "'");
}
const aggregate_expression = aggregate_definition.sql(column_name, columns[column_name]);
const aggregate_expression = aggregateExpression(column_name, columns[column_name]);
return `${aggregate_expression} AS ${column_name}`;
}));
};
const aggregateDimensions = ctx => ctx.dimensions || {};
const dimensionNames = (ctx, table) => {
@@ -127,6 +131,111 @@ const dimensionDefs = ctx => {
}));
};
const aggregateFilters = ctx => ctx.filters || {};
const filterConditionSQL = (expr, filter) => {
// TODO: validate filter parameters (e.g. cannot have both greater_than and greater_than or equal to)
if (filter) {
if (!Array.isArray(filter)) {
filter = [filter];
}
if (filter.length > 0) {
return filter.map(f => filterSingleConditionSQL(expr, f)).join(' OR ');
}
}
};
const filterSingleConditionSQL = (expr, filter) => {
let cond;
Object.keys(FILTERS).some(f => {
cond = FILTERS[f](expr, filter);
return cond;
});
return cond;
};
const sqlQ = (value) => {
if (isFinite(value)) {
return String(value);
}
return `'${value}'`; // TODO: escape single quotes! (by doubling them)
};
/* jshint eqeqeq: false */
/* x != null is used to check for both null and undefined; triple !== wouldn't do the trick */
const FILTERS = {
between: (expr, filter) => {
const lo = filter.greater_than_or_equal_to, hi = filter.less_than_or_equal_to;
if (lo != null && hi != null) {
return `(${expr} BETWEEN ${sqlQ(lo)} AND ${sqlQ(hi)})`;
}
},
in: (expr, filter) => {
if (filter.in != null) {
return `(${expr} IN (${filter.in.map(v => sqlQ(v)).join(',')}))`;
}
},
notin: (expr, filter) => {
if (filter.not_in != null) {
return `(${expr} NOT IN (${filter.not_in.map(v => sqlQ(v)).join(',')}))`;
}
},
equal: (expr, filter) => {
if (filter.equal != null) {
return `(${expr} = ${sqlQ(filter.equal)})`;
}
},
not_equal: (expr, filter) => {
if (filter.not_equal != null) {
return `(${expr} <> ${sqlQ(filter.not_equal)})`;
}
},
range: (expr, filter) => {
let conds = [];
if (filter.greater_than_or_equal_to != null) {
conds.push(`(${expr} >= ${sqlQ(filter.greater_than_or_equal_to)})`);
}
if (filter.greater_than != null) {
conds.push(`(${expr} > ${sqlQ(filter.greater_than)})`);
}
if (filter.less_than_or_equal_to != null) {
conds.push(`(${expr} <= ${sqlQ(filter.less_than_or_equal_to)})`);
}
if (filter.less_than != null) {
conds.push(`(${expr} < ${sqlQ(filter.less_than)})`);
}
if (conds.length > 0) {
return conds.join(' AND ');
}
}
};
const filterConditions = ctx => {
let columns = aggregateColumns(ctx);
let dimensions = aggregateDimensions(ctx);
let filters = aggregateFilters(ctx);
return Object.keys(filters).map(filtered_column => {
let filtered_expr;
if (columns[filtered_column]) {
filtered_expr = aggregateExpression(filtered_column, columns[filtered_column]);
}
else if (dimensions[filtered_column]) {
filtered_expr = dimensions[filtered_column];
}
if (!filtered_expr) {
throw new Error("Invalid filtered column: '" + filtered_column + "'");
}
return filterConditionSQL(filtered_expr, filters[filtered_column]);
}).join(' AND ');
};
const havingClause = ctx => {
let cond = filterConditions(ctx);
return cond ? `HAVING ${cond}` : '';
};
// SQL expression to compute the aggregation resolution (grid cell size).
// This is equivalent to `${256/ctx.res}*CDB_XYZ_Resolution(CDB_ZoomFromScale(!scale_denominator!))`
// This is defined by the ctx.res parameter, which is the number of grid cells per tile linear dimension
@@ -181,7 +290,7 @@ const aggregationQueryTemplates = {
!bbox! AS bbox
)
SELECT
row_number() over() AS cartodb_id,
MIN(_cdb_query.cartodb_id) AS cartodb_id,
ST_SetSRID(
ST_MakePoint(
AVG(ST_X(_cdb_query.the_geom_webmercator)),
@@ -196,6 +305,7 @@ const aggregationQueryTemplates = {
Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res),
Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)
${dimensionNames(ctx)}
${havingClause(ctx)}
`,
'point-grid': ctx => `
@@ -207,6 +317,7 @@ const aggregationQueryTemplates = {
),
_cdb_clusters AS (
SELECT
MIN(_cdb_query.cartodb_id) AS cartodb_id,
Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res)::int AS _cdb_gx,
Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)::int AS _cdb_gy
${dimensionDefs(ctx)}
@@ -214,9 +325,10 @@ const aggregationQueryTemplates = {
FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params
WHERE the_geom_webmercator && _cdb_params.bbox
GROUP BY _cdb_gx, _cdb_gy ${dimensionNames(ctx)}
${havingClause(ctx)}
)
SELECT
row_number() over() AS cartodb_id,
_cdb_clusters.cartodb_id AS cartodb_id,
ST_SetSRID(ST_MakePoint((_cdb_gx+0.5)*res, (_cdb_gy+0.5)*res), 3857) AS the_geom_webmercator
${dimensionNames(ctx)}
${aggregateColumnNames(ctx)}
@@ -241,11 +353,12 @@ const aggregationQueryTemplates = {
Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res),
Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)
${dimensionNames(ctx)}
${havingClause(ctx)}
)
SELECT
_cdb_clusters.cartodb_id,
the_geom, the_geom_webmercator
${dimensionNames(ctx, '_cdb_query')}
${dimensionNames(ctx, '_cdb_clusters')}
${aggregateColumnNames(ctx, '_cdb_clusters')}
FROM
_cdb_clusters INNER JOIN (${ctx.sourceQuery}) _cdb_query
lib/cartodb/models/aggregation/aggregation-validator.js
+32
View File
@@ -42,6 +42,38 @@ module.exports.createAggregationColumnsValidator = function (mapconfig, validAgg
};
};
module.exports.createAggregationFiltersValidator = function (mapconfig, validParameters) {
return function validateAggregationFilters (value, key, index) {
const dims = mapconfig.getAggregation(index).dimensions || {};
const cols = mapconfig.getAggregation(index).columns || {};
const validKeys = Object.keys(dims).concat(Object.keys(cols));
Object.keys(value).forEach((filteredName) => {
// filteredName must be the name of either an aggregated column or a dimension in the same layer
if (!validKeys.includes(filteredName)) {
const message = `Invalid filtered column: ${filteredName}`;
throw createLayerError(message, mapconfig, index);
}
// The filter parameters must be valid
let filters = value[filteredName];
// a single filter or an array of filters (to be OR-combined) are accepted
if (!Array.isArray(filters)) {
filters = [filters];
}
filters.forEach(params => {
Object.keys(params).forEach(paramName => {
if (!validParameters.includes(paramName)) {
const message = `Invalid filter parameter name: ${paramName}`;
throw createLayerError(message, mapconfig, index);
}
});
// TODO: check parameter value (params[paramName]) to be of the correct type
});
// TODO: if multiple parameters within params check the combination is valid,
// i.e. one of the *less* parameters and one of the *greater* parameters.
});
};
};
function createAggregationColumnNamesValidator(mapconfig) {
return function validateAggregationColumnNames (value, key, index) {
Object.keys(value).forEach((columnName) => {
lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js
+19 -6
View File
@@ -58,7 +58,7 @@ CreateLayergroupMapConfigProvider.prototype.filter = MapStoreMapConfigProvider.p
CreateLayergroupMapConfigProvider.prototype.createKey = MapStoreMapConfigProvider.prototype.createKey;
CreateLayergroupMapConfigProvider.prototype.getAffectedTables = function (callback) {
CreateLayergroupMapConfigProvider.prototype.createAffectedTables = function (callback) {
this.getMapConfig((err, mapConfig) => {
if (err) {
return callback(err);
@@ -67,11 +67,6 @@ CreateLayergroupMapConfigProvider.prototype.getAffectedTables = function (callba
const { dbname } = this.params;
const token = mapConfig.id();
if (this.affectedTablesCache.hasAffectedTables(dbname, token)) {
const affectedTables = this.affectedTablesCache.get(dbname, token);
return callback(null, affectedTables);
}
const queries = [];
this.mapConfig.getLayers().forEach(layer => {
@@ -106,3 +101,21 @@ CreateLayergroupMapConfigProvider.prototype.getAffectedTables = function (callba
});
});
};
CreateLayergroupMapConfigProvider.prototype.getAffectedTables = function (callback) {
this.getMapConfig((err, mapConfig) => {
if (err) {
return callback(err);
}
const { dbname } = this.params;
const token = mapConfig.id();
if (this.affectedTablesCache.hasAffectedTables(dbname, token)) {
const affectedTables = this.affectedTablesCache.get(dbname, token);
return callback(null, affectedTables);
}
return this.createAffectedTables(callback);
});
};
lib/cartodb/models/mapconfig/provider/map-store-provider.js
+19 -7
View File
@@ -89,7 +89,7 @@ MapStoreMapConfigProvider.prototype.createKey = function(base) {
return (base) ? baseKeyTpl(tplValues) : rendererKeyTpl(tplValues);
};
MapStoreMapConfigProvider.prototype.getAffectedTables = function(callback) {
MapStoreMapConfigProvider.prototype.createAffectedTables = function(callback) {
this.getMapConfig((err, mapConfig) => {
if (err) {
return callback(err);
@@ -98,12 +98,6 @@ MapStoreMapConfigProvider.prototype.getAffectedTables = function(callback) {
const { dbname } = this.params;
const token = mapConfig.id();
if (this.affectedTablesCache.hasAffectedTables(dbname, token)) {
const affectedTables = this.affectedTablesCache.get(dbname, token);
return callback(null, affectedTables);
}
const queries = [];
mapConfig.getLayers().forEach(layer => {
@@ -138,3 +132,21 @@ MapStoreMapConfigProvider.prototype.getAffectedTables = function(callback) {
});
});
};
MapStoreMapConfigProvider.prototype.getAffectedTables = function (callback) {
this.getMapConfig((err, mapConfig) => {
if (err) {
return callback(err);
}
const { dbname } = this.params;
const token = mapConfig.id();
if (this.affectedTablesCache.hasAffectedTables(dbname, token)) {
const affectedTables = this.affectedTablesCache.get(dbname, token);
return callback(null, affectedTables);
}
return this.createAffectedTables(callback);
});
};
lib/cartodb/models/mapconfig/provider/named-map-provider.js
+19 -6
View File
@@ -262,7 +262,7 @@ NamedMapMapConfigProvider.prototype.getTemplateName = function() {
return this.templateName;
};
NamedMapMapConfigProvider.prototype.getAffectedTables = function(callback) {
NamedMapMapConfigProvider.prototype.createAffectedTables = function(callback) {
this.getMapConfig((err, mapConfig) => {
if (err) {
return callback(err);
@@ -271,11 +271,6 @@ NamedMapMapConfigProvider.prototype.getAffectedTables = function(callback) {
const { dbname } = this.rendererParams;
const token = mapConfig.id();
if (this.affectedTablesCache.hasAffectedTables(dbname, token)) {
const affectedTables = this.affectedTablesCache.get(dbname, token);
return callback(null, affectedTables);
}
const queries = [];
mapConfig.getLayers().forEach(layer => {
@@ -310,3 +305,21 @@ NamedMapMapConfigProvider.prototype.getAffectedTables = function(callback) {
});
});
};
NamedMapMapConfigProvider.prototype.getAffectedTables = function (callback) {
this.getMapConfig((err, mapConfig) => {
if (err) {
return callback(err);
}
const { dbname } = this.params;
const token = mapConfig.id();
if (this.affectedTablesCache.hasAffectedTables(dbname, token)) {
const affectedTables = this.affectedTablesCache.get(dbname, token);
return callback(null, affectedTables);
}
return this.createAffectedTables(callback);
});
};