cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Dimensions metadata

Browse Source
This commit is contained in:
Javier Goizueta
2018-10-04 19:50:14 +02:00
parent c9786ee3f6
commit f841f65a1e
3 changed files with 136 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/cartodb/backends/layer-stats/mapnik-layer-stats.js
+74 -34
View File
@@ -1,5 +1,7 @@
const queryUtils = require('../../utils/query-utils');
const AggregationMapConfig = require('../../models/aggregation/aggregation-mapconfig');
const aggregationQuery = require('../../models/aggregation/aggregation-query');
function MapnikLayerStats () {
this._types = {
@@ -19,6 +21,9 @@ function columnAggregations(field) {
if (field.type === 'date') { // TODO other types too?
return ['min', 'max'];
}
if (field.type === 'timeDimension') {
return ['min', 'max'];
}
return [];
}
@@ -137,35 +142,53 @@ function _sample(ctx, numRows) {
return Promise.resolve();
}
function _columnStats(ctx, columns) {
function _columnStats(ctx, columns, dimensions) {
if (!columns) {
return Promise.resolve();
}
if (ctx.metaOptions.columnStats) {
if (ctx.metaOptions.columnStats || ctx.metaOptions.dimensions) {
let queries = [];
let aggr = [];
queries.push(new Promise(resolve => resolve(columns))); // add columns as first result
Object.keys(columns).forEach(name => {
aggr = aggr.concat(
columnAggregations(columns[name])
.map(fn => `${fn}("${name}") AS "${name}_${fn}"`)
);
if (columns[name].type === 'string') {
const topN = ctx.metaOptions.columnStats.topCategories || 1024;
const includeNulls = ctx.metaOptions.columnStats.hasOwnProperty('includeNulls') ?
ctx.metaOptions.columnStats.includeNulls :
true;
// TODO: ctx.metaOptions.columnStats.maxCategories
// => use PG stats to dismiss columns with more distinct values
queries.push(
queryUtils.queryPromise(
ctx.dbConnection,
_getSQL(ctx, sql => queryUtils.getQueryTopCategories(sql, name, topN, includeNulls))
).then(res => ({ [name]: { categories: res.rows } }))
if (ctx.metaOptions.columnStats) {
queries.push(new Promise(resolve => resolve(columns))); // add columns as first result
Object.keys(columns).forEach(name => {
aggr = aggr.concat(
columnAggregations(columns[name])
.map(fn => `${fn}("${name}") AS "${name}_${fn}"`)
);
}
});
if (columns[name].type === 'string') {
const topN = ctx.metaOptions.columnStats.topCategories || 1024;
const includeNulls = ctx.metaOptions.columnStats.hasOwnProperty('includeNulls') ?
ctx.metaOptions.columnStats.includeNulls :
true;
// TODO: ctx.metaOptions.columnStats.maxCategories
// => use PG stats to dismiss columns with more distinct values
queries.push(
queryUtils.queryPromise(
ctx.dbConnection,
_getSQL(ctx, sql => queryUtils.getQueryTopCategories(sql, name, topN, includeNulls))
).then(res => ({ [name]: { categories: res.rows } }))
);
}
});
}
const dimensionsStats = {};
let dimensionsInfo = {};
if (ctx.metaOptions.dimensions && dimensions) {
dimensionsInfo = aggregationQuery.infoForOptions({ dimensions });
Object.keys(dimensionsInfo).forEach(dimName => {
const info = dimensionsInfo[dimName];
if (info.type === 'timeDimension') {
dimensionsStats[dimName] = {
params: info.params
};
aggr = aggr.concat(
columnAggregations(info).map(fn => `${fn}(${info.sql}) AS "${dimName}_${fn}"`)
);
}
});
}
queries.push(
queryUtils.queryPromise(
ctx.dbConnection,
@@ -178,6 +201,17 @@ function _columnStats(ctx, columns) {
stats[name][fn] = res.rows[0][`${name}_${fn}`];
});
});
Object.keys(dimensionsInfo).forEach(name => {
// Temporalily place dimensions info in stats.columns.__dimensions
stats.__dimensions = stats.__dimensions || {};
stats.__dimensions[name] = stats.__dimensions[name] || Object.assign({}, dimensionsStats[name]);
let type = null;
columnAggregations(dimensionsInfo[name]).forEach(fn => {
type = type || fieldTypeSafe(ctx.dbConnection, res.fields.find(f => f.name === `${name}_${fn}`));
stats.__dimensions[name][fn] = res.rows[0][`${name}_${fn}`];
});
stats.__dimensions[name].type = type;
});
return stats;
})
);
@@ -211,19 +245,17 @@ function fieldType(cname) {
return tname;
}
function fieldTypeSafe(dbConnection, field) {
const cname = dbConnection.typeName(field.dataTypeID);
return cname ? fieldType(cname) : `unknown(${field.dataTypeID})`;
}
// columns are returned as an object { columnName1: { type1: ...}, ..}
// for consistency with SQL API
function formatResultFields(dbConnection, fields = []) {
let nfields = {};
for (let field of fields) {
const cname = dbConnection.typeName(field.dataTypeID);
let tname;
if ( ! cname ) {
tname = 'unknown(' + field.dataTypeID + ')';
} else {
tname = fieldType(cname);
}
nfields[field.name] = { type: tname };
nfields[field.name] = { type: fieldTypeSafe(dbConnection, field) };
}
return nfields;
}
@@ -237,7 +269,7 @@ function (layer, dbConnection, callback) {
dbConnection,
preQuery,
aggrQuery,
metaOptions: layer.options.metadata || {}
metaOptions: layer.options.metadata || {},
};
// TODO: could save some queries if queryUtils.getAggregationMetadata() has been used and kept somewhere
@@ -248,6 +280,8 @@ function (layer, dbConnection, callback) {
// TODO: add support for sample.exclude option by, in that case, forcing the columns query and
// passing the results to the sample query function.
const dimensions = (layer.options.aggregation || {}).dimensions;
Promise.all([
_estimatedFeatureCount(ctx).then(
({ estimatedFeatureCount }) => _sample(ctx, estimatedFeatureCount)
@@ -256,9 +290,15 @@ function (layer, dbConnection, callback) {
_featureCount(ctx),
_aggrFeatureCount(ctx),
_geometryType(ctx),
_columns(ctx).then(columns => _columnStats(ctx, columns))
_columns(ctx).then(columns => _columnStats(ctx, columns, dimensions))
]).then(results => {
callback(null, mergeResults(results));
results = mergeResults(results);
const dimensions = results.columns && results.columns.__dimensions;
if (dimensions) {
delete results.columns.__dimensions;
results.dimensions = dimensions;
}
callback(null, results);
}).catch(error => {
callback(error);
});
lib/cartodb/models/aggregation/aggregation-query.js
+40 -22
View File
@@ -27,6 +27,16 @@ const templateForOptions = (options) => {
return templateFn;
};
function optionsToParams (options) {
return {
sourceQuery: options.query,
res: 256/options.resolution,
columns: options.columns,
dimensions: options.dimensions,
filters: options.filters
};
}
/**
* Generates an aggregation query given the aggregation options:
* - query
@@ -41,18 +51,23 @@ const templateForOptions = (options) => {
* When placement, columns or dimensions are specified, columns are aggregated as requested
* (by default only _cdb_feature_count) and with the_geom_webmercator as defined by placement.
*/
const queryForOptions = (options) => {
return templateForOptions(options)({
sourceQuery: options.query,
res: 256/options.resolution,
columns: options.columns,
dimensions: options.dimensions,
filters: options.filters
});
};
const queryForOptions = (options) => templateForOptions(options)(optionsToParams(options));
module.exports = queryForOptions;
module.exports.infoForOptions = (options) => {
const params = optionsToParams(options);
const dimensions = {};
dimensionNamesAndExpressions(params).forEach(([dimensionName, info]) => {
dimensions[dimensionName] = {
sql: info.sql,
params: info.effectiveParams,
type: info.type
};
});
return dimensions;
};
const SUPPORTED_AGGREGATE_FUNCTIONS = {
'count': {
sql: (column_name, params) => `count(${params.aggregated_column || '*'})`
@@ -123,8 +138,8 @@ const timeDimensionParameters = definition => {
return {
time: `to_timestamp("${definition.column}")`,
timezone: definition.timezone || 'utc',
grouping: definition.group_by,
count: definition.group_by_count || 1,
grouping: definition.grouping,
count: definition.count || 1,
starting: definition.starting,
format: definition.format
};
@@ -139,28 +154,31 @@ const adaptDimensionDefinition = definition => {
};
const dimensionExpression = definition => {
if (definition.group_by) {
if (definition.grouping) {
// Currently only time dimensions are supported with parameters
return timeDimension(timeDimensionParameters(definition));
return Object.assign({ type: 'timeDimension' }, timeDimension(timeDimensionParameters(definition)));
} else {
return `"${definition.column}"`;
return { sql: `"${definition.column}"` };
}
};
const dimensionNames = (ctx, table) => {
const dimensionNamesAndExpressions = (ctx) => {
let dimensions = aggregateDimensions(ctx);
return sep(Object.keys(dimensions).map(dimensionName => {
return Object.keys(dimensions).map(dimensionName => {
const dimension = adaptDimensionDefinition(dimensions[dimensionName]);
const expression = dimensionExpression(dimension);
return [dimensionName, expression];
});
};
const dimensionNames = (ctx, table) => {
return sep(dimensionNamesAndExpressions(ctx).map(([dimensionName, _]) => {
return table ? `${table}."${dimensionName}"` : `"${dimensionName}"`;
}));
};
const dimensionDefs = ctx => {
let dimensions = aggregateDimensions(ctx);
return sep(Object.keys(dimensions).map(dimensionName => {
const dimension = adaptDimensionDefinition(dimensions[dimensionName]);
const expression = dimensionExpression(dimension);
return `${expression} AS "${dimensionName}"`;
}));
return sep(dimensionNamesAndExpressions(ctx).map(([dimensionName, expression]) => `${expression.sql} AS "${dimensionName}"`));
};
const aggregateFilters = ctx => ctx.filters || {};
lib/cartodb/models/aggregation/time-dimension.js
+22 -35
View File
@@ -24,12 +24,7 @@ function timeExpression(t, tz) {
return t;
}
// Epoch should be an ISO timestamp literal without time zone
// (it is interpreted as in the defined timzezone for the input time)
// It can be partial, e.g. 'YYYY', 'YYYY-MM', 'YYYY-MM-DDTHH', etc.
// Defaults are applied: YYYY=0001, MM=01, DD=01, HH=00, MM=00, S=00
// It returns a timestamp without time zone
function epochExpression(epoch) {
function epochWithDefaults(epoch) {
/* jshint maxcomplexity:8 */ // goddammit linter, I like this as is!!
const format = /^(\d\d\d\d)(?:\-?(\d\d)(?:\-?(\d\d)(?:[T\s]?(\d\d)(?:(\d\d)(?:\:(\d\d))?)?)?)?)?$/;
const match = (epoch || '').match(format) || [];
@@ -39,7 +34,15 @@ function epochExpression(epoch) {
const hour = match[4] || '00';
const minute = match[5] || '00';
const second = match[6] || '00';
epoch = `${year}-${month}-${day}T${hour}:${minute}:${second}`;
return `${year}-${month}-${day}T${hour}:${minute}:${second}`;
}
// Epoch should be an ISO timestamp literal without time zone
// (it is interpreted as in the defined timzezone for the input time)
// It can be partial, e.g. 'YYYY', 'YYYY-MM', 'YYYY-MM-DDTHH', etc.
// Defaults are applied: YYYY=0001, MM=01, DD=01, HH=00, MM=00, S=00
// It returns a timestamp without time zone
function epochExpression(epoch) {
return `TIMESTAMP '${epoch}'`;
}
@@ -182,43 +185,27 @@ function validateParameters(params, checker) {
if (missingParams.length) {
errors.push(`Missing parameters: ${missingParams.join(', ')}`);
}
errors.push(...checker(params));
const params_errors = checker(params);
errors.push(...params_errors.errors);
if (errors.length) {
throw new Error(`Invalid time dimension:\n${errors.join("\n")}`);
}
return params_errors.params;
}
const VALID_CYCLIC_GROUPINGS = Object.keys(cyclicParts);
const VALID_SERIAL_GROUPINGS = Object.keys(serialParts);
const VALID_ISO_GROUPINGS = Object.keys(isoParts);
const MONTH_GROUPING = {
3: 'quarterOfYear',
6: 'semesterOfYear',
4: 'trimesterOfYear'
};
function cyclicCheckParams(params) {
const errors = [];
if (!VALID_CYCLIC_GROUPINGS.includes(params.grouping)) {
errors.push(`Invalid grouping "${params.grouping}"`);
} else {
if (params.count && params.count > 1) {
let fixed = false;
if (params.grouping === 'monthOfYear') {
const grouping = MONTH_GROUPING[params.count];
if (grouping) {
params.grouping = grouping;
params.count = 1;
fixed = true;
}
}
if (!fixed) {
errors.push(`Invalid count ${params.count} for cyclic ${params.grouping}`);
}
}
}
return errors;
if (params.count && params.count > 1) {
errors.push(`Count ${params.count} not supported for cyclic ${params.grouping}`);
}
return { errors: errors, params: params };
}
function serialCheckParams(params) {
@@ -226,7 +213,7 @@ function serialCheckParams(params) {
if (!VALID_SERIAL_GROUPINGS.includes(params.grouping)) {
errors.push(`Invalid grouping "${params.grouping}"`);
}
return errors;
return { errors: errors, params: Object.assign({}, params, { starting: epochWithDefaults(params.starting) }) };
}
function isoCheckParams(params) {
@@ -237,7 +224,7 @@ function isoCheckParams(params) {
if (params.starting) {
errors.push("Parameter 'starting' not supported for ISO format");
}
return errors;
return { errors: errors, params: params };
}
const CLASSIFIERS = {
@@ -271,8 +258,8 @@ function classifierFor(params) {
function classificationSql(params) {
const classifier = classifierFor(params);
validateParameters(params, classifier.checkParams);
return classifier.sqlExpr(params);
params = validateParameters(params, classifier.checkParams);
return { sql: classifier.sqlExpr(params), effectiveParams: params };
}
module.exports = classificationSql;
module.exports = classificationSql;