diff --git a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js index 85eeecb2..fb62d532 100644 --- a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js +++ b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -1,5 +1,7 @@ const queryUtils = require('../../utils/query-utils'); const AggregationMapConfig = require('../../models/aggregation/aggregation-mapconfig'); +const aggregationQuery = require('../../models/aggregation/aggregation-query'); + function MapnikLayerStats () { this._types = { @@ -19,6 +21,9 @@ function columnAggregations(field) { if (field.type === 'date') { // TODO other types too? return ['min', 'max']; } + if (field.type === 'timeDimension') { + return ['min', 'max']; + } return []; } @@ -137,35 +142,53 @@ function _sample(ctx, numRows) { return Promise.resolve(); } -function _columnStats(ctx, columns) { +function _columnStats(ctx, columns, dimensions) { if (!columns) { return Promise.resolve(); } - if (ctx.metaOptions.columnStats) { + if (ctx.metaOptions.columnStats || ctx.metaOptions.dimensions) { let queries = []; let aggr = []; - queries.push(new Promise(resolve => resolve(columns))); // add columns as first result - Object.keys(columns).forEach(name => { - aggr = aggr.concat( - columnAggregations(columns[name]) - .map(fn => `${fn}("${name}") AS "${name}_${fn}"`) - ); - if (columns[name].type === 'string') { - const topN = ctx.metaOptions.columnStats.topCategories || 1024; - const includeNulls = ctx.metaOptions.columnStats.hasOwnProperty('includeNulls') ? - ctx.metaOptions.columnStats.includeNulls : - true; - - // TODO: ctx.metaOptions.columnStats.maxCategories - // => use PG stats to dismiss columns with more distinct values - queries.push( - queryUtils.queryPromise( - ctx.dbConnection, - _getSQL(ctx, sql => queryUtils.getQueryTopCategories(sql, name, topN, includeNulls)) - ).then(res => ({ [name]: { categories: res.rows } })) + if (ctx.metaOptions.columnStats) { + queries.push(new Promise(resolve => resolve(columns))); // add columns as first result + Object.keys(columns).forEach(name => { + aggr = aggr.concat( + columnAggregations(columns[name]) + .map(fn => `${fn}("${name}") AS "${name}_${fn}"`) ); - } - }); + if (columns[name].type === 'string') { + const topN = ctx.metaOptions.columnStats.topCategories || 1024; + const includeNulls = ctx.metaOptions.columnStats.hasOwnProperty('includeNulls') ? + ctx.metaOptions.columnStats.includeNulls : + true; + + // TODO: ctx.metaOptions.columnStats.maxCategories + // => use PG stats to dismiss columns with more distinct values + queries.push( + queryUtils.queryPromise( + ctx.dbConnection, + _getSQL(ctx, sql => queryUtils.getQueryTopCategories(sql, name, topN, includeNulls)) + ).then(res => ({ [name]: { categories: res.rows } })) + ); + } + }); + } + const dimensionsStats = {}; + let dimensionsInfo = {}; + if (ctx.metaOptions.dimensions && dimensions) { + dimensionsInfo = aggregationQuery.infoForOptions({ dimensions }); + Object.keys(dimensionsInfo).forEach(dimName => { + const info = dimensionsInfo[dimName]; + if (info.type === 'timeDimension') { + dimensionsStats[dimName] = { + params: info.params + }; + aggr = aggr.concat( + columnAggregations(info).map(fn => `${fn}(${info.sql}) AS "${dimName}_${fn}"`) + ); + } + }); + } queries.push( queryUtils.queryPromise( ctx.dbConnection, @@ -178,6 +201,17 @@ function _columnStats(ctx, columns) { stats[name][fn] = res.rows[0][`${name}_${fn}`]; }); }); + Object.keys(dimensionsInfo).forEach(name => { + // Temporalily place dimensions info in stats.columns.__dimensions + stats.__dimensions = stats.__dimensions || {}; + stats.__dimensions[name] = stats.__dimensions[name] || Object.assign({}, dimensionsStats[name]); + let type = null; + columnAggregations(dimensionsInfo[name]).forEach(fn => { + type = type || fieldTypeSafe(ctx.dbConnection, res.fields.find(f => f.name === `${name}_${fn}`)); + stats.__dimensions[name][fn] = res.rows[0][`${name}_${fn}`]; + }); + stats.__dimensions[name].type = type; + }); return stats; }) ); @@ -211,19 +245,17 @@ function fieldType(cname) { return tname; } +function fieldTypeSafe(dbConnection, field) { + const cname = dbConnection.typeName(field.dataTypeID); + return cname ? fieldType(cname) : `unknown(${field.dataTypeID})`; +} + // columns are returned as an object { columnName1: { type1: ...}, ..} // for consistency with SQL API function formatResultFields(dbConnection, fields = []) { let nfields = {}; for (let field of fields) { - const cname = dbConnection.typeName(field.dataTypeID); - let tname; - if ( ! cname ) { - tname = 'unknown(' + field.dataTypeID + ')'; - } else { - tname = fieldType(cname); - } - nfields[field.name] = { type: tname }; + nfields[field.name] = { type: fieldTypeSafe(dbConnection, field) }; } return nfields; } @@ -237,7 +269,7 @@ function (layer, dbConnection, callback) { dbConnection, preQuery, aggrQuery, - metaOptions: layer.options.metadata || {} + metaOptions: layer.options.metadata || {}, }; // TODO: could save some queries if queryUtils.getAggregationMetadata() has been used and kept somewhere @@ -248,6 +280,8 @@ function (layer, dbConnection, callback) { // TODO: add support for sample.exclude option by, in that case, forcing the columns query and // passing the results to the sample query function. + const dimensions = (layer.options.aggregation || {}).dimensions; + Promise.all([ _estimatedFeatureCount(ctx).then( ({ estimatedFeatureCount }) => _sample(ctx, estimatedFeatureCount) @@ -256,9 +290,15 @@ function (layer, dbConnection, callback) { _featureCount(ctx), _aggrFeatureCount(ctx), _geometryType(ctx), - _columns(ctx).then(columns => _columnStats(ctx, columns)) + _columns(ctx).then(columns => _columnStats(ctx, columns, dimensions)) ]).then(results => { - callback(null, mergeResults(results)); + results = mergeResults(results); + const dimensions = results.columns && results.columns.__dimensions; + if (dimensions) { + delete results.columns.__dimensions; + results.dimensions = dimensions; + } + callback(null, results); }).catch(error => { callback(error); }); diff --git a/lib/cartodb/models/aggregation/aggregation-query.js b/lib/cartodb/models/aggregation/aggregation-query.js index 792a1bd7..8f8d2de8 100644 --- a/lib/cartodb/models/aggregation/aggregation-query.js +++ b/lib/cartodb/models/aggregation/aggregation-query.js @@ -27,6 +27,16 @@ const templateForOptions = (options) => { return templateFn; }; +function optionsToParams (options) { + return { + sourceQuery: options.query, + res: 256/options.resolution, + columns: options.columns, + dimensions: options.dimensions, + filters: options.filters + }; +} + /** * Generates an aggregation query given the aggregation options: * - query @@ -41,18 +51,23 @@ const templateForOptions = (options) => { * When placement, columns or dimensions are specified, columns are aggregated as requested * (by default only _cdb_feature_count) and with the_geom_webmercator as defined by placement. */ -const queryForOptions = (options) => { - return templateForOptions(options)({ - sourceQuery: options.query, - res: 256/options.resolution, - columns: options.columns, - dimensions: options.dimensions, - filters: options.filters - }); -}; +const queryForOptions = (options) => templateForOptions(options)(optionsToParams(options)); module.exports = queryForOptions; +module.exports.infoForOptions = (options) => { + const params = optionsToParams(options); + const dimensions = {}; + dimensionNamesAndExpressions(params).forEach(([dimensionName, info]) => { + dimensions[dimensionName] = { + sql: info.sql, + params: info.effectiveParams, + type: info.type + }; + }); + return dimensions; +}; + const SUPPORTED_AGGREGATE_FUNCTIONS = { 'count': { sql: (column_name, params) => `count(${params.aggregated_column || '*'})` @@ -123,8 +138,8 @@ const timeDimensionParameters = definition => { return { time: `to_timestamp("${definition.column}")`, timezone: definition.timezone || 'utc', - grouping: definition.group_by, - count: definition.group_by_count || 1, + grouping: definition.grouping, + count: definition.count || 1, starting: definition.starting, format: definition.format }; @@ -139,28 +154,31 @@ const adaptDimensionDefinition = definition => { }; const dimensionExpression = definition => { - if (definition.group_by) { + if (definition.grouping) { // Currently only time dimensions are supported with parameters - return timeDimension(timeDimensionParameters(definition)); + return Object.assign({ type: 'timeDimension' }, timeDimension(timeDimensionParameters(definition))); } else { - return `"${definition.column}"`; + return { sql: `"${definition.column}"` }; } }; -const dimensionNames = (ctx, table) => { +const dimensionNamesAndExpressions = (ctx) => { let dimensions = aggregateDimensions(ctx); - return sep(Object.keys(dimensions).map(dimensionName => { + return Object.keys(dimensions).map(dimensionName => { + const dimension = adaptDimensionDefinition(dimensions[dimensionName]); + const expression = dimensionExpression(dimension); + return [dimensionName, expression]; + }); +}; + +const dimensionNames = (ctx, table) => { + return sep(dimensionNamesAndExpressions(ctx).map(([dimensionName, _]) => { return table ? `${table}."${dimensionName}"` : `"${dimensionName}"`; })); }; const dimensionDefs = ctx => { - let dimensions = aggregateDimensions(ctx); - return sep(Object.keys(dimensions).map(dimensionName => { - const dimension = adaptDimensionDefinition(dimensions[dimensionName]); - const expression = dimensionExpression(dimension); - return `${expression} AS "${dimensionName}"`; - })); + return sep(dimensionNamesAndExpressions(ctx).map(([dimensionName, expression]) => `${expression.sql} AS "${dimensionName}"`)); }; const aggregateFilters = ctx => ctx.filters || {}; diff --git a/lib/cartodb/models/aggregation/time-dimension.js b/lib/cartodb/models/aggregation/time-dimension.js index 1a1909ed..9e897825 100644 --- a/lib/cartodb/models/aggregation/time-dimension.js +++ b/lib/cartodb/models/aggregation/time-dimension.js @@ -24,12 +24,7 @@ function timeExpression(t, tz) { return t; } -// Epoch should be an ISO timestamp literal without time zone -// (it is interpreted as in the defined timzezone for the input time) -// It can be partial, e.g. 'YYYY', 'YYYY-MM', 'YYYY-MM-DDTHH', etc. -// Defaults are applied: YYYY=0001, MM=01, DD=01, HH=00, MM=00, S=00 -// It returns a timestamp without time zone -function epochExpression(epoch) { +function epochWithDefaults(epoch) { /* jshint maxcomplexity:8 */ // goddammit linter, I like this as is!! const format = /^(\d\d\d\d)(?:\-?(\d\d)(?:\-?(\d\d)(?:[T\s]?(\d\d)(?:(\d\d)(?:\:(\d\d))?)?)?)?)?$/; const match = (epoch || '').match(format) || []; @@ -39,7 +34,15 @@ function epochExpression(epoch) { const hour = match[4] || '00'; const minute = match[5] || '00'; const second = match[6] || '00'; - epoch = `${year}-${month}-${day}T${hour}:${minute}:${second}`; + return `${year}-${month}-${day}T${hour}:${minute}:${second}`; +} + +// Epoch should be an ISO timestamp literal without time zone +// (it is interpreted as in the defined timzezone for the input time) +// It can be partial, e.g. 'YYYY', 'YYYY-MM', 'YYYY-MM-DDTHH', etc. +// Defaults are applied: YYYY=0001, MM=01, DD=01, HH=00, MM=00, S=00 +// It returns a timestamp without time zone +function epochExpression(epoch) { return `TIMESTAMP '${epoch}'`; } @@ -182,43 +185,27 @@ function validateParameters(params, checker) { if (missingParams.length) { errors.push(`Missing parameters: ${missingParams.join(', ')}`); } - errors.push(...checker(params)); + const params_errors = checker(params); + errors.push(...params_errors.errors); if (errors.length) { throw new Error(`Invalid time dimension:\n${errors.join("\n")}`); } + return params_errors.params; } const VALID_CYCLIC_GROUPINGS = Object.keys(cyclicParts); const VALID_SERIAL_GROUPINGS = Object.keys(serialParts); const VALID_ISO_GROUPINGS = Object.keys(isoParts); -const MONTH_GROUPING = { - 3: 'quarterOfYear', - 6: 'semesterOfYear', - 4: 'trimesterOfYear' -}; - function cyclicCheckParams(params) { const errors = []; if (!VALID_CYCLIC_GROUPINGS.includes(params.grouping)) { errors.push(`Invalid grouping "${params.grouping}"`); - } else { - if (params.count && params.count > 1) { - let fixed = false; - if (params.grouping === 'monthOfYear') { - const grouping = MONTH_GROUPING[params.count]; - if (grouping) { - params.grouping = grouping; - params.count = 1; - fixed = true; - } - } - if (!fixed) { - errors.push(`Invalid count ${params.count} for cyclic ${params.grouping}`); - } - } } - return errors; + if (params.count && params.count > 1) { + errors.push(`Count ${params.count} not supported for cyclic ${params.grouping}`); + } + return { errors: errors, params: params }; } function serialCheckParams(params) { @@ -226,7 +213,7 @@ function serialCheckParams(params) { if (!VALID_SERIAL_GROUPINGS.includes(params.grouping)) { errors.push(`Invalid grouping "${params.grouping}"`); } - return errors; + return { errors: errors, params: Object.assign({}, params, { starting: epochWithDefaults(params.starting) }) }; } function isoCheckParams(params) { @@ -237,7 +224,7 @@ function isoCheckParams(params) { if (params.starting) { errors.push("Parameter 'starting' not supported for ISO format"); } - return errors; + return { errors: errors, params: params }; } const CLASSIFIERS = { @@ -271,8 +258,8 @@ function classifierFor(params) { function classificationSql(params) { const classifier = classifierFor(params); - validateParameters(params, classifier.checkParams); - return classifier.sqlExpr(params); + params = validateParameters(params, classifier.checkParams); + return { sql: classifier.sqlExpr(params), effectiveParams: params }; } -module.exports = classificationSql; \ No newline at end of file +module.exports = classificationSql;