diff --git a/NEWS.md b/NEWS.md index 44723922..02c66777 100644 --- a/NEWS.md +++ b/NEWS.md @@ -31,11 +31,13 @@ New features: - moment: 2.22.1 - nock: 9.2.6 - strftime: 0.10.0 +- Optional instantiation metadata stats (https://github.com/CartoDB/Windshaft-cartodb/pull/952) Bug Fixes: - Validates tile coordinates (z/x/y) from request params to be a valid integer value. - Static maps fails for unsupported formats - Handling errors extracting the column type on dataviews +- Fix `meta.stats.estimatedFeatureCount` for aggregations and queries with tokens ## 6.1.0 Released 2018-04-16 diff --git a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js index c060f964..d1e292de 100644 --- a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js +++ b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -1,4 +1,27 @@ var queryUtils = require('../../utils/query-utils'); +const AggregationMapConfig = require('../../models/aggregation/aggregation-mapconfig'); +var SubstitutionTokens = require('../../utils/substitution-tokens'); + +// Instantiate a query with tokens for a given zoom level +function queryForZoom(sql, zoom, singleTile=false) { + const tileRes = 256; + const wmSize = 6378137.0*2*Math.PI; + const nTiles = Math.pow(2, zoom); + const tileSize = wmSize / nTiles; + const resolution = tileSize / tileRes; + const scaleDenominator = resolution / 0.00028; + const x0 = -wmSize/2, y0 = -wmSize/2; + let bbox = `ST_MakeEnvelope(${x0}, ${y0}, ${x0+wmSize}, ${y0+wmSize})`; + if (singleTile) { + bbox = `ST_MakeEnvelope(${x0}, ${y0}, ${x0 + tileSize}, ${y0 + tileSize})`; + } + return SubstitutionTokens.replace(sql, { + bbox: bbox, + scale_denominator: scaleDenominator, + pixel_width: resolution, + pixel_height: resolution + }); +} function MapnikLayerStats () { this._types = { @@ -11,17 +34,278 @@ MapnikLayerStats.prototype.is = function (type) { return this._types[type] ? this._types[type] : false; }; +function queryPromise(dbConnection, query, adaptResults, errorHandler) { + return new Promise(function(resolve, reject) { + dbConnection.query(query, function (err, res) { + if (err) { + if (errorHandler) { + resolve(errorHandler(err)); + } + else { + reject(err); + } + } + else { + resolve(adaptResults(res)); + } + }); + + }); +} + +function columnAggregations(field) { + if (field.type === 'number') { + return ['min', 'max', 'avg', 'sum']; + } + if (field.type === 'date') { // TODO other types too? + return ['min', 'max']; + } + return []; +} + +function _getSQL(ctx, query, type='pre', zoom=0) { + let sql; + if (type === 'pre') { + sql = ctx.preQuery; + } + else { + sql = ctx.aggrQuery; + } + sql = queryForZoom(sql, zoom || 0); + return query(sql); +} + +function _estimatedFeatureCount(ctx) { + return queryPromise( + ctx.dbConnection, + _getSQL(ctx, queryUtils.getQueryRowEstimation), + res => ({ estimatedFeatureCount: res.rows[0].rows }), + () => ({ estimatedFeatureCount: -1 }) + ); +} + +function _featureCount(ctx) { + if (ctx.metaOptions.featureCount) { + // TODO: if ctx.metaOptions.columnStats we can combine this with column stats query + return queryPromise( + ctx.dbConnection, + _getSQL(ctx, queryUtils.getQueryActualRowCount), + res => ({ featureCount: res.rows[0].rows }) + ); + } + return Promise.resolve(); +} + +function _aggrFeatureCount(ctx) { + if (ctx.metaOptions.hasOwnProperty('aggrFeatureCount')) { + // We expect as zoom level as the value of aggrFeatureCount + // TODO: it'd be nice to admit an array of zoom levels to + // return metadata for multiple levels. + return queryPromise( + ctx.dbConnection, + _getSQL(ctx, queryUtils.getQueryActualRowCount, 'post', ctx.metaOptions.aggrFeatureCount), + res => ({ aggrFeatureCount: res.rows[0].rows }) + ); + } + return Promise.resolve(); +} + +function _geometryType(ctx) { + if (ctx.metaOptions.geometryType) { + const geometryColumn = AggregationMapConfig.getAggregationGeometryColumn(); + return queryPromise( + ctx.dbConnection, + _getSQL(ctx, sql => queryUtils.getQueryGeometryType(sql, geometryColumn)), + res => ({ geometryType: res.rows[0].geom_type }) + ); + } + return Promise.resolve(); +} + +function _columns(ctx) { + if (ctx.metaOptions.columns || ctx.metaOptions.columnStats) { + // note: post-aggregation columns are in layer.options.columns when aggregation is present + return queryPromise( + ctx.dbConnection, + _getSQL(ctx, sql => queryUtils.getQueryLimited(sql, 0)), + res => formatResultFields(ctx.dbConnection, res.fields) + ); + } + return Promise.resolve(); +} + +// combine a list of results merging the properties of all the objects +// undefined results are admitted and ignored +function mergeResults(results) { + if (results) { + if (results.length === 0) { + return {}; + } + return results.reduce((a, b) => { + if (a === undefined) { + return b; + } + if (b === undefined) { + return a; + } + return Object.assign({}, a, b); + }); + } +} + +// deeper (1 level) combination of a list of objects: +// mergeColumns([{ col1: { a: 1 }, col2: { a: 2 } }, { col1: { b: 3 } }]) => { col1: { a: 1, b: 3 }, col2: { a: 2 } } +function mergeColumns(results) { + if (results) { + if (results.length === 0) { + return {}; + } + return results.reduce((a, b) => { + let c = Object.assign({}, b || {}, a || {}); + Object.keys(c).forEach(key => { + if (b.hasOwnProperty(key)) { + c[key] = Object.assign(c[key], b[key]); + } + }); + return c; + }); + } +} + +function _sample(ctx, numRows) { + if (ctx.metaOptions.sample) { + const sampleProb = Math.min(ctx.metaOptions.sample / numRows, 1); + // We'll use a safety limit just in case numRows is a bad estimate + const limit = Math.ceil(ctx.metaOptions.sample * 1.5); + return queryPromise( + ctx.dbConnection, + _getSQL(ctx, sql => queryUtils.getQuerySample(sql, sampleProb, limit)), + res => ({ sample: res.rows }) + ); + } + return Promise.resolve(); +} + +function _columnStats(ctx, columns) { + if (!columns) { + return Promise.resolve(); + } + if (ctx.metaOptions.columnStats) { + let queries = []; + let aggr = []; + queries.push(new Promise(resolve => resolve(columns))); // add columns as first result + Object.keys(columns).forEach(name => { + aggr = aggr.concat( + columnAggregations(columns[name]) + .map(fn => `${fn}(${name}) AS ${name}_${fn}`) + ); + if (columns[name].type === 'string') { + const topN = ctx.metaOptions.columnStats.topCategories || 1024; + // TODO: ctx.metaOptions.columnStats.maxCategories + // => use PG stats to dismiss columns with more distinct values + queries.push( + queryPromise( + ctx.dbConnection, + _getSQL(ctx, sql => queryUtils.getQueryTopCategories(sql, name, topN)), + res => ({ [name]: { categories: res.rows } }) + ) + ); + } + }); + queries.push( + queryPromise( + ctx.dbConnection, + _getSQL(ctx, sql => `SELECT ${aggr.join(',')} FROM (${sql}) AS __cdb_query`), + res => { + let stats = {}; + Object.keys(columns).forEach(name => { + stats[name] = {}; + columnAggregations(columns[name]).forEach(fn => { + stats[name][fn] = res.rows[0][`${name}_${fn}`]; + }); + }); + return stats; + } + ) + ); + return Promise.all(queries).then(results => ({ columns: mergeColumns(results) })); + } + return Promise.resolve({ columns }); +} + +// This is adapted from SQL API: +function fieldType(cname) { + let tname; + switch (true) { + case /bool/.test(cname): + tname = 'boolean'; + break; + case /int|float|numeric/.test(cname): + tname = 'number'; + break; + case /text|char|unknown/.test(cname): + tname = 'string'; + break; + case /date|time/.test(cname): + tname = 'date'; + break; + default: + tname = cname; + } + if ( tname && cname.match(/^_/) ) { + tname += '[]'; + } + return tname; +} + +// columns are returned as an object { columnName1: { type1: ...}, ..} +// for consistency with SQL API +function formatResultFields(dbConnection, fields = []) { + let nfields = {}; + for (let field of fields) { + const cname = dbConnection.typeName(field.dataTypeID); + let tname; + if ( ! cname ) { + tname = 'unknown(' + field.dataTypeID + ')'; + } else { + tname = fieldType(cname); + } + nfields[field.name] = { type: tname }; + } + return nfields; +} + MapnikLayerStats.prototype.getStats = function (layer, dbConnection, callback) { - var queryRowCountSql = queryUtils.getQueryRowCount(layer.options.sql); - // This query would gather stats for postgresql table if not exists - dbConnection.query(queryRowCountSql, function (err, res) { - if (err) { - return callback(null, {estimatedFeatureCount: -1}); - } else { - // We decided that the relation is 1 row == 1 feature - return callback(null, {estimatedFeatureCount: res.rows[0].rows}); - } + let aggrQuery = layer.options.sql; + let preQuery = layer.options.sql_raw || aggrQuery; + + let ctx = { + dbConnection, + preQuery, + aggrQuery, + metaOptions: layer.options.metadata || {} + }; + + // TODO: could save some queries if queryUtils.getAggregationMetadata() has been used and kept somewhere + // we would set queries.results.estimatedFeatureCount and queries.results.geometryType + // (if metaOptions.geometryType) from it. + + // TODO: compute _sample with _featureCount when available + + Promise.all([ + _estimatedFeatureCount(ctx).then( + ({ estimatedFeatureCount }) => _sample(ctx, estimatedFeatureCount) + .then(sampleResults => mergeResults([sampleResults, { estimatedFeatureCount }])) + ), + _featureCount(ctx), + _aggrFeatureCount(ctx), + _geometryType(ctx), + _columns(ctx).then(columns => _columnStats(ctx, columns)) + ]).then(results => { + callback(null, mergeResults(results)); + }).catch(error => { + callback(error); }); }; diff --git a/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js b/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js index 67c5eac4..9752ab57 100644 --- a/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js @@ -101,6 +101,12 @@ module.exports = class AggregationMapConfigAdapter { aggregationSql = sqlQueryWrap.replace(/<%=\s*sql\s*%>/g, aggregationSql); } + if (!layer.options.sql_raw) { + // if sql_wrap is present, the original query should already be + // in sql_raw (with sql being the wrapped query); + // otherwise we keep the now the original query in sql_raw + layer.options.sql_raw = layer.options.sql; + } layer.options.sql = aggregationSql; mapConfig.getLayerAggregationColumns(index, (err, columns) => { @@ -159,6 +165,7 @@ module.exports = class AggregationMapConfigAdapter { } _getAggregationMetadata (mapConfig, layer, adapted) { + // also: pre-aggr query, columns, ... if (!adapted) { return { png: false, mvt: false }; } diff --git a/lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js b/lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js index 2bd30152..31c4f7c4 100644 --- a/lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js @@ -117,7 +117,9 @@ TurboCartoAdapter.prototype._parseCartoCss = function (username, params, layer, var layerSql = layer.options.sql; var layerRawSql = layer.options.sql_raw; - if (SubstitutionTokens.hasTokens(layerSql) && layerRawSql) { + if (SubstitutionTokens.hasTokens(layerSql) && layerRawSql && layer.options.sql_wrap) { + // For wrapped queries we'll derive the tokens from the data extent + // instead of the whole Earth/root tile. var self = this; var tokensQuery = tokensQueryTpl({_sql: layerRawSql}); return pg.query(tokensQuery, function(err, resultSet) { diff --git a/lib/cartodb/utils/query-utils.js b/lib/cartodb/utils/query-utils.js index c1b56276..22d99eef 100644 --- a/lib/cartodb/utils/query-utils.js +++ b/lib/cartodb/utils/query-utils.js @@ -21,10 +21,15 @@ module.exports.extractTableNames = function extractTableNames(query) { ].join(''); }; +module.exports.getQueryActualRowCount = function (query) { + return `select COUNT(*) AS rows FROM (${query}) AS __cdb_query`; +}; + function getQueryRowEstimation(query) { return 'select CDB_EstimateRowCount($windshaft$' + query + '$windshaft$) as rows'; } -module.exports.getQueryRowCount = getQueryRowEstimation; + +module.exports.getQueryRowEstimation = getQueryRowEstimation; module.exports.getAggregationMetadata = ctx => ` WITH @@ -71,3 +76,81 @@ module.exports.countNaNs = function countNaNs(ctx) { `sum(CASE WHEN (${ctx.column} = 'NaN'::float) THEN 1 ELSE 0 END)` }`; }; + +module.exports.getQueryTopCategories = function(query, column, topN, includeNulls=false) { + const where = includeNulls ? '' : `WHERE ${column} IS NOT NULL`; + return ` + SELECT ${column} AS category, COUNT(*) AS frequency + FROM (${query}) AS __cdb_query + ${where} + GROUP BY ${column} ORDER BY 2 DESC + LIMIT ${topN} + `; +}; + +module.exports.getQuerySample = function(query, sampleProb, limit = null, randomSeed = 0.5) { + const singleTable = simpleQueryTable(query); + if (singleTable) { + return getTableSample(singleTable.table, singleTable.columns, sampleProb, limit, randomSeed); + } + const limitClause = limit ? `LIMIT ${limit}` : ''; + return ` + WITH __cdb_rndseed AS ( + SELECT setseed(${randomSeed}) + ) + SELECT * + FROM (${query}) AS __cdb_query + WHERE random() < ${sampleProb} + ${limitClause} + `; +}; + +function getTableSample(table, columns, sampleProb, limit = null, randomSeed = 0.5) { + const limitClause = limit ? `LIMIT ${limit}` : ''; + sampleProb *= 100; + randomSeed *= Math.pow(2, 31) -1; + return ` + SELECT ${columns} FROM ${table} TABLESAMPLE BERNOULLI (${sampleProb}) REPEATABLE (${randomSeed}) ${limitClause} + `; +} + +function simpleQueryTable(sql) { + const basicQuery = + /\s*SELECT\s+([\*a-z0-9_,\s]+?)\s+FROM\s+((\"[^"]+\"|[a-z0-9_]+)\.)?(\"[^"]+\"|[a-z0-9_]+)\s*;?\s*/i; + const unwrappedQuery = new RegExp("^"+basicQuery.source+"$", 'i'); + // queries for named maps are wrapped like this: + var wrappedQuery = new RegExp( + "^\\s*SELECT\\s+\\*\\s+FROM\\s+\\(" + + basicQuery.source + + "\\)\\s+AS\\s+wrapped_query\\s+WHERE\\s+\\d+=1\\s*$", + 'i' + ); + let match = sql.match(unwrappedQuery); + if (!match) { + match = sql.match(wrappedQuery); + } + if (match) { + const columns = match[1]; + const schema = match[3]; + const table = match[4]; + return { table: schema ? `${schema}.${table}` : table, columns }; + } + return false; +} + +module.exports.getQueryGeometryType = function(query, geometryColumn) { + return ` + SELECT ST_GeometryType(${geometryColumn}) AS geom_type + FROM (${query}) AS __cdb_query + WHERE ${geometryColumn} IS NOT NULL + LIMIT 1 + `; +}; + +module.exports.getQueryLimited = function(query, limit=0) { + return ` + SELECT * + FROM (${query}) AS __cdb_query + LIMIT ${limit} + `; +}; diff --git a/test/acceptance/aggregation.js b/test/acceptance/aggregation.js index 6658eb81..96401f15 100644 --- a/test/acceptance/aggregation.js +++ b/test/acceptance/aggregation.js @@ -100,6 +100,18 @@ describe('aggregation', function () { from generate_series(-3, 3) x `; + const POINTS_SQL_PAIRS = ` + -- Generate pairs of near points + select + x + 4 as cartodb_id, + st_setsrid(st_makepoint(Floor(x/2)*10 + x/1000.0, Floor(x/2)*10 + x/1000.0), 4326) as the_geom, + st_transform( + st_setsrid(st_makepoint(Floor(x/2)*10 + x/1000.0, Floor(x/2)*10 + x/1000.0),4326), + 3857) as the_geom_webmercator, + x as value + from generate_series(-6, 6) x + `; + function createVectorMapConfig (layers = [ { type: 'cartodb', @@ -130,6 +142,7 @@ describe('aggregation', function () { before(function () { serverOptions.renderer.mvt.usePostGIS = usePostGIS; + this.layerStatsConfig = global.environment.enabledFeatures.layerStats; }); after(function (){ @@ -138,6 +151,7 @@ describe('aggregation', function () { afterEach(function (done) { this.testClient.drain(done); + global.environment.enabledFeatures.layerStats = this.layerStatsConfig; }); it('should return a layergroup indicating the mapconfig was aggregated', function (done) { @@ -2173,6 +2187,168 @@ describe('aggregation', function () { }); }); + ['default', 'centroid', 'point-sample', 'point-grid'].forEach(placement => { + it(`default pre-aggregation stats are available with ${placement} aggregation`, function (done) { + global.environment.enabledFeatures.layerStats = true; + this.mapConfig = { + version: '1.6.0', + buffersize: { 'mvt': 0 }, + layers: [ + { + type: 'cartodb', + + options: { + sql: POINTS_SQL_PAIRS, + resolution: 1, + aggregation: { + threshold: 1 + } + } + } + ] + }; + if (placement !== 'default') { + this.mapConfig.layers[0].options.aggregation.placement = placement; + } + + this.testClient = new TestClient(this.mapConfig); + this.testClient.getLayergroup((err, body) => { + if (err) { + return done(err); + } + + assert.equal(typeof body.metadata, 'object'); + assert.ok(Array.isArray(body.metadata.layers)); + assert.ok(body.metadata.layers[0].meta.aggregation.mvt); + assert.ok(body.metadata.layers[0].meta.stats.estimatedFeatureCount > 0); + + done(); + }); + }); + + it(`on demand post-aggregation stats are available with ${placement} aggregation`, function (done) { + global.environment.enabledFeatures.layerStats = true; + this.mapConfig = { + version: '1.6.0', + buffersize: { 'mvt': 0 }, + layers: [ + { + type: 'cartodb', + + options: { + sql: POINTS_SQL_PAIRS, + resolution: 1, + aggregation: { + threshold: 1 + }, + metadata: { + aggrFeatureCount: 10 + } + } + } + ] + }; + if (placement !== 'default') { + this.mapConfig.layers[0].options.aggregation.placement = placement; + } + + this.testClient = new TestClient(this.mapConfig); + this.testClient.getLayergroup((err, body) => { + if (err) { + return done(err); + } + + assert.equal(typeof body.metadata, 'object'); + assert.ok(Array.isArray(body.metadata.layers)); + assert.ok(body.metadata.layers[0].meta.aggregation.mvt); + assert.equal(body.metadata.layers[0].meta.stats.aggrFeatureCount, 13); + + done(); + }); + }); + + it(`post-aggregation count adapts to zoom level with ${placement} aggregation`, function (done) { + global.environment.enabledFeatures.layerStats = true; + this.mapConfig = { + version: '1.6.0', + buffersize: { 'mvt': 0 }, + layers: [ + { + type: 'cartodb', + + options: { + sql: POINTS_SQL_PAIRS, + resolution: 1, + aggregation: { + threshold: 1 + }, + metadata: { + aggrFeatureCount: 0 + } + } + } + ] + }; + if (placement !== 'default') { + this.mapConfig.layers[0].options.aggregation.placement = placement; + } + + this.testClient = new TestClient(this.mapConfig); + this.testClient.getLayergroup((err, body) => { + if (err) { + return done(err); + } + + assert.equal(typeof body.metadata, 'object'); + assert.ok(Array.isArray(body.metadata.layers)); + assert.ok(body.metadata.layers[0].meta.aggregation.mvt); + assert.equal(body.metadata.layers[0].meta.stats.aggrFeatureCount, 9); + + done(); + }); + }); + + it(`on-demand pre-aggregation stats are available with ${placement} aggregation`, function (done) { + global.environment.enabledFeatures.layerStats = true; + this.mapConfig = { + version: '1.6.0', + buffersize: { 'mvt': 0 }, + layers: [ + { + type: 'cartodb', + + options: { + sql: POINTS_SQL_PAIRS, + resolution: 1, + aggregation: { + threshold: 1 + }, + metadata: { + featureCount: true + } + } + } + ] + }; + if (placement !== 'default') { + this.mapConfig.layers[0].options.aggregation.placement = placement; + } + + this.testClient = new TestClient(this.mapConfig); + this.testClient.getLayergroup((err, body) => { + if (err) { + return done(err); + } + + assert.equal(typeof body.metadata, 'object'); + assert.ok(Array.isArray(body.metadata.layers)); + assert.ok(body.metadata.layers[0].meta.aggregation.mvt); + assert.equal(body.metadata.layers[0].meta.stats.featureCount, 13); + + done(); + }); + }); + }); }); }); diff --git a/test/acceptance/stats/mapnik_stats_layergroup.js b/test/acceptance/stats/mapnik_stats_layergroup.js index cdac7bb1..f6cad84e 100644 --- a/test/acceptance/stats/mapnik_stats_layergroup.js +++ b/test/acceptance/stats/mapnik_stats_layergroup.js @@ -49,7 +49,7 @@ describe('Create mapnik layergroup', function() { sql: [ 'select t1.cartodb_id, t1.the_geom, t1.the_geom_webmercator, t2.address', ' from test_table t1, test_table_2 t2', - ' where t1.cartodb_id = t2.cartodb_id;' + ' where t1.cartodb_id = t2.cartodb_id' ].join(''), cartocss_version: cartocssVersion, cartocss: cartocss @@ -90,7 +90,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1); testClient.drain(done); @@ -107,7 +107,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); @@ -127,7 +127,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); @@ -147,7 +147,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); testClient.drain(done); @@ -164,7 +164,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); @@ -183,7 +183,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 3); assert.ok(!layergroup.metadata.layers[0].meta.stats[1]); @@ -204,7 +204,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].type, 'mapnik'); assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1); @@ -224,7 +224,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function (err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, typeLayerId('http', 0)); assert.equal(layergroup.metadata.layers[0].type, 'http'); assert.ok(!layergroup.metadata.layers[0].meta.cartocss); @@ -245,7 +245,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); // we don't care about stats here as is an aliased column assert.ok(layergroup.metadata.layers[0].meta.stats.hasOwnProperty('estimatedFeatureCount')); @@ -265,7 +265,7 @@ describe('Create mapnik layergroup', function() { }); testClient.getLayergroup(function(err, layergroup) { - assert.ok(!err); + assert.ifError(err); assert.equal(layergroup.metadata.layers[0].id, typeLayerId('http', 0)); assert.equal(layergroup.metadata.layers[0].type, 'http'); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(0)); @@ -273,7 +273,206 @@ describe('Create mapnik layergroup', function() { assert.ok(!layergroup.metadata.layers[1].meta.hasOwnProperty('stats')); assert.equal(layergroup.metadata.layers[2].id, typeLayerId('http', 1)); assert.equal(layergroup.metadata.layers[2].type, 'http'); + global.environment.enabledFeatures.layerStats = true; testClient.drain(done); }); }); + + function layerWithMetadata(layer, metadata) { + return Object.assign(layer, { + options: Object.assign(layer.options, { metadata }) + }); + } + + it('should provide columns as optional metadata', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + layerWithMetadata(mapnikLayer4, { + columns: true + }) + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + const expectedColumns = { + cartodb_id: { type: 'number' }, + the_geom: { type: 'geometry' }, + the_geom_webmercator: { type: 'geometry' }, + address: { type: 'string' } + }; + assert.deepEqual(layergroup.metadata.layers[0].meta.stats.columns, expectedColumns); + testClient.drain(done); + }); + }); + + it('should provide column stats as optional metadata', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + layerWithMetadata(mapnikLayer4, { + columnStats: true + }) + ] + }); + + // metadata categories are ordered only partially by descending frequency; + // this orders them completely to avoid ambiguities when comparing + function withSortedCategories(columns) { + function catOrder(a, b) { + if (a.frequency !== b.frequency) { + return b.frequency - a.frequency; + } + if (a.category < b.category) { + return -1; + } + if (a.category > b.category) { + return +1; + } + return 0; + } + let sorted = {}; + Object.keys(columns).forEach(name => { + let data = columns[name]; + if (data.hasOwnProperty('categories')) { + data = Object.assign(data, { categories: data.categories.sort(catOrder)}); + } + sorted[name] = data; + }); + return sorted; + } + + testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + const expectedColumns = { + cartodb_id: { + type: 'number', + avg: 3, + max: 5, + min: 1, + sum: 15 + }, + the_geom: { type: 'geometry' }, + the_geom_webmercator: { type: 'geometry' }, + address: { + type: 'string', + categories: [ + { + category: "Calle de la Palma 72, Madrid, Spain", + frequency: 1 + }, + { + category: "Calle de Pérez Galdós 9, Madrid, Spain", + frequency: 1 + }, + { + category: "Calle Divino Pastor 12, Madrid, Spain", + frequency: 1 + }, + { + category: "Manuel Fernández y González 8, Madrid, Spain", + frequency: 1 + }, + { + category: "Plaza Conde de Toreno 2, Madrid, Spain", + frequency: 1 + } + ] + } + }; + assert.deepEqual( + withSortedCategories(layergroup.metadata.layers[0].meta.stats.columns), + withSortedCategories(expectedColumns) + ); + testClient.drain(done); + }); + }); + + it('should provide row count as optional metadata', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + layerWithMetadata(mapnikLayer4, { + featureCount: true + }) + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.featureCount, 5); + testClient.drain(done); + }); + }); + + it('should provide geometry type as optional metadata', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + layerWithMetadata(mapnikLayer4, { + geometryType: true + }) + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.geometryType, 'ST_Point'); + testClient.drain(done); + }); + }); + + it('should provide a sample as optional metadata', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + layerWithMetadata(mapnikLayer4, { + sample: 3 + }) + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + assert(layergroup.metadata.layers[0].meta.stats.sample.length > 0); + const expectedCols = [ 'cartodb_id', 'address', 'the_geom', 'the_geom_webmercator' ].sort(); + assert.deepEqual(Object.keys(layergroup.metadata.layers[0].meta.stats.sample[0]).sort(), expectedCols); + testClient.drain(done); + }); + }); + + it('should only provide requested optional metadata', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + layerWithMetadata(mapnikLayer4, { + geometryType: true, + featureCount: true + }) + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.geometryType, 'ST_Point'); + assert.equal(layergroup.metadata.layers[0].meta.stats.featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.sample, undefined); + assert.equal(layergroup.metadata.layers[0].meta.stats.columns, undefined); + testClient.drain(done); + }); + }); + });