Merge branch 'master' into upgrade-dependencies

NEWS.md

@@ -31,11 +31,13 @@ New features:
   - moment: 2.22.1
   - nock: 9.2.6
   - strftime: 0.10.0
+- Optional instantiation metadata stats (https://github.com/CartoDB/Windshaft-cartodb/pull/952)
 
 Bug Fixes:
 - Validates tile coordinates (z/x/y) from request params to be a valid integer value.
 - Static maps fails for unsupported formats
 - Handling errors extracting the column type on dataviews
+- Fix `meta.stats.estimatedFeatureCount` for aggregations and queries with tokens
 
 ## 6.1.0
 Released 2018-04-16

lib/cartodb/backends/layer-stats/mapnik-layer-stats.js

@@ -1,4 +1,27 @@
 var queryUtils = require('../../utils/query-utils');
+const AggregationMapConfig = require('../../models/aggregation/aggregation-mapconfig');
+var SubstitutionTokens = require('../../utils/substitution-tokens');
+
+// Instantiate a query with tokens for a given zoom level
+function queryForZoom(sql, zoom, singleTile=false) {
+    const tileRes = 256;
+    const wmSize = 6378137.0*2*Math.PI;
+    const nTiles = Math.pow(2, zoom);
+    const tileSize = wmSize / nTiles;
+    const resolution = tileSize / tileRes;
+    const scaleDenominator = resolution / 0.00028;
+    const x0 = -wmSize/2, y0 = -wmSize/2;
+    let bbox = `ST_MakeEnvelope(${x0}, ${y0}, ${x0+wmSize}, ${y0+wmSize})`;
+    if (singleTile) {
+        bbox = `ST_MakeEnvelope(${x0}, ${y0}, ${x0 + tileSize}, ${y0 + tileSize})`;
+    }
+    return SubstitutionTokens.replace(sql, {
+        bbox: bbox,
+        scale_denominator: scaleDenominator,
+        pixel_width: resolution,
+        pixel_height: resolution
+    });
+}
 
 function MapnikLayerStats () {
     this._types = {
@@ -11,17 +34,278 @@ MapnikLayerStats.prototype.is = function (type) {
     return this._types[type] ? this._types[type] : false;
 };
 
+function queryPromise(dbConnection, query, adaptResults, errorHandler) {
+    return new Promise(function(resolve, reject) {
+        dbConnection.query(query, function (err, res) {
+            if (err) {
+                if (errorHandler) {
+                    resolve(errorHandler(err));
+                }
+                else {
+                    reject(err);
+                }
+            }
+            else {
+                resolve(adaptResults(res));
+            }
+        });
+
+    });
+}
+
+function columnAggregations(field) {
+    if (field.type === 'number') {
+        return ['min', 'max', 'avg', 'sum'];
+    }
+    if (field.type === 'date') { // TODO other types too?
+        return ['min', 'max'];
+    }
+    return [];
+}
+
+function _getSQL(ctx, query, type='pre', zoom=0) {
+    let sql;
+    if (type === 'pre') {
+        sql = ctx.preQuery;
+    }
+    else {
+        sql = ctx.aggrQuery;
+    }
+    sql = queryForZoom(sql, zoom || 0);
+    return query(sql);
+}
+
+function _estimatedFeatureCount(ctx) {
+    return queryPromise(
+        ctx.dbConnection,
+        _getSQL(ctx, queryUtils.getQueryRowEstimation),
+        res => ({ estimatedFeatureCount: res.rows[0].rows }),
+        () => ({ estimatedFeatureCount: -1 })
+    );
+}
+
+function _featureCount(ctx) {
+    if (ctx.metaOptions.featureCount) {
+        // TODO: if ctx.metaOptions.columnStats we can combine this with column stats query
+        return queryPromise(
+            ctx.dbConnection,
+            _getSQL(ctx, queryUtils.getQueryActualRowCount),
+            res => ({ featureCount: res.rows[0].rows })
+        );
+    }
+    return Promise.resolve();
+}
+
+function _aggrFeatureCount(ctx) {
+    if (ctx.metaOptions.hasOwnProperty('aggrFeatureCount')) {
+        // We expect as zoom level as the value of aggrFeatureCount
+        // TODO: it'd be nice to admit an array of zoom levels to
+        // return metadata for multiple levels.
+        return queryPromise(
+            ctx.dbConnection,
+            _getSQL(ctx, queryUtils.getQueryActualRowCount, 'post',  ctx.metaOptions.aggrFeatureCount),
+            res => ({ aggrFeatureCount: res.rows[0].rows })
+        );
+    }
+    return Promise.resolve();
+}
+
+function _geometryType(ctx) {
+    if (ctx.metaOptions.geometryType) {
+        const geometryColumn = AggregationMapConfig.getAggregationGeometryColumn();
+        return queryPromise(
+            ctx.dbConnection,
+            _getSQL(ctx, sql => queryUtils.getQueryGeometryType(sql, geometryColumn)),
+            res => ({ geometryType: res.rows[0].geom_type })
+        );
+    }
+    return Promise.resolve();
+}
+
+function _columns(ctx) {
+    if (ctx.metaOptions.columns || ctx.metaOptions.columnStats) {
+        // note: post-aggregation columns are in layer.options.columns when aggregation is present
+        return queryPromise(
+            ctx.dbConnection,
+            _getSQL(ctx, sql => queryUtils.getQueryLimited(sql, 0)),
+            res => formatResultFields(ctx.dbConnection, res.fields)
+        );
+    }
+    return Promise.resolve();
+}
+
+// combine a list of results merging the properties of all the objects
+// undefined results are admitted and ignored
+function mergeResults(results) {
+    if (results) {
+        if (results.length === 0) {
+            return {};
+        }
+        return results.reduce((a, b) => {
+            if (a === undefined) {
+                return b;
+            }
+            if (b === undefined) {
+                return a;
+            }
+            return Object.assign({}, a, b);
+        });
+    }
+}
+
+// deeper (1 level) combination of a list of objects:
+// mergeColumns([{ col1: { a: 1 }, col2: { a: 2 } }, { col1: { b: 3 } }]) => { col1: { a: 1, b: 3 }, col2: { a: 2 } }
+function mergeColumns(results) {
+    if (results) {
+        if (results.length === 0) {
+            return {};
+        }
+        return results.reduce((a, b) => {
+            let c = Object.assign({}, b || {}, a || {});
+            Object.keys(c).forEach(key => {
+                if (b.hasOwnProperty(key)) {
+                    c[key] = Object.assign(c[key], b[key]);
+                }
+            });
+            return c;
+        });
+    }
+}
+
+function _sample(ctx, numRows) {
+    if (ctx.metaOptions.sample) {
+        const sampleProb = Math.min(ctx.metaOptions.sample / numRows, 1);
+        // We'll use a safety limit just in case numRows is a bad estimate
+        const limit = Math.ceil(ctx.metaOptions.sample * 1.5);
+        return queryPromise(
+            ctx.dbConnection,
+            _getSQL(ctx, sql => queryUtils.getQuerySample(sql, sampleProb, limit)),
+            res => ({ sample: res.rows })
+        );
+    }
+    return Promise.resolve();
+}
+
+function _columnStats(ctx, columns) {
+    if (!columns) {
+        return Promise.resolve();
+    }
+    if (ctx.metaOptions.columnStats) {
+        let queries = [];
+        let aggr = [];
+        queries.push(new Promise(resolve => resolve(columns))); // add columns as first result
+        Object.keys(columns).forEach(name => {
+            aggr = aggr.concat(
+                columnAggregations(columns[name])
+                .map(fn => `${fn}(${name}) AS ${name}_${fn}`)
+            );
+            if (columns[name].type === 'string') {
+                const topN = ctx.metaOptions.columnStats.topCategories || 1024;
+                // TODO:  ctx.metaOptions.columnStats.maxCategories
+                //        => use PG stats to dismiss columns with more distinct values
+                queries.push(
+                    queryPromise(
+                        ctx.dbConnection,
+                        _getSQL(ctx, sql => queryUtils.getQueryTopCategories(sql, name, topN)),
+                        res => ({ [name]: { categories: res.rows } })
+                    )
+                );
+            }
+        });
+        queries.push(
+            queryPromise(
+                ctx.dbConnection,
+                _getSQL(ctx, sql => `SELECT ${aggr.join(',')} FROM (${sql}) AS __cdb_query`),
+                res => {
+                    let stats = {};
+                    Object.keys(columns).forEach(name => {
+                        stats[name] = {};
+                        columnAggregations(columns[name]).forEach(fn => {
+                            stats[name][fn] = res.rows[0][`${name}_${fn}`];
+                        });
+                    });
+                    return stats;
+                }
+            )
+        );
+        return Promise.all(queries).then(results => ({ columns: mergeColumns(results) }));
+    }
+    return Promise.resolve({ columns });
+}
+
+// This is adapted from SQL API:
+function fieldType(cname) {
+    let tname;
+    switch (true) {
+        case /bool/.test(cname):
+            tname = 'boolean';
+            break;
+        case /int|float|numeric/.test(cname):
+            tname = 'number';
+            break;
+        case /text|char|unknown/.test(cname):
+            tname = 'string';
+            break;
+        case /date|time/.test(cname):
+            tname = 'date';
+            break;
+        default:
+            tname = cname;
+    }
+    if ( tname && cname.match(/^_/) ) {
+        tname += '[]';
+    }
+    return tname;
+}
+
+// columns are returned as an object { columnName1: { type1: ...}, ..}
+// for consistency with SQL API
+function formatResultFields(dbConnection, fields = []) {
+    let nfields = {};
+    for (let field of fields) {
+      const cname = dbConnection.typeName(field.dataTypeID);
+      let tname;
+      if ( ! cname ) {
+        tname = 'unknown(' + field.dataTypeID + ')';
+      } else {
+        tname = fieldType(cname);
+      }
+      nfields[field.name] = { type: tname };
+    }
+    return nfields;
+}
+
 MapnikLayerStats.prototype.getStats =
 function (layer, dbConnection, callback) {
-    var queryRowCountSql = queryUtils.getQueryRowCount(layer.options.sql);
-    // This query would gather stats for postgresql table if not exists
-    dbConnection.query(queryRowCountSql, function (err, res) {
-        if (err) {
-            return callback(null, {estimatedFeatureCount: -1});
-        } else {
-            // We decided that the relation is 1 row == 1 feature
-            return callback(null, {estimatedFeatureCount: res.rows[0].rows});
-        }
+    let aggrQuery = layer.options.sql;
+    let preQuery =  layer.options.sql_raw || aggrQuery;
+
+    let ctx = {
+        dbConnection,
+        preQuery,
+        aggrQuery,
+        metaOptions: layer.options.metadata || {}
+    };
+
+    // TODO: could save some queries if queryUtils.getAggregationMetadata() has been used and kept somewhere
+    // we would set queries.results.estimatedFeatureCount and queries.results.geometryType
+    // (if metaOptions.geometryType) from it.
+
+    // TODO: compute _sample with _featureCount when available
+
+    Promise.all([
+        _estimatedFeatureCount(ctx).then(
+            ({ estimatedFeatureCount }) => _sample(ctx, estimatedFeatureCount)
+                .then(sampleResults => mergeResults([sampleResults, { estimatedFeatureCount }]))
+        ),
+        _featureCount(ctx),
+        _aggrFeatureCount(ctx),
+        _geometryType(ctx),
+        _columns(ctx).then(columns => _columnStats(ctx, columns))
+    ]).then(results => {
+        callback(null, mergeResults(results));
+    }).catch(error => {
+        callback(error);
     });
 };
 

lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js

@@ -101,6 +101,12 @@ module.exports = class AggregationMapConfigAdapter {
                     aggregationSql = sqlQueryWrap.replace(/<%=\s*sql\s*%>/g, aggregationSql);
                 }
 
+                if (!layer.options.sql_raw) {
+                    // if sql_wrap is present, the original query should already be
+                    // in sql_raw (with sql being the wrapped query);
+                    // otherwise we keep the now the original query in sql_raw
+                    layer.options.sql_raw = layer.options.sql;
+                }
                 layer.options.sql = aggregationSql;
 
                 mapConfig.getLayerAggregationColumns(index, (err, columns) => {
@@ -159,6 +165,7 @@ module.exports = class AggregationMapConfigAdapter {
     }
 
     _getAggregationMetadata (mapConfig, layer, adapted) {
+        // also: pre-aggr query, columns, ...
         if (!adapted) {
             return { png: false, mvt: false };
         }

lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js

@@ -117,7 +117,9 @@ TurboCartoAdapter.prototype._parseCartoCss = function (username, params, layer,
 
     var layerSql = layer.options.sql;
     var layerRawSql = layer.options.sql_raw;
-    if (SubstitutionTokens.hasTokens(layerSql) && layerRawSql) {
+    if (SubstitutionTokens.hasTokens(layerSql) && layerRawSql && layer.options.sql_wrap) {
+        // For wrapped queries we'll derive the tokens from the data extent
+        // instead of the whole Earth/root tile.
         var self = this;
         var tokensQuery = tokensQueryTpl({_sql: layerRawSql});
         return pg.query(tokensQuery, function(err, resultSet) {

lib/cartodb/utils/query-utils.js

@@ -21,10 +21,15 @@ module.exports.extractTableNames = function extractTableNames(query) {
     ].join('');
 };
 
+module.exports.getQueryActualRowCount = function (query) {
+    return `select COUNT(*) AS rows FROM (${query}) AS __cdb_query`;
+};
+
 function getQueryRowEstimation(query) {
     return 'select CDB_EstimateRowCount($windshaft$' + query + '$windshaft$) as rows';
 }
-module.exports.getQueryRowCount = getQueryRowEstimation;
+
+module.exports.getQueryRowEstimation = getQueryRowEstimation;
 
 module.exports.getAggregationMetadata = ctx => `
     WITH
@@ -71,3 +76,81 @@ module.exports.countNaNs = function countNaNs(ctx) {
         `sum(CASE WHEN (${ctx.column} = 'NaN'::float) THEN 1 ELSE 0 END)`
     }`;
 };
+
+module.exports.getQueryTopCategories = function(query, column, topN, includeNulls=false) {
+    const where = includeNulls ? '' : `WHERE ${column} IS NOT NULL`;
+    return `
+        SELECT ${column} AS category, COUNT(*) AS frequency
+        FROM (${query}) AS __cdb_query
+        ${where}
+        GROUP BY ${column} ORDER BY 2 DESC
+        LIMIT ${topN}
+    `;
+};
+
+module.exports.getQuerySample = function(query, sampleProb, limit = null, randomSeed = 0.5) {
+    const singleTable = simpleQueryTable(query);
+    if (singleTable) {
+        return getTableSample(singleTable.table, singleTable.columns, sampleProb, limit, randomSeed);
+    }
+    const limitClause = limit ? `LIMIT ${limit}` : '';
+    return `
+        WITH __cdb_rndseed AS (
+            SELECT setseed(${randomSeed})
+        )
+        SELECT *
+            FROM (${query}) AS __cdb_query
+            WHERE random() < ${sampleProb}
+            ${limitClause}
+    `;
+};
+
+function getTableSample(table, columns, sampleProb, limit = null, randomSeed = 0.5) {
+    const limitClause = limit ? `LIMIT ${limit}` : '';
+    sampleProb *= 100;
+    randomSeed *= Math.pow(2, 31) -1;
+    return `
+        SELECT ${columns} FROM ${table} TABLESAMPLE BERNOULLI (${sampleProb}) REPEATABLE (${randomSeed}) ${limitClause}
+    `;
+}
+
+function simpleQueryTable(sql) {
+    const basicQuery =
+        /\s*SELECT\s+([\*a-z0-9_,\s]+?)\s+FROM\s+((\"[^"]+\"|[a-z0-9_]+)\.)?(\"[^"]+\"|[a-z0-9_]+)\s*;?\s*/i;
+    const unwrappedQuery = new RegExp("^"+basicQuery.source+"$", 'i');
+    // queries for named maps are wrapped like this:
+    var wrappedQuery = new RegExp(
+        "^\\s*SELECT\\s+\\*\\s+FROM\\s+\\(" +
+        basicQuery.source +
+        "\\)\\s+AS\\s+wrapped_query\\s+WHERE\\s+\\d+=1\\s*$",
+        'i'
+    );
+    let match = sql.match(unwrappedQuery);
+    if (!match) {
+        match = sql.match(wrappedQuery);
+    }
+    if (match) {
+        const columns = match[1];
+        const schema = match[3];
+        const table  = match[4];
+        return { table: schema ? `${schema}.${table}` : table, columns };
+    }
+    return false;
+}
+
+module.exports.getQueryGeometryType = function(query, geometryColumn) {
+    return `
+        SELECT ST_GeometryType(${geometryColumn}) AS geom_type
+            FROM (${query}) AS __cdb_query
+            WHERE ${geometryColumn} IS NOT NULL
+            LIMIT 1
+    `;
+};
+
+module.exports.getQueryLimited = function(query, limit=0) {
+    return `
+        SELECT *
+            FROM (${query}) AS __cdb_query
+            LIMIT ${limit}
+    `;
+};

test/acceptance/aggregation.js

@@ -100,6 +100,18 @@ describe('aggregation', function () {
     from generate_series(-3, 3) x
     `;
 
+    const POINTS_SQL_PAIRS = `
+    -- Generate pairs of near points
+    select
+        x + 4 as cartodb_id,
+        st_setsrid(st_makepoint(Floor(x/2)*10 + x/1000.0, Floor(x/2)*10 + x/1000.0), 4326) as the_geom,
+        st_transform(
+            st_setsrid(st_makepoint(Floor(x/2)*10 + x/1000.0, Floor(x/2)*10 + x/1000.0),4326),
+            3857) as the_geom_webmercator,
+        x as value
+    from generate_series(-6, 6) x
+    `;
+
     function createVectorMapConfig (layers = [
         {
             type: 'cartodb',
@@ -130,6 +142,7 @@ describe('aggregation', function () {
 
             before(function () {
                 serverOptions.renderer.mvt.usePostGIS = usePostGIS;
+                this.layerStatsConfig = global.environment.enabledFeatures.layerStats;
             });
 
             after(function (){
@@ -138,6 +151,7 @@ describe('aggregation', function () {
 
             afterEach(function (done) {
                 this.testClient.drain(done);
+                global.environment.enabledFeatures.layerStats = this.layerStatsConfig;
             });
 
             it('should return a layergroup indicating the mapconfig was aggregated', function (done) {
@@ -2173,6 +2187,168 @@ describe('aggregation', function () {
                 });
             });
 
+            ['default', 'centroid', 'point-sample', 'point-grid'].forEach(placement => {
+                it(`default pre-aggregation stats are available with ${placement} aggregation`, function (done) {
+                    global.environment.enabledFeatures.layerStats = true;
+                    this.mapConfig = {
+                        version: '1.6.0',
+                        buffersize: { 'mvt': 0 },
+                        layers: [
+                            {
+                                type: 'cartodb',
+
+                                options: {
+                                    sql: POINTS_SQL_PAIRS,
+                                    resolution: 1,
+                                    aggregation: {
+                                        threshold: 1
+                                    }
+                                }
+                            }
+                        ]
+                    };
+                    if (placement !== 'default') {
+                        this.mapConfig.layers[0].options.aggregation.placement = placement;
+                    }
+
+                    this.testClient = new TestClient(this.mapConfig);
+                    this.testClient.getLayergroup((err, body) => {
+                        if (err) {
+                            return done(err);
+                        }
+
+                        assert.equal(typeof body.metadata, 'object');
+                        assert.ok(Array.isArray(body.metadata.layers));
+                        assert.ok(body.metadata.layers[0].meta.aggregation.mvt);
+                        assert.ok(body.metadata.layers[0].meta.stats.estimatedFeatureCount > 0);
+
+                        done();
+                    });
+                });
+
+                it(`on demand post-aggregation stats are available with ${placement} aggregation`, function (done) {
+                    global.environment.enabledFeatures.layerStats = true;
+                    this.mapConfig = {
+                        version: '1.6.0',
+                        buffersize: { 'mvt': 0 },
+                        layers: [
+                            {
+                                type: 'cartodb',
+
+                                options: {
+                                    sql: POINTS_SQL_PAIRS,
+                                    resolution: 1,
+                                    aggregation: {
+                                        threshold: 1
+                                    },
+                                    metadata: {
+                                        aggrFeatureCount: 10
+                                    }
+                                }
+                            }
+                        ]
+                    };
+                    if (placement !== 'default') {
+                        this.mapConfig.layers[0].options.aggregation.placement = placement;
+                    }
+
+                    this.testClient = new TestClient(this.mapConfig);
+                    this.testClient.getLayergroup((err, body) => {
+                        if (err) {
+                            return done(err);
+                        }
+
+                        assert.equal(typeof body.metadata, 'object');
+                        assert.ok(Array.isArray(body.metadata.layers));
+                        assert.ok(body.metadata.layers[0].meta.aggregation.mvt);
+                        assert.equal(body.metadata.layers[0].meta.stats.aggrFeatureCount, 13);
+
+                        done();
+                    });
+                });
+
+                it(`post-aggregation count adapts to zoom level with ${placement} aggregation`, function (done) {
+                    global.environment.enabledFeatures.layerStats = true;
+                    this.mapConfig = {
+                        version: '1.6.0',
+                        buffersize: { 'mvt': 0 },
+                        layers: [
+                            {
+                                type: 'cartodb',
+
+                                options: {
+                                    sql: POINTS_SQL_PAIRS,
+                                    resolution: 1,
+                                    aggregation: {
+                                        threshold: 1
+                                    },
+                                    metadata: {
+                                        aggrFeatureCount: 0
+                                    }
+                                }
+                            }
+                        ]
+                    };
+                    if (placement !== 'default') {
+                        this.mapConfig.layers[0].options.aggregation.placement = placement;
+                    }
+
+                    this.testClient = new TestClient(this.mapConfig);
+                    this.testClient.getLayergroup((err, body) => {
+                        if (err) {
+                            return done(err);
+                        }
+
+                        assert.equal(typeof body.metadata, 'object');
+                        assert.ok(Array.isArray(body.metadata.layers));
+                        assert.ok(body.metadata.layers[0].meta.aggregation.mvt);
+                        assert.equal(body.metadata.layers[0].meta.stats.aggrFeatureCount, 9);
+
+                        done();
+                    });
+                });
+
+                it(`on-demand pre-aggregation stats are available with ${placement} aggregation`, function (done) {
+                    global.environment.enabledFeatures.layerStats = true;
+                    this.mapConfig = {
+                        version: '1.6.0',
+                        buffersize: { 'mvt': 0 },
+                        layers: [
+                            {
+                                type: 'cartodb',
+
+                                options: {
+                                    sql: POINTS_SQL_PAIRS,
+                                    resolution: 1,
+                                    aggregation: {
+                                        threshold: 1
+                                    },
+                                    metadata: {
+                                        featureCount: true
+                                    }
+                                }
+                            }
+                        ]
+                    };
+                    if (placement !== 'default') {
+                        this.mapConfig.layers[0].options.aggregation.placement = placement;
+                    }
+
+                    this.testClient = new TestClient(this.mapConfig);
+                    this.testClient.getLayergroup((err, body) => {
+                        if (err) {
+                            return done(err);
+                        }
+
+                        assert.equal(typeof body.metadata, 'object');
+                        assert.ok(Array.isArray(body.metadata.layers));
+                        assert.ok(body.metadata.layers[0].meta.aggregation.mvt);
+                        assert.equal(body.metadata.layers[0].meta.stats.featureCount, 13);
+
+                        done();
+                    });
+                });
+            });
 
         });
     });

test/acceptance/stats/mapnik_stats_layergroup.js

@@ -49,7 +49,7 @@ describe('Create mapnik layergroup', function() {
             sql: [
                 'select t1.cartodb_id, t1.the_geom, t1.the_geom_webmercator, t2.address',
                 ' from test_table t1, test_table_2 t2',
-                ' where t1.cartodb_id = t2.cartodb_id;'
+                ' where t1.cartodb_id = t2.cartodb_id'
             ].join(''),
             cartocss_version: cartocssVersion,
             cartocss: cartocss
@@ -90,7 +90,7 @@ describe('Create mapnik layergroup', function() {
         });
 
         testClient.getLayergroup(function(err, layergroup) {
-            assert.ok(!err);
+            assert.ifError(err);
             assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
             assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1);
             testClient.drain(done);
@@ -107,7 +107,7 @@ describe('Create mapnik layergroup', function() {
         });
 
         testClient.getLayergroup(function(err, layergroup) {
-            assert.ok(!err);
+            assert.ifError(err);
             assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
             assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1);
             assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1));
@@ -127,7 +127,7 @@ describe('Create mapnik layergroup', function() {
         });
 
         testClient.getLayergroup(function(err, layergroup) {
-            assert.ok(!err);
+            assert.ifError(err);
             assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
             assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1);
             assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1));
@@ -147,7 +147,7 @@ describe('Create mapnik layergroup', function() {
         });
 
         testClient.getLayergroup(function(err, layergroup) {
-            assert.ok(!err);
+            assert.ifError(err);
             assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
             assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5);
             testClient.drain(done);
@@ -164,7 +164,7 @@ describe('Create mapnik layergroup', function() {
         });
 
         testClient.getLayergroup(function(err, layergroup) {
-            assert.ok(!err);
+            assert.ifError(err);
             assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
             assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5);
             assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1));
@@ -183,7 +183,7 @@ describe('Create mapnik layergroup', function() {
         });
 
         testClient.getLayergroup(function(err, layergroup) {
-            assert.ok(!err);
+            assert.ifError(err);
             assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
             assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 3);
             assert.ok(!layergroup.metadata.layers[0].meta.stats[1]);
@@ -204,7 +204,7 @@ describe('Create mapnik layergroup', function() {
         });
 
         testClient.getLayergroup(function(err, layergroup) {
-            assert.ok(!err);
+            assert.ifError(err);
             assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
             assert.equal(layergroup.metadata.layers[0].type, 'mapnik');
             assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1);
@@ -224,7 +224,7 @@ describe('Create mapnik layergroup', function() {
         });
 
         testClient.getLayergroup(function (err, layergroup) {
-            assert.ok(!err);
+            assert.ifError(err);
             assert.equal(layergroup.metadata.layers[0].id, typeLayerId('http', 0));
             assert.equal(layergroup.metadata.layers[0].type, 'http');
             assert.ok(!layergroup.metadata.layers[0].meta.cartocss);
@@ -245,7 +245,7 @@ describe('Create mapnik layergroup', function() {
         });
 
         testClient.getLayergroup(function(err, layergroup) {
-            assert.ok(!err);
+            assert.ifError(err);
             assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
             // we don't care about stats here as is an aliased column
             assert.ok(layergroup.metadata.layers[0].meta.stats.hasOwnProperty('estimatedFeatureCount'));
@@ -265,7 +265,7 @@ describe('Create mapnik layergroup', function() {
         });
 
         testClient.getLayergroup(function(err, layergroup) {
-            assert.ok(!err);
+            assert.ifError(err);
             assert.equal(layergroup.metadata.layers[0].id, typeLayerId('http', 0));
             assert.equal(layergroup.metadata.layers[0].type, 'http');
             assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(0));
@@ -273,7 +273,206 @@ describe('Create mapnik layergroup', function() {
             assert.ok(!layergroup.metadata.layers[1].meta.hasOwnProperty('stats'));
             assert.equal(layergroup.metadata.layers[2].id, typeLayerId('http', 1));
             assert.equal(layergroup.metadata.layers[2].type, 'http');
+            global.environment.enabledFeatures.layerStats = true;
             testClient.drain(done);
         });
     });
+
+    function layerWithMetadata(layer, metadata) {
+        return Object.assign(layer, {
+            options: Object.assign(layer.options, { metadata })
+        });
+    }
+
+    it('should provide columns as optional metadata', function(done) {
+        var testClient = new TestClient({
+            version: '1.4.0',
+            layers: [
+                layerWithMetadata(mapnikLayer4, {
+                    columns: true
+                })
+            ]
+        });
+
+        testClient.getLayergroup(function(err, layergroup) {
+            assert.ifError(err);
+            assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
+            assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5);
+            const expectedColumns = {
+                cartodb_id: { type: 'number' },
+                the_geom: { type: 'geometry' },
+                the_geom_webmercator: { type: 'geometry' },
+                address: { type: 'string' }
+            };
+            assert.deepEqual(layergroup.metadata.layers[0].meta.stats.columns, expectedColumns);
+            testClient.drain(done);
+        });
+    });
+
+    it('should provide column stats as optional metadata', function(done) {
+        var testClient = new TestClient({
+            version: '1.4.0',
+            layers: [
+                layerWithMetadata(mapnikLayer4, {
+                    columnStats: true
+                })
+            ]
+        });
+
+        // metadata categories are ordered only partially by descending frequency;
+        // this orders them completely to avoid ambiguities when comparing
+        function withSortedCategories(columns) {
+            function catOrder(a, b) {
+                if (a.frequency !== b.frequency) {
+                    return b.frequency - a.frequency;
+                }
+                if (a.category < b.category) {
+                    return -1;
+                }
+                if (a.category > b.category) {
+                    return +1;
+                }
+                return 0;
+            }
+            let sorted = {};
+            Object.keys(columns).forEach(name => {
+                let data = columns[name];
+                if (data.hasOwnProperty('categories')) {
+                    data = Object.assign(data, { categories: data.categories.sort(catOrder)});
+                }
+                sorted[name] = data;
+            });
+            return sorted;
+        }
+
+        testClient.getLayergroup(function(err, layergroup) {
+            assert.ifError(err);
+            assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
+            assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5);
+            const expectedColumns = {
+                cartodb_id: {
+                    type: 'number',
+                    avg: 3,
+                    max: 5,
+                    min: 1,
+                    sum: 15
+                },
+                the_geom: { type: 'geometry' },
+                the_geom_webmercator: { type: 'geometry' },
+                address: {
+                    type: 'string',
+                    categories: [
+                        {
+                            category: "Calle de la Palma 72, Madrid, Spain",
+                            frequency: 1
+                        },
+                        {
+                            category: "Calle de Pérez Galdós 9, Madrid, Spain",
+                            frequency: 1
+                        },
+                        {
+                            category: "Calle Divino Pastor 12, Madrid, Spain",
+                            frequency: 1
+                        },
+                        {
+                            category: "Manuel Fernández y González 8, Madrid, Spain",
+                            frequency: 1
+                        },
+                        {
+                            category: "Plaza Conde de Toreno 2, Madrid, Spain",
+                            frequency: 1
+                        }
+                    ]
+                }
+            };
+            assert.deepEqual(
+                withSortedCategories(layergroup.metadata.layers[0].meta.stats.columns),
+                withSortedCategories(expectedColumns)
+            );
+            testClient.drain(done);
+        });
+    });
+
+    it('should provide row count as optional metadata', function(done) {
+        var testClient = new TestClient({
+            version: '1.4.0',
+            layers: [
+                layerWithMetadata(mapnikLayer4, {
+                    featureCount: true
+                })
+            ]
+        });
+
+        testClient.getLayergroup(function(err, layergroup) {
+            assert.ifError(err);
+            assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
+            assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5);
+            assert.equal(layergroup.metadata.layers[0].meta.stats.featureCount, 5);
+            testClient.drain(done);
+        });
+    });
+
+    it('should provide geometry type as optional metadata', function(done) {
+        var testClient = new TestClient({
+            version: '1.4.0',
+            layers: [
+                layerWithMetadata(mapnikLayer4, {
+                    geometryType: true
+                })
+            ]
+        });
+
+        testClient.getLayergroup(function(err, layergroup) {
+            assert.ifError(err);
+            assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
+            assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5);
+            assert.equal(layergroup.metadata.layers[0].meta.stats.geometryType, 'ST_Point');
+            testClient.drain(done);
+        });
+    });
+
+    it('should provide a sample as optional metadata', function(done) {
+        var testClient = new TestClient({
+            version: '1.4.0',
+            layers: [
+                layerWithMetadata(mapnikLayer4, {
+                    sample: 3
+                })
+            ]
+        });
+
+        testClient.getLayergroup(function(err, layergroup) {
+            assert.ifError(err);
+            assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
+            assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5);
+            assert(layergroup.metadata.layers[0].meta.stats.sample.length > 0);
+            const expectedCols = [ 'cartodb_id', 'address', 'the_geom', 'the_geom_webmercator' ].sort();
+            assert.deepEqual(Object.keys(layergroup.metadata.layers[0].meta.stats.sample[0]).sort(), expectedCols);
+            testClient.drain(done);
+        });
+    });
+
+    it('should only provide requested optional metadata', function(done) {
+        var testClient = new TestClient({
+            version: '1.4.0',
+            layers: [
+                layerWithMetadata(mapnikLayer4, {
+                    geometryType: true,
+                    featureCount: true
+                })
+            ]
+        });
+
+        testClient.getLayergroup(function(err, layergroup) {
+            assert.ifError(err);
+            assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
+            assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5);
+            assert.equal(layergroup.metadata.layers[0].meta.stats.geometryType, 'ST_Point');
+            assert.equal(layergroup.metadata.layers[0].meta.stats.featureCount, 5);
+            assert.equal(layergroup.metadata.layers[0].meta.stats.sample, undefined);
+            assert.equal(layergroup.metadata.layers[0].meta.stats.columns, undefined);
+            testClient.drain(done);
+        });
+    });
+
 });