Metadata fixes

lib/cartodb/backends/layer-stats/mapnik-layer-stats.js

@@ -35,6 +35,7 @@ function columnAggregations(field) {
     if (field.type === 'date') { // TODO other types too?
         return ['min', 'max'];
     }
+    return [];
 }
 
 function firstPhaseQueries(queries, ctx) {
@@ -58,8 +59,9 @@ function firstPhaseQueries(queries, ctx) {
         // TODO: if ctx.metaOptions.columnStats we can combine this with column stats query
         queries.task(
             queryPromise(
+                ctx.dbConnection,
                 queryUtils.getQueryActualRowCount(ctx.rawQuery),
-                function(err, res) {
+                (err, res) => {
                     if (err) {
                         queries.results.featureCount = -1;
                     } else {
@@ -74,12 +76,16 @@ function firstPhaseQueries(queries, ctx) {
     if (ctx.metaOptions.geometryType && queries.results.geometryType === undefined) {
         const geometryColumn = AggregationMapConfig.getAggregationGeometryColumn();
         queries.task(
-            queryPromise(queryUtils.getQueryGeometryType(ctx.rawQuery, geometryColumn), function(err, res) {
-                if (!err) {
-                    queries.results.geometryType = res.geom_type;
+            queryPromise(
+                ctx.dbConnection,
+                queryUtils.getQueryGeometryType(ctx.rawQuery, geometryColumn),
+                (err, res) => {
+                    if (!err) {
+                        queries.results.geometryType = res.rows[0].geom_type;
+                    }
+                    return err;
                 }
-                return err;
-            })
+            )
         );
     }
 
@@ -87,12 +93,16 @@ function firstPhaseQueries(queries, ctx) {
         queries.task(
             // TODO: note we have getLayerColumns in aggregation mapconfig.
             // and also getLayerAggregationColumns which either uses getLayerColumns or derives columns from parameters
-            queryPromise(queryUtils.getQueryLimited(ctx.rawQuery, 0), function(err, res) {
-                if (!err) {
-                    queries.results.columns = res.fields;
+            queryPromise(
+                ctx.dbConnection,
+                queryUtils.getQueryLimited(ctx.rawQuery, 0),
+                (err, res) => {
+                    if (!err) {
+                        queries.results.columns = formatResultFields(ctx.dbConnection, res.fields);
+                    }
+                    return err;
                 }
-                return err;
-            })
+            )
         );
     }
 }
@@ -105,8 +115,9 @@ function secondPhaseQueries(queries, ctx) {
         const sampleProb = Math.min(ctx.metaOptions.sample / numRows, 1);
         queries.task(
             queryPromise(
+                ctx.dbConnection,
                 queryUtils.getQuerySample(ctx.rawQuery, sampleProb),
-                function(err, res) {
+                (err, res) => {
                     if (err) {
                         queries.results.sample = [];
                     } else {
@@ -121,38 +132,90 @@ function secondPhaseQueries(queries, ctx) {
     if (ctx.metaOptions.columnStats) {
         let aggr = [];
         Object.keys(queries.results.columns).forEach(name => {
-            aggr = aggr.concat(columnAggregations(queries.results.columns[name])
-                .map(fn => `${fn}(${name}) AS ${name}_${fn}`));
+            aggr = aggr.concat(
+                columnAggregations(queries.results.columns[name])
+                .map(fn => `${fn}(${name}) AS ${name}_${fn}`)
+            );
             if (queries.results.columns[name].type === 'string') {
                 const topN = ctx.metaOptions.columnStats.topCategories || 1024;
                 // TODO:  ctx.metaOptions.columnStats.maxCategories
                 //        => use PG stats to dismiss columns with more distinct values
                 queries.task(
-                    queryPromise(queryUtils.getQueryTopCategories(ctx.rawQuery, name, topN), function(err, res){
-                        if (!err) {
-                            queries.results.columns[name].categories = res.rows;
+                    queryPromise(
+                        ctx.dbConnection,
+                        queryUtils.getQueryTopCategories(ctx.rawQuery, name, topN),
+                        (err, res) => {
+                            if (!err) {
+                                queries.results.columns[name].categories = res.rows;
+                            }
+                            return err;
                         }
-                        return err;
-                    })
+                    )
                 );
             }
         });
         queries.task(
-            queryPromise(`SELECT ${aggr.join(',')} FROM (${ctx.rawQuery})`, function(err, res){
-                if (!err) {
-                    Object.keys(queries.results.columns).forEach(name => {
-                        columnAggregations(queries.results.columns[name]).forEach(fn => {
-                            queries.results.columns[name][fn] = res.rows[0][`${name}_${fn}`];
+            queryPromise(
+                ctx.dbConnection,
+                `SELECT ${aggr.join(',')} FROM (${ctx.rawQuery}) AS __cdb_query`,
+                (err, res) => {
+                    if (!err) {
+                        Object.keys(queries.results.columns).forEach(name => {
+                            columnAggregations(queries.results.columns[name]).forEach(fn => {
+                                queries.results.columns[name][fn] = res.rows[0][`${name}_${fn}`];
+                            });
                         });
-                    });
+                    }
+                    return err;
                 }
-                return err;
-            })
+            )
         );
     }
 
 }
 
+// This is adapted from SQL API:
+function fieldType(cname) {
+    let tname;
+    switch (true) {
+        case /bool/.test(cname):
+            tname = 'boolean';
+            break;
+        case /int|float|numeric/.test(cname):
+            tname = 'number';
+            break;
+        case /text|char|unknown/.test(cname):
+            tname = 'string';
+            break;
+        case /date|time/.test(cname):
+            tname = 'date';
+            break;
+        default:
+            tname = cname;
+    }
+    if ( tname && cname.match(/^_/) ) {
+        tname += '[]';
+    }
+    return tname;
+}
+
+function formatResultFields(dbConnection, flds) {
+    flds = flds || [];
+    var nfields = {};
+    for (var i=0; i<flds.length; ++i) {
+      var f = flds[i];
+      var cname = dbConnection.typeName(f.dataTypeID);
+      var tname;
+      if ( ! cname ) {
+        tname = 'unknown(' + f.dataTypeID + ')';
+      } else {
+        tname = fieldType(cname);
+      }
+      nfields[f.name] = { type: tname };
+    }
+    return nfields;
+}
+
 MapnikLayerStats.prototype.getStats =
 function (layer, dbConnection, callback) {
     let context = {
@@ -168,11 +231,6 @@ function (layer, dbConnection, callback) {
     // we would set queries.results.estimatedFeatureCount and queries.results.geometryType
     // (if metaOptions.geometryType) from it.
 
-    // We'll add promises for queries to be executed to the next two lists;
-    // the queries in statQueries2 will be executed after all of statQueries are completed,
-    // so any results from them can be used.
-    // Query promises will store results in the shared stats object.
-
     // Queries will be executed in two phases, with results from the first phase needed
     // to define the queries of the second phase
     queries.phase(() => firstPhaseQueries(queries, context));

lib/cartodb/utils/query-utils.js

@@ -21,10 +21,15 @@ module.exports.extractTableNames = function extractTableNames(query) {
     ].join('');
 };
 
+module.exports.getQueryActualRowCount = function (query) {
+    return `select COUNT(*) AS rows FROM (${query}) AS __cdb_query`;
+};
+
 function getQueryRowEstimation(query) {
     return 'select CDB_EstimateRowCount($windshaft$' + query + '$windshaft$) as rows';
 }
-module.exports.getQueryRowCount = getQueryRowEstimation;
+
+module.exports.getQueryRowEstimation = getQueryRowEstimation;
 
 module.exports.getAggregationMetadata = ctx => `
     WITH
@@ -83,11 +88,6 @@ module.exports.getQueryTopCategories = function(query, column, topN, includeNull
     `;
 };
 
-module.exports.getQueryActualRowCount = function (query) {
-    return `select COUNT(*) AS rows FROM (${query}) AS __cdb_query`;
-};
-
-
 module.exports.getQuerySample = function(query, sampleProb, randomSeed = 0.5) {
     const table = simpleQueryTable(query);
     if (table) {
@@ -99,7 +99,7 @@ module.exports.getQuerySample = function(query, sampleProb, randomSeed = 0.5) {
         )
         SELECT *
             FROM (${query}) AS __cdb_query
-            WHERE random() < $
+            WHERE random() < ${sampleProb}
     `;
 };