Merge branch 'master' into tilejson-url-apikey

NEWS.md

@@ -3,6 +3,9 @@
 ## 6.2.0
 Released 2018-mm-dd
 
+Notice:
+- This release changes the way that authentication works internally. You'll need to run `bundle exec rake carto:api_key:create_default` in your development environment to keep working.
+
 New features:
 - CI tests with Ubuntu Xenial + PostgreSQL 10.1 and Ubuntu Precise + PostgreSQL 9.5
 - Upgrades Windshaft to [4.8.1](https://github.com/CartoDB/Windshaft/blob/4.8.1/NEWS.md#version-481) which includes:

lib/cartodb/backends/layer-stats/mapnik-layer-stats.js

@@ -1,27 +1,5 @@
-var queryUtils = require('../../utils/query-utils');
+const queryUtils = require('../../utils/query-utils');
 const AggregationMapConfig = require('../../models/aggregation/aggregation-mapconfig');
-var SubstitutionTokens = require('../../utils/substitution-tokens');
-
-// Instantiate a query with tokens for a given zoom level
-function queryForZoom(sql, zoom, singleTile=false) {
-    const tileRes = 256;
-    const wmSize = 6378137.0*2*Math.PI;
-    const nTiles = Math.pow(2, zoom);
-    const tileSize = wmSize / nTiles;
-    const resolution = tileSize / tileRes;
-    const scaleDenominator = resolution / 0.00028;
-    const x0 = -wmSize/2, y0 = -wmSize/2;
-    let bbox = `ST_MakeEnvelope(${x0}, ${y0}, ${x0+wmSize}, ${y0+wmSize})`;
-    if (singleTile) {
-        bbox = `ST_MakeEnvelope(${x0}, ${y0}, ${x0 + tileSize}, ${y0 + tileSize})`;
-    }
-    return SubstitutionTokens.replace(sql, {
-        bbox: bbox,
-        scale_denominator: scaleDenominator,
-        pixel_width: resolution,
-        pixel_height: resolution
-    });
-}
 
 function MapnikLayerStats () {
     this._types = {
@@ -52,7 +30,7 @@ function _getSQL(ctx, query, type='pre', zoom=0) {
     else {
         sql = ctx.aggrQuery;
     }
-    sql = queryForZoom(sql, zoom || 0);
+    sql = queryUtils.subsituteTokensForZoom(sql, zoom || 0);
     return query(sql);
 }
 

lib/cartodb/backends/overviews-metadata.js

@@ -1,4 +1,4 @@
-var SubstitutionTokens = require('../utils/substitution-tokens');
+const queryUtils = require('../utils/query-utils');
 
 function OverviewsMetadataBackend(pgQueryRunner) {
     this.pgQueryRunner = pgQueryRunner;
@@ -6,20 +6,15 @@ function OverviewsMetadataBackend(pgQueryRunner) {
 
 module.exports = OverviewsMetadataBackend;
 
-function prepareSql(sql) {
-    return sql && SubstitutionTokens.replace(sql, {
-        bbox: 'ST_MakeEnvelope(0,0,0,0)',
-        scale_denominator: '0',
-        pixel_width: '1',
-        pixel_height: '1'
-    });
-}
-
 OverviewsMetadataBackend.prototype.getOverviewsMetadata = function (username, sql, callback) {
     // FIXME: Currently using internal function _cdb_schema_name
     //        CDB_Overviews should provide the schema information directly.
-    var query = 'SELECT *, _cdb_schema_name(base_table)' +
-                '  FROM CDB_Overviews(CDB_QueryTablesText($windshaft$' + prepareSql(sql) + '$windshaft$))';
+    const query = `
+      SELECT *, _cdb_schema_name(base_table)
+        FROM CDB_Overviews(
+            CDB_QueryTablesText($windshaft$${queryUtils.substituteDummyTokens(sql)}$windshaft$)
+        );
+    `;
     this.pgQueryRunner.run(username, query, function handleOverviewsRows(err, rows) {
         if (err){
             callback(err);

lib/cartodb/models/aggregation/aggregation-mapconfig.js

@@ -8,19 +8,10 @@ const {
     createAggregationFiltersValidator
 } = aggregationValidator;
 
-const SubstitutionTokens = require('../../utils/substitution-tokens');
+const queryUtils = require('../../utils/query-utils');
 
 const removeDuplicates = arr => [...new Set(arr)];
 
-function prepareSql(sql) {
-    return sql && SubstitutionTokens.replace(sql, {
-        bbox: 'ST_MakeEnvelope(0,0,0,0)',
-        scale_denominator: '0',
-        pixel_width: '1',
-        pixel_height: '1'
-    });
-}
-
 module.exports = class AggregationMapConfig extends MapConfig {
     static get AGGREGATIONS () {
         return aggregationQuery.SUPPORTED_AGGREGATE_FUNCTIONS;
@@ -205,7 +196,7 @@ module.exports = class AggregationMapConfig extends MapConfig {
             }
 
             const sql = limitedQuery({
-                query: prepareSql(layer.options.sql)
+                query: queryUtils.substituteDummyTokens(layer.options.sql)
             });
 
             connection.query(sql, (err, result) => {

lib/cartodb/models/mapconfig/adapter/vector-mapconfig-adapter.js

@@ -46,7 +46,7 @@ class VectorMapConfigAdapter {
                 if (err) {
                     return reject(err);
                 }
-                const query = queryUtils.getQueryLimited(originalQuery, 0);
+                const query = queryUtils.getQueryLimited(queryUtils.substituteDummyTokens(originalQuery), 0);
                 queryUtils.queryPromise(connection, query)
                     .then(resolve)
                     .catch(reject);

lib/cartodb/utils/query-utils.js

@@ -1,3 +1,5 @@
+const SubstitutionTokens = require('./substitution-tokens');
+
 function prepareQuery(sql) {
     var affectedTableRegexCache = {
         bbox: /!bbox!/g,
@@ -176,5 +178,37 @@ function queryPromise(dbConnection, query) {
     });
 }
 
+function substituteDummyTokens(sql) {
+    return sql && SubstitutionTokens.replace(sql, {
+        bbox: 'ST_MakeEnvelope(0,0,0,0)',
+        scale_denominator: '0',
+        pixel_width: '1',
+        pixel_height: '1'
+    });
+}
+
+function subsituteTokensForZoom(sql, zoom, singleTile=false) {
+    const tileRes = 256;
+    const wmSize = 6378137.0*2*Math.PI;
+    const nTiles = Math.pow(2, zoom);
+    const tileSize = wmSize / nTiles;
+    const resolution = tileSize / tileRes;
+    const scaleDenominator = resolution / 0.00028;
+    const x0 = -wmSize/2, y0 = -wmSize/2;
+    let bbox = `ST_MakeEnvelope(${x0}, ${y0}, ${x0+wmSize}, ${y0+wmSize})`;
+    if (singleTile) {
+        bbox = `ST_MakeEnvelope(${x0}, ${y0}, ${x0 + tileSize}, ${y0 + tileSize})`;
+    }
+    return SubstitutionTokens.replace(sql, {
+        bbox: bbox,
+        scale_denominator: scaleDenominator,
+        pixel_width: resolution,
+        pixel_height: resolution
+    });
+}
+
 module.exports.queryPromise = queryPromise;
-module.exports.getQueryLimited = getQueryLimited;
+module.exports.getQueryLimited = getQueryLimited;
+module.exports.substituteDummyTokens = substituteDummyTokens;
+module.exports.subsituteTokensForZoom = subsituteTokensForZoom;
+

test/acceptance/date-wrapping.spec.js

@@ -262,4 +262,55 @@ describe('date-wrapping', () => {
             });
         });
     });
-});
+
+    describe('when sql queries use mapnik tokens', () => {
+        beforeEach(() => {
+            const mapConfig = mapConfigFactory.getVectorMapConfig({
+                layerOptions: [{
+                    dates_as_numbers: true,
+                    additionalColumns: [
+                        '!scale_denominator! AS sc'
+                    ]
+                }]
+            });
+            testClient = new TestClient(mapConfig);
+        });
+
+        afterEach(done => testClient.drain(done));
+
+        it('should work', done => {
+            testClient.getLayergroup(function(err, layergroup) {
+                assert.ifError(err);
+                assert.deepEqual(layergroup.metadata.layers[0].meta.dates_as_numbers, ['date']);
+                done();
+            });
+
+        });
+
+        it('should return correct tiles', done => {
+            testClient.getTile(0, 0, 0, { format: 'mvt' }, (err, res, mvt) => {
+                const expected = [
+                    {
+                        type: 'Feature',
+                        id: 1,
+                        geometry: { type: 'Point', coordinates: [0, 0] },
+                        properties: { cartodb_id: 0, date: 1527810000, sc: 559082000 }
+                    },
+                    {
+                        type: 'Feature',
+                        id: 2,
+                        geometry: { type: 'Point', coordinates: [0, 0] },
+                        properties: { cartodb_id: 1, date: 1527900000, sc: 559082000 }
+                    }
+                ];
+                const actual = JSON.parse(mvt.toGeoJSONSync(0)).features;
+
+                assert.deepEqual(actual, expected);
+                done();
+            });
+        });
+    });
+
+});
+
+

test/fixtures/test_mapconfigFactory.js

@@ -18,16 +18,18 @@ function _generateLayers(opts) {
 }
 
 function _generateLayerConfig(opts) {
+    const additionalColumns = opts.additionalColumns ? opts.additionalColumns.join(',')+',' : '';
     return {
         type: 'mapnik',
         options: {
             sql: `
             SELECT
+                ${additionalColumns}
                 (DATE '2018-06-01' + x) as date,
                 x as cartodb_id,
                 st_makepoint(x * 10, x * 10) as the_geom,
                 st_makepoint(x * 10, x * 10) as the_geom_webmercator
-            FROM 
+            FROM
                 generate_series(0, 1) x`,
             aggregation: {
                 columns: {},