From c7f37047b053700823b43ba3abf2aa75c45758ec Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 19 May 2016 11:41:06 +0200 Subject: [PATCH 1/7] Save original query from analysis before wrapping it --- lib/cartodb/models/analysis-mapconfig-adapter.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/cartodb/models/analysis-mapconfig-adapter.js b/lib/cartodb/models/analysis-mapconfig-adapter.js index 6d54a3af..fa6aee47 100644 --- a/lib/cartodb/models/analysis-mapconfig-adapter.js +++ b/lib/cartodb/models/analysis-mapconfig-adapter.js @@ -145,6 +145,7 @@ AnalysisMapConfigAdapter.prototype.getMapConfig = function(analysisConfiguration var analysisSql = layerQuery(layerNode); var sqlQueryWrap = layer.options.sql_wrap; if (sqlQueryWrap) { + layer.options.sql_raw = analysisSql; analysisSql = sqlQueryWrap.replace(/<%=\s*sql\s*%>/g, analysisSql); } layer.options.sql = analysisSql; From 31344a1c75eb537c34d4dc4e303de1f8c78ffe19 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 19 May 2016 11:42:28 +0200 Subject: [PATCH 2/7] Adds test case with analysis --- test/acceptance/turbo-cartocss/regressions.js | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/test/acceptance/turbo-cartocss/regressions.js b/test/acceptance/turbo-cartocss/regressions.js index 4f152da3..a9d5de52 100644 --- a/test/acceptance/turbo-cartocss/regressions.js +++ b/test/acceptance/turbo-cartocss/regressions.js @@ -102,4 +102,71 @@ describe('turbo-carto regressions', function() { done(); }); }); + + it('should work with mapnik substitution tokens and analyses', function(done) { + var cartocss = [ + "#layer {", + " line-width: 2;", + " line-color: #3B3B58;", + " line-opacity: 1;", + " polygon-opacity: 0.7;", + " polygon-fill: ramp([points_count], (#E5F5F9,#99D8C9,#2CA25F))", + "}" + ].join('\n'); + + console.log(cartocss); + + var sqlWrap = [ + 'WITH hgrid AS (', + ' SELECT CDB_HexagonGrid(', + ' ST_Expand(!bbox!, greatest(!pixel_width!,!pixel_height!) * 100),', + ' greatest(!pixel_width!,!pixel_height!) * 100', + ' ) as cell', + ')', + 'SELECT', + ' hgrid.cell as the_geom_webmercator,', + ' count(1) as points_count,', + ' count(1)/power(100 * CDB_XYZ_Resolution(CDB_ZoomFromScale(!scale_denominator!)), 2) as points_density,', + ' 1 as cartodb_id', + 'FROM hgrid, (<%= sql %>) i', + 'where ST_Intersects(i.the_geom_webmercator, hgrid.cell)', + 'GROUP BY hgrid.cell' + ].join('\n'); + + var mapConfig = { + "version": "1.5.0", + "layers": [ + { + "type": 'mapnik', + "options": { + "cartocss_version": '2.3.0', + "source": { + "id": "head" + }, + sql_wrap: sqlWrap, + "cartocss": cartocss + } + } + ], + "analyses": [ + { + "id": "head", + "type": "source", + "params": { + "query": "SELECT * FROM car_log_nurburgring_2013_05_30" + } + } + ] + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getLayergroup(function(err, layergroup) { + assert.ok(!err, err); + + assert.ok(layergroup.hasOwnProperty('layergroupid')); + assert.ok(!layergroup.hasOwnProperty('errors')); + + done(); + }); + }); }); From 573932efba82b4e52d0df5eabee7254a41e5fcb1 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 19 May 2016 11:48:57 +0200 Subject: [PATCH 3/7] Simplify condition and use positive naming for parsing cartocss --- lib/cartodb/utils/style/turbo-carto-adapter.js | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/lib/cartodb/utils/style/turbo-carto-adapter.js b/lib/cartodb/utils/style/turbo-carto-adapter.js index d1c0c81f..fa754835 100644 --- a/lib/cartodb/utils/style/turbo-carto-adapter.js +++ b/lib/cartodb/utils/style/turbo-carto-adapter.js @@ -32,10 +32,8 @@ TurboCartoAdapter.prototype.getLayers = function (username, layers, callback) { }; TurboCartoAdapter.prototype._parseCartoCss = function (username, layer, callback) { - if (isNotLayerToParseCartocss(layer)) { - return process.nextTick(function () { - callback(null, layer); - }); + if (!shouldParseLayerCartocss(layer)) { + return callback(null, layer); } var sql = SubstitutionTokens.replace(layer.options.sql, { @@ -61,10 +59,6 @@ TurboCartoAdapter.prototype._parseCartoCss = function (username, layer, callback }); }; -function isNotLayerToParseCartocss(layer) { - if (!layer || !layer.options || !layer.options.cartocss || !layer.options.sql) { - return true; - } - - return false; +function shouldParseLayerCartocss(layer) { + return layer && layer.options && layer.options.cartocss && layer.options.sql; } From ba08745c2346d7eb9ba1b842e683c4f8a9d26009 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 19 May 2016 12:10:19 +0200 Subject: [PATCH 4/7] Adds hasTokens method to SubstitutionTokens --- lib/cartodb/utils/substitution-tokens.js | 10 +++++ test/unit/cartodb/substitution-tokens.test.js | 40 +++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 test/unit/cartodb/substitution-tokens.test.js diff --git a/lib/cartodb/utils/substitution-tokens.js b/lib/cartodb/utils/substitution-tokens.js index af09f838..5905fa4d 100644 --- a/lib/cartodb/utils/substitution-tokens.js +++ b/lib/cartodb/utils/substitution-tokens.js @@ -6,6 +6,16 @@ var SUBSTITUTION_TOKENS = { }; var SubstitutionTokens = { + tokens: function(sql) { + return Object.keys(SUBSTITUTION_TOKENS).filter(function(tokenName) { + return !!sql.match(SUBSTITUTION_TOKENS[tokenName]); + }); + }, + + hasTokens: function(sql) { + return this.tokens(sql).length > 0; + }, + replace: function(sql, replaceValues) { Object.keys(replaceValues).forEach(function(token) { if (SUBSTITUTION_TOKENS[token]) { diff --git a/test/unit/cartodb/substitution-tokens.test.js b/test/unit/cartodb/substitution-tokens.test.js new file mode 100644 index 00000000..70c3a0d0 --- /dev/null +++ b/test/unit/cartodb/substitution-tokens.test.js @@ -0,0 +1,40 @@ +var assert = require('assert'); +var SubstitutionTokens = require('../../../lib/cartodb/utils/substitution-tokens'); + +describe('SubstitutionTokens', function() { + + var sql = [ + 'WITH hgrid AS (', + ' SELECT CDB_HexagonGrid(', + ' ST_Expand(!bbox!, greatest(!pixel_width!,!pixel_height!) * 100),', + ' greatest(!pixel_width!,!pixel_height!) * 100', + ' ) as cell', + ')', + 'SELECT', + ' hgrid.cell as the_geom_webmercator,', + ' count(1) as points_count,', + ' count(1)/power(100 * CDB_XYZ_Resolution(CDB_ZoomFromScale(!scale_denominator!)), 2) as points_density,', + ' 1 as cartodb_id', + 'FROM hgrid, (select * from table) i', + 'where ST_Intersects(i.the_geom_webmercator, hgrid.cell)', + 'GROUP BY hgrid.cell' + ].join('\n'); + + it('should return tokens present in sql', function() { + assert.deepEqual(SubstitutionTokens.tokens(sql), ['bbox', 'scale_denominator', 'pixel_width', 'pixel_height']); + }); + + it('should return just one token', function() { + assert.deepEqual(SubstitutionTokens.tokens('select !bbox! from wadus'), ['bbox']); + }); + + it('should not return other tokens', function() { + assert.deepEqual(SubstitutionTokens.tokens('select !wadus! from wadus'), []); + }); + + it('should report sql has tokens', function() { + assert.equal(SubstitutionTokens.hasTokens(sql), true); + assert.equal(SubstitutionTokens.hasTokens('select !bbox! from wadus'), true); + assert.equal(SubstitutionTokens.hasTokens('select !wadus! from wadus'), false); + }); +}); From fd3f928d81e71f43479f1d7db39100ecc6e23148 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 19 May 2016 12:13:37 +0200 Subject: [PATCH 5/7] Fix test table --- test/acceptance/turbo-cartocss/regressions.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/turbo-cartocss/regressions.js b/test/acceptance/turbo-cartocss/regressions.js index a9d5de52..51df3847 100644 --- a/test/acceptance/turbo-cartocss/regressions.js +++ b/test/acceptance/turbo-cartocss/regressions.js @@ -153,7 +153,7 @@ describe('turbo-carto regressions', function() { "id": "head", "type": "source", "params": { - "query": "SELECT * FROM car_log_nurburgring_2013_05_30" + "query": "SELECT * FROM populated_places_simple_reduced" } } ] From ee94b8a587f2d60413a90073a836d80d9cb3a6a8 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 19 May 2016 12:13:49 +0200 Subject: [PATCH 6/7] Very raw implementation of SubstitutionTokens based on origin data --- .../utils/style/turbo-carto-adapter.js | 38 ++++++++++++++++++- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/utils/style/turbo-carto-adapter.js b/lib/cartodb/utils/style/turbo-carto-adapter.js index fa754835..0c86d0c8 100644 --- a/lib/cartodb/utils/style/turbo-carto-adapter.js +++ b/lib/cartodb/utils/style/turbo-carto-adapter.js @@ -1,5 +1,7 @@ 'use strict'; +var dot = require('dot'); +dot.templateSettings.strip = false; var queue = require('queue-async'); var SubstitutionTokens = require('../substitution-tokens'); @@ -31,17 +33,49 @@ TurboCartoAdapter.prototype.getLayers = function (username, layers, callback) { }); }; +var bboxTemplate = dot.template('(select ST_SetSRID(st_extent(the_geom_webmercator), 3857) from ({{=it._sql}}) __c)'); +var zoomTemplate = dot.template([ + 'GREATEST(', + 'ceil(log(40075017000 / 256 / GREATEST(', + ' st_xmax({{=it._bbox}}) - st_xmin({{=it._bbox}}),', + ' st_ymax({{=it._bbox}}) - st_ymin({{=it._bbox}})', + '))/log(2)),', + '0', + ')' +].join('\n')); +var pixelSizeTemplate = dot.template('40075017 * cos(ST_Y(ST_Centroid({{=it._bbox}}))) / 2 ^ (({{=it._zoom}}) + 8)'); +var scaleDenominatorTemplate = dot.template('({{=it._pixelSize}} / 0.00028)::numeric'); + + TurboCartoAdapter.prototype._parseCartoCss = function (username, layer, callback) { if (!shouldParseLayerCartocss(layer)) { return callback(null, layer); } - var sql = SubstitutionTokens.replace(layer.options.sql, { + var tokens = { bbox: 'ST_MakeEnvelope(-20037508.34,-20037508.34,20037508.34,20037508.34,3857)', scale_denominator: '500000001', pixel_width: '156412', pixel_height: '156412' - }); + }; + + var layerSql = layer.options.sql; + var layerRawSql = layer.options.sql_raw; + if (SubstitutionTokens.hasTokens(layerSql) && layerRawSql) { + var bbox = bboxTemplate({ _sql: layerRawSql }); + var zoom = zoomTemplate({ _bbox: bbox }); + var pixelSize = pixelSizeTemplate({ _bbox: bbox, _zoom: zoom }); + var scaleDenominator = scaleDenominatorTemplate({ _pixelSize: pixelSize }); + + tokens = { + bbox: bbox, + scale_denominator: scaleDenominator, + pixel_width: pixelSize, + pixel_height: pixelSize + }; + } + + var sql = SubstitutionTokens.replace(layerSql, tokens); this.turboCartoParser.process(username, layer.options.cartocss, sql, function (err, cartocss) { // Only return turbo-carto errors From f30be00eb9231503cb9b45e948eff2925f591311 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 19 May 2016 12:14:46 +0200 Subject: [PATCH 7/7] Remove console --- test/acceptance/turbo-cartocss/regressions.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/acceptance/turbo-cartocss/regressions.js b/test/acceptance/turbo-cartocss/regressions.js index 51df3847..e2dd38cc 100644 --- a/test/acceptance/turbo-cartocss/regressions.js +++ b/test/acceptance/turbo-cartocss/regressions.js @@ -114,8 +114,6 @@ describe('turbo-carto regressions', function() { "}" ].join('\n'); - console.log(cartocss); - var sqlWrap = [ 'WITH hgrid AS (', ' SELECT CDB_HexagonGrid(',