Release 3.3.0

Static maps layers filter

NEWS.md

@@ -1,5 +1,27 @@
 # Changelog
 
+## 3.3.0
+Released 2017-04-03
+
+New features:
+ - Static map endpoints allow specifying the layers to render #653.
+
+
+## 3.2.0
+Released 2017-03-30
+
+Announcements:
+ - Upgrades windshaft to [3.1.0](https://github.com/CartoDB/windshaft/releases/tag/3.1.0).
+ - Active GC interval.
+
+
+## 3.1.1
+Released 2017-03-23
+
+Bug fixes:
+ - Use crc32 instead of md5 for computing subdomain candidate #642
+
+
 ## 3.1.0
 Released 2017-03-22
 

app.js

@@ -136,3 +136,17 @@ process.on('SIGHUP', function() {
 process.on('uncaughtException', function(err) {
     global.logger.error('Uncaught exception: ' + err.stack);
 });
+
+if (global.gc) {
+    var gcInterval = Number.isFinite(global.environment.gc_interval) ?
+        global.environment.gc_interval :
+        10000;
+
+    if (gcInterval > 0) {
+        setInterval(function gcForcedCycle() {
+            var start = Date.now();
+            global.gc();
+            global.statsClient.timing('windshaft.gc', Date.now() - start);
+        }, gcInterval);
+    }
+}

docs/named_maps.md

@@ -124,7 +124,7 @@ view (optional) | extra keys to specify the view area for the map. It can be use
 
 ### Placeholder Format
 
-Placeholders are variables that can be placed in your template.json file. Placeholders need to be defined with a `type` and a default value for MapConfigs. See details about defining a MapConfig `type` for [Layergoup configurations](http://docs.carto.com/carto-engine/maps-api/mapconfig/#layergroup-configurations).
+Placeholders are variables that can be placed in your template.json file. Placeholders need to be defined with a `type` and a default value for MapConfigs. See details about defining a MapConfig `type` for [Layergroup configurations](http://docs.carto.com/carto-engine/maps-api/mapconfig/#layergroup-configurations).
 
 Valid placeholder names start with a letter and can only contain letters, numbers, or underscores. They have to be written between the `<%=` and `%>` strings in order to be replaced inside the Named Maps API.
 
@@ -527,11 +527,11 @@ If you are creating a Torque layer in a Named Map without using the Torque.js li
 
 #### Examples of Named Maps created with CARTO.js
 
-- [Named Map selectors with interaction](http://bl.ocks.org/ohasselblad/515a8af1f99d5e690484)
+- [Named Map selectors with interaction](http://bl.ocks.org/andy-esch/515a8af1f99d5e690484)
 
-- [Named Map with interactivity](http://bl.ocks.org/ohasselblad/d1a45b8ff5e7bd90cd68)
+- [Named Map with interactivity](http://bl.ocks.org/andy-esch/d1a45b8ff5e7bd90cd68)
 
-- [Toggling sublayers in a Named Map](http://bl.ocks.org/ohasselblad/c1a0f4913610eec53cd3)
+- [Toggling sublayers in a Named Map](http://bl.ocks.org/andy-esch/c1a0f4913610eec53cd3)
 
 ## Fetching XYZ Tiles for Named Maps
 

lib/cartodb/api/auth_api.js

@@ -95,9 +95,7 @@ AuthApi.prototype.authorize = function(req, callback) {
             self.authorizedByAPIKey(user, req, this);
         },
         function checkApiKey(err, authorized){
-            if (req.profiler) {
-                req.profiler.done('authorizedByAPIKey');
-            }
+            req.profiler.done('authorizedByAPIKey');
             assert.ifError(err);
 
             // if not authorized by api_key, continue
@@ -131,9 +129,7 @@ AuthApi.prototype.authorize = function(req, callback) {
             }
 
             self.pgConnection.setDBAuth(user, req.params, function(err) {
-                if (req.profiler) {
-                    req.profiler.done('setDBAuth');
-                }
+                req.profiler.done('setDBAuth');
                 callback(err, true); // authorized (or error)
             });
         }

lib/cartodb/controllers/base.js

@@ -36,7 +36,7 @@ function BaseController(authApi, pgConnection) {
 
 module.exports = BaseController;
 
-// jshint maxcomplexity:9
+// jshint maxcomplexity:10
 /**
  * Whitelist input and get database name & default geometry type from
  * subdomain/user metadata held in CartoDB Redis
@@ -61,9 +61,7 @@ BaseController.prototype.req2params = function(req, callback){
         lzmaWorker.decompress(
             lzma,
             function(result) {
-                if (req.profiler) {
-                    req.profiler.done('lzma');
-                }
+                req.profiler.done('lzma');
                 try {
                     delete req.query.lzma;
                     _.extend(req.query, JSON.parse(result));
@@ -77,7 +75,11 @@ BaseController.prototype.req2params = function(req, callback){
         return;
     }
 
-    req.query = _.pick(req.query, REQUEST_QUERY_PARAMS_WHITELIST);
+    var allowedQueryParams = REQUEST_QUERY_PARAMS_WHITELIST;
+    if (Array.isArray(req.context.allowedQueryParams)) {
+        allowedQueryParams = allowedQueryParams.concat(req.context.allowedQueryParams);
+    }
+    req.query = _.pick(req.query, allowedQueryParams);
     req.params = _.extend({}, req.params); // shuffle things as request is a strange array/object
 
     var user = req.context.user;
@@ -115,18 +117,14 @@ BaseController.prototype.req2params = function(req, callback){
     // bring all query values onto req.params object
     _.extend(req.params, req.query);
 
-    if (req.profiler) {
-        req.profiler.done('req2params.setup');
-    }
+    req.profiler.done('req2params.setup');
 
     step(
         function getPrivacy(){
             self.authApi.authorize(req, this);
         },
         function validateAuthorization(err, authorized) {
-            if (req.profiler) {
-                req.profiler.done('authorize');
-            }
+            req.profiler.done('authorize');
             assert.ifError(err);
             if(!authorized) {
                 err = new Error("Sorry, you are unauthorized (permission denied)");
@@ -167,9 +165,7 @@ BaseController.prototype.send = function(req, res, body, status, headers) {
         res.set('X-Served-By-DB-Host', req.params.dbhost);
     }
 
-    if (req.profiler) {
-        res.set('X-Tiler-Profiler', req.profiler.toJSONString());
-    }
+    res.set('X-Tiler-Profiler', req.profiler.toJSONString());
 
     if (headers) {
         res.set(headers);
@@ -187,14 +183,12 @@ BaseController.prototype.send = function(req, res, body, status, headers) {
         res.send(body);
     }
 
-    if (req.profiler) {
-        try {
-            // May throw due to dns, see
-            // See http://github.com/CartoDB/Windshaft/issues/166
-            req.profiler.sendStats();
-        } catch (err) {
-            debug("error sending profiling stats: " + err);
-        }
+    try {
+        // May throw due to dns, see
+        // See http://github.com/CartoDB/Windshaft/issues/166
+        req.profiler.sendStats();
+    } catch (err) {
+        debug("error sending profiling stats: " + err);
     }
 };
 // jshint maxcomplexity:6

lib/cartodb/controllers/layergroup.js

@@ -6,6 +6,7 @@ var BaseController = require('./base');
 
 var cors = require('../middleware/cors');
 var userMiddleware = require('../middleware/user');
+var allowQueryParams = require('../middleware/allow-query-params');
 
 var DataviewBackend = require('../backends/dataview');
 var AnalysisStatusBackend = require('../backends/analysis-status');
@@ -67,11 +68,13 @@ LayergroupController.prototype.register = function(app) {
         this.attributes.bind(this));
 
     app.get(app.base_url_mapconfig +
-        '/static/center/:token/:z/:lat/:lng/:width/:height.:format', cors(), userMiddleware,
+        '/static/center/:token/:z/:lat/:lng/:width/:height.:format',
+        cors(), userMiddleware, allowQueryParams(['layer']),
         this.center.bind(this));
 
     app.get(app.base_url_mapconfig +
-        '/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format', cors(), userMiddleware,
+        '/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format',
+        cors(), userMiddleware, allowQueryParams(['layer']),
         this.bbox.bind(this));
 
     // Undocumented/non-supported API endpoint methods.

lib/cartodb/controllers/map.js

@@ -87,9 +87,7 @@ MapController.prototype.createPost = function(req, res) {
 };
 
 MapController.prototype.instantiate = function(req, res) {
-    if (req.profiler) {
-        req.profiler.start('windshaft-cartodb.instance_template_post');
-    }
+    req.profiler.start('windshaft-cartodb.instance_template_post');
 
     this.instantiateTemplate(req, res, function prepareTemplateParams(callback) {
         if (!req.is('application/json')) {
@@ -100,9 +98,7 @@ MapController.prototype.instantiate = function(req, res) {
 };
 
 MapController.prototype.jsonp = function(req, res) {
-    if (req.profiler) {
-        req.profiler.start('windshaft-cartodb.instance_template_get');
-    }
+    req.profiler.start('windshaft-cartodb.instance_template_get');
 
     this.instantiateTemplate(req, res, function prepareJsonTemplateParams(callback) {
         var err = null;
@@ -303,9 +299,7 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
     // take place before proceeding. Error will be logged
     // asynchronously
     this.metadataBackend.incMapviewCount(username, mapconfig.obj().stat_tag, function(err) {
-        if (req.profiler) {
-            req.profiler.done('incMapviewCount');
-        }
+        req.profiler.done('incMapviewCount');
         if ( err ) {
             global.logger.log("ERROR: failed to increment mapview count for user '" + username + "': " + err);
         }
@@ -328,9 +322,7 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
             QueryTables.getAffectedTablesFromQuery(connection, sql, this);
         },
         function handleAffectedTablesAndLastUpdatedTime(err, result) {
-            if (req.profiler) {
-                req.profiler.done('queryTablesAndLastUpdated');
-            }
+            req.profiler.done('queryTablesAndLastUpdated');
             assert.ifError(err);
             // feed affected tables cache so it can be reused from, for instance, layergroup controller
             self.layergroupAffectedTables.set(dbName, layergroupId, result);

lib/cartodb/controllers/named_maps.js

@@ -8,6 +8,7 @@ var BaseController = require('./base');
 
 var cors = require('../middleware/cors');
 var userMiddleware = require('../middleware/user');
+var allowQueryParams = require('../middleware/allow-query-params');
 
 function NamedMapsController(authApi, pgConnection, namedMapProviderCache, tileBackend, previewBackend,
                              surrogateKeysCache, tablesExtentApi, metadataBackend) {
@@ -31,7 +32,7 @@ NamedMapsController.prototype.register = function(app) {
         this.tile.bind(this));
 
     app.get(app.base_url_mapconfig +
-        '/static/named/:template_id/:width/:height.:format', cors(), userMiddleware,
+        '/static/named/:template_id/:width/:height.:format', cors(), userMiddleware, allowQueryParams(['layer']),
         this.staticMap.bind(this));
 };
 
@@ -100,9 +101,7 @@ NamedMapsController.prototype.tile = function(req, res) {
             self.tileBackend.getTile(namedMapProvider, req.params, this);
         },
         function handleImage(err, tile, headers, stats) {
-            if (req.profiler) {
-                req.profiler.add(stats);
-            }
+            req.profiler.add(stats);
             if (err) {
                 self.sendError(req, res, err, 'NAMED_MAP_TILE');
             } else {
@@ -176,10 +175,8 @@ NamedMapsController.prototype.staticMap = function(req, res) {
             });
         },
         function handleImage(err, image, headers, stats) {
-            if (req.profiler) {
-                req.profiler.done('render-' + format);
-                req.profiler.add(stats || {});
-            }
+            req.profiler.done('render-' + format);
+            req.profiler.add(stats || {});
 
             if (err) {
                 self.sendError(req, res, err, 'STATIC_VIZ_MAP');

lib/cartodb/controllers/named_maps_admin.js

@@ -91,9 +91,7 @@ NamedMapsAdminController.prototype.update = function(req, res) {
 NamedMapsAdminController.prototype.retrieve = function(req, res) {
     var self = this;
 
-    if (req.profiler) {
-        req.profiler.start('windshaft-cartodb.get_template');
-    }
+    req.profiler.start('windshaft-cartodb.get_template');
 
     var cdbuser = req.context.user;
     var tpl_id;
@@ -127,9 +125,7 @@ NamedMapsAdminController.prototype.retrieve = function(req, res) {
 NamedMapsAdminController.prototype.destroy = function(req, res) {
     var self = this;
 
-    if (req.profiler) {
-        req.profiler.start('windshaft-cartodb.delete_template');
-    }
+    req.profiler.start('windshaft-cartodb.delete_template');
 
     var cdbuser = req.context.user;
     var tpl_id;
@@ -154,9 +150,7 @@ NamedMapsAdminController.prototype.destroy = function(req, res) {
 
 NamedMapsAdminController.prototype.list = function(req, res) {
     var self = this;
-    if ( req.profiler ) {
-        req.profiler.start('windshaft-cartodb.get_template_list');
-    }
+    req.profiler.start('windshaft-cartodb.get_template_list');
 
     var cdbuser = req.context.user;
 

lib/cartodb/middleware/allow-query-params.js

@@ -0,0 +1,9 @@
+module.exports = function allowQueryParams(params) {
+    if (!Array.isArray(params)) {
+        throw new Error('allowQueryParams must receive an Array of params');
+    }
+    return function allowQueryParamsMiddleware(req, res, next) {
+        req.context.allowedQueryParams = params;
+        next();
+    };
+};

lib/cartodb/models/resource-locator.js

@@ -1,5 +1,3 @@
-var crypto = require('crypto');
-
 var dot = require('dot');
 dot.templateSettings.strip = false;
 
@@ -91,9 +89,31 @@ function getCdnDomain(serverMetadata, resource) {
     return null;
 }
 
+// ref https://jsperf.com/js-crc32
+function crcTable() {
+    var c;
+    var table = [];
+    for (var n = 0; n < 256; n++) {
+      c = n;
+      for (var k = 0; k < 8; k++) {
+        c = ((c & 1) ? (0xEDB88320 ^ (c >>> 1)) : (c >>> 1));
+      }
+      table[n] = c;
+    }
+    return table;
+}
+var CRC_TABLE = crcTable();
+
+function crc32(str) {
+    var crc = 0 ^ (-1);
+    for (var i = 0; i < str.length; i++) {
+      crc = (crc >>> 8) ^ CRC_TABLE[(crc ^ str.charCodeAt(i)) & 0xFF];
+    }
+    return (crc ^ (-1)) >>> 0;
+}
+
 function subdomain(subdomains, resource) {
-    var resourceHash = crypto.createHash('md5').update(resource, 'binary').digest('hex');
-    var index = parseInt(resourceHash, 16) % subdomains.length;
+    var index = crc32(resource) % subdomains.length;
     return subdomains[index];
 }
-module.exports.subdomain = subdomain;
+module.exports.subdomain = subdomain;

package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "name": "windshaft-cartodb",
-  "version": "3.1.0",
+  "version": "3.3.0",
   "description": "A map tile server for CartoDB",
   "keywords": [
     "cartodb"
@@ -39,7 +39,7 @@
     "step-profiler": "~0.3.0",
     "turbo-carto": "0.19.0",
     "underscore": "~1.6.0",
-    "windshaft": "3.0.1",
+    "windshaft": "3.1.0",
     "yargs": "~5.0.0"
   },
   "devDependencies": {

test/acceptance/named_maps_static_view.js

@@ -21,7 +21,7 @@ describe('named maps static view', function() {
 
     var IMAGE_TOLERANCE = 20;
 
-    function createTemplate(view) {
+    function createTemplate(view, layers) {
         return {
             version: '0.0.1',
             name: templateName,
@@ -36,7 +36,7 @@ describe('named maps static view', function() {
             },
             view: view,
             layergroup: {
-                layers: [
+                layers: layers || [
                     {
                         type: 'mapnik',
                         options: {
@@ -198,4 +198,43 @@ describe('named maps static view', function() {
         });
     });
 
+    it('should allow to select the layers to render', function (done) {
+        var view = {
+            bounds: {
+                west: 0,
+                south: 0,
+                east: 45,
+                north: 45
+            }
+        };
+
+        var layers = [
+            {
+                type: 'mapnik',
+                options: {
+                    sql: 'select * from populated_places_simple_reduced',
+                    cartocss: '#layer { marker-fill: <%= color %>; }',
+                    cartocss_version: '2.3.0'
+                }
+            },
+            {
+                type: 'mapnik',
+                options: {
+                    sql: 'select ST_Transform(ST_MakeEnvelope(-45, -45, 45, 45, 4326), 3857) the_geom_webmercator',
+                    cartocss: '#layer { polygon-fill: <%= color %>; }',
+                    cartocss_version: '2.3.0'
+                }
+            }
+        ];
+        templateMaps.addTemplate(username, createTemplate(view, layers), function (err) {
+            if (err) {
+                return done(err);
+            }
+            getStaticMap({ layer: 0 }, function(err, img) {
+                assert.ok(!err);
+                assert.imageIsSimilarToFile(img, previewFixture('bounds'), IMAGE_TOLERANCE, done);
+            });
+        });
+    });
+
 });

test/unit/cartodb/ported/tile_stats.test.js

@@ -26,12 +26,14 @@ describe('tile stats', function() {
         var layergroupController = new LayergroupController();
 
         var reqMock = {
+            profiler: { toJSONString:function() {} },
             params: {
                 format: invalidFormat
             }
         };
         var resMock = {
             status: function() { return this; },
+            set: function() {},
             json: function() {},
             jsonp: function() {},
             send: function() {}
@@ -54,12 +56,14 @@ describe('tile stats', function() {
             }
         });
         var reqMock = {
+            profiler: { toJSONString:function() {} },
             params: {
                 format: validFormat
             }
         };
         var resMock = {
             status: function() { return this; },
+            set: function() {},
             json: function() {},
             jsonp: function() {},
             send: function() {}

yarn.lock

@@ -223,9 +223,9 @@ carto@0.16.3:
     semver "^5.1.0"
     yargs "^4.2.0"
 
-"carto@github:cartodb/carto#0.15.1-cdb1":
+carto@CartoDB/carto#0.15.1-cdb1:
   version "0.15.1-cdb1"
-  resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398"
+  resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398"
   dependencies:
     mapnik-reference "~6.0.2"
     optimist "~0.6.0"
@@ -794,8 +794,8 @@ graceful-fs@~2.0.0:
   resolved "https://registry.yarnpkg.com/graceful-readlink/-/graceful-readlink-1.0.1.tgz#4cafad76bc62f02fa039b2f94e9a3dd3a391a725"
 
 grainstore@~1.6.0:
-  version "1.6.1"
-  resolved "https://registry.yarnpkg.com/grainstore/-/grainstore-1.6.1.tgz#8950279ea737eb0ce403a85693642b4bed7f8e48"
+  version "1.6.2"
+  resolved "https://registry.yarnpkg.com/grainstore/-/grainstore-1.6.2.tgz#f5d098a8da607f23db08f3263894c8f234a8a59e"
   dependencies:
     carto "0.16.3"
     debug "~2.2.0"
@@ -984,9 +984,9 @@ isarray@~1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11"
 
-isexe@^1.1.1:
-  version "1.1.2"
-  resolved "https://registry.yarnpkg.com/isexe/-/isexe-1.1.2.tgz#36f3e22e60750920f5e7241a476a8c6a42275ad0"
+isexe@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10"
 
 isstream@~0.1.2:
   version "0.1.2"
@@ -1206,15 +1206,15 @@ millstone@0.6.17:
     underscore "~1.6.0"
     zipfile "~0.5.5"
 
-mime-db@~1.26.0:
-  version "1.26.0"
-  resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.26.0.tgz#eaffcd0e4fc6935cf8134da246e2e6c35305adff"
+mime-db@~1.27.0:
+  version "1.27.0"
+  resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.27.0.tgz#820f572296bbd20ec25ed55e5b5de869e5436eb1"
 
 mime-types@^2.1.12, mime-types@~2.1.13, mime-types@~2.1.6, mime-types@~2.1.7:
-  version "2.1.14"
-  resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.14.tgz#f7ef7d97583fcaf3b7d282b6f8b5679dab1e94ee"
+  version "2.1.15"
+  resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.15.tgz#a4ebf5064094569237b8cf70046776d09fc92aed"
   dependencies:
-    mime-db "~1.26.0"
+    mime-db "~1.27.0"
 
 mime@1.3.4, mime@~1.3.4:
   version "1.3.4"
@@ -1509,7 +1509,7 @@ pg-types@1.*:
     postgres-date "~1.0.0"
     postgres-interval "~1.0.0"
 
-"pg@github:cartodb/node-postgres#6.1.2-cdb1":
+pg@cartodb/node-postgres#6.1.2-cdb1:
   version "6.1.2"
   resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3c81aea432ce58d20a795786c58bbb14f68f9689"
   dependencies:
@@ -2170,12 +2170,13 @@ type-is@~1.6.10, type-is@~1.6.6:
     mime-types "~2.1.13"
 
 uglify-js@^2.6:
-  version "2.8.14"
-  resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-2.8.14.tgz#25b15d1af39b21752ee33703adbf432e8bc8f77d"
+  version "2.8.18"
+  resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-2.8.18.tgz#925d14bae48ab62d1883b41afe6e2261662adb8e"
   dependencies:
     source-map "~0.5.1"
-    uglify-to-browserify "~1.0.0"
     yargs "~3.10.0"
+  optionalDependencies:
+    uglify-to-browserify "~1.0.0"
 
 uglify-to-browserify@~1.0.0:
   version "1.0.2"
@@ -2235,10 +2236,10 @@ which-module@^1.0.0:
   resolved "https://registry.yarnpkg.com/which-module/-/which-module-1.0.0.tgz#bba63ca861948994ff307736089e3b96026c2a4f"
 
 which@^1.1.1:
-  version "1.2.12"
-  resolved "https://registry.yarnpkg.com/which/-/which-1.2.12.tgz#de67b5e450269f194909ef23ece4ebe416fa1192"
+  version "1.2.14"
+  resolved "https://registry.yarnpkg.com/which/-/which-1.2.14.tgz#9a87c4378f03e827cecaf1acdf56c736c01c14e5"
   dependencies:
-    isexe "^1.1.1"
+    isexe "^2.0.0"
 
 wide-align@^1.1.0:
   version "1.1.0"
@@ -2254,9 +2255,9 @@ window-size@^0.2.0:
   version "0.2.0"
   resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075"
 
-windshaft@3.0.1:
-  version "3.0.1"
-  resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.0.1.tgz#d06b4673704fe8f8f2e87c1f590c836659ab46f3"
+windshaft@3.1.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.1.0.tgz#dfac2dd27a2db97e35231510743ad7db574da18f"
   dependencies:
     abaculus cartodb/abaculus#2.0.3-cdb1
     canvas cartodb/node-canvas#1.6.2-cdb2