Release 1.17.2

TTL for template locks so they are not kept forever

.travis.yml

@@ -4,11 +4,13 @@ before_install:
   - sudo apt-add-repository --yes ppa:cartodb/postgresql-9.3
   - sudo apt-add-repository --yes ppa:cartodb/gis
   - sudo rm -Rf /var/lib/postgresql /etc/postgresql
-  - sudo apt-add-repository --yes ppa:mapnik/v2.1.0
+  - sudo apt-add-repository --yes ppa:mapnik/nightly-2.3
   - sudo apt-get update
   - sudo apt-get install -y postgresql-9.3-postgis-2.1
   - sudo apt-get install -y postgresql-contrib-9.3
-  - sudo apt-get install -y libmapnik-dev
+  - sudo apt-get install -q libprotobuf-dev protobuf-compiler
+  - sudo apt-get install -q libmapnik-dev
+  - sudo apt-get install -q mapnik-input-plugin-gdal mapnik-input-plugin-ogr mapnik-input-plugin-postgis
   - sudo apt-get install -y gdal-bin
   - echo -e "local\tall\tall\ttrust\nhost\tall\tall\t127.0.0.1/32\ttrust\nhost\tall\tall\t::1/128\ttrust" |sudo tee /etc/postgresql/9.3/main/pg_hba.conf
   - sudo service postgresql restart

NEWS.md

@@ -1,3 +1,67 @@
+1.17.2 -- 2014-10-01
+--------------------
+
+Announcements:
+ - Upgrades windshaft to 0.27.2 which downgrades node-mapnik to 0.7.26-cdb1
+
+
+1.17.1 -- 2014-09-30
+--------------------
+
+Announcements:
+ - Upgrades windshaft to 0.27.1 which downgrades node-mapnik to 1.4.10
+
+Enhancements:
+ - TTL for template locks so they are not kept forever
+ - Upgrades mocha
+
+
+1.17.0 -- 2014-09-25
+--------------------
+
+New features:
+ - Starts using mapnik 2.3.x
+
+Enhancements:
+ - Upgrades windshaft and cartodb-redis
+ - Supports `!scale_denominator!` dynamic param in SQL queries
+ - Metrics revamp: removes and adds some metrics
+ - Adds poolSize configuration for mapnik
+
+1.16.1 -- 2014-08-19
+--------------------
+
+Enhancements:
+ - Upgrades cartodb-redis
+
+1.16.0 -- 2014-08-18
+--------------------
+
+New features:
+ - Configurable QueryTablesAPI to call directly postgresql using cartodb-psql
+   or to keep using a request to the SQL API
+
+Enhancements:
+ - Removes mapnik dependency as it now relies on Windshaft to check mapnik version
+ - Upgrades dependencies:
+    - underscore
+    - lzma
+    - log4js
+    - rollbar
+    - windshaft
+    - request
+
+1.15.0 -- 2014-08-13
+--------------------
+Enhancements:
+ - Upgrades dependencies:
+    - redis-mpool
+    - cartodb-redis
+    - windshaft
+ - Specifies name in the redis pool
+ - Slow pool configuration in example configurations
+
+
 1.14.0 -- 2014-08-07
 --------------------
 
@@ -5,8 +69,10 @@ Enhancements:
  - SQL API requests moved to its own entity
 
 New features:
- - Affected tables and last updated time for a query are performed in a single request to the SQL API
- - Allow specifying the tile format, upgrades windshaft and grainstore dependencies for this matter
+ - Affected tables and last updated time for a query are performed in a single
+   request to the SQL API
+ - Allow specifying the tile format, upgrades windshaft and grainstore
+   dependencies for this matter
 
 
 1.13.1 -- 2014-08-04

config/environments/development.js.example

@@ -103,7 +103,11 @@ var config = {
         // kept open by the server. The default is 50.
         max: 50,
         idleTimeoutMillis: 1, // idle time before dropping connection
-        reapIntervalMillis: 1 // time between cleanups
+        reapIntervalMillis: 1, // time between cleanups
+        slowPool: {
+            log: true, // whether a slow acquire must be logged or not
+            elapsedThreshold: 25 // the threshold to determine an slow acquire must be reported or not
+        }
     }
     ,sqlapi: {
         protocol: 'http',

config/environments/production.js.example

@@ -97,7 +97,11 @@ var config = {
         // kept open by the server. The default is 50.
         max: 50,
         idleTimeoutMillis: 30000, // idle time before dropping connection
-        reapIntervalMillis: 1000 // time between cleanups
+        reapIntervalMillis: 1000, // time between cleanups
+        slowPool: {
+            log: true, // whether a slow acquire must be logged or not
+            elapsedThreshold: 25 // the threshold to determine an slow acquire must be reported or not
+        }
     }
     ,sqlapi: {
         protocol: 'https',

config/environments/staging.js.example

@@ -97,7 +97,11 @@ var config = {
         // kept open by the server. The default is 50.
         max: 50,
         idleTimeoutMillis: 30000, // idle time before dropping connection
-        reapIntervalMillis: 1000 // time between cleanups
+        reapIntervalMillis: 1000, // time between cleanups
+        slowPool: {
+            log: true, // whether a slow acquire must be logged or not
+            elapsedThreshold: 25 // the threshold to determine an slow acquire must be reported or not
+        }
     }
     ,sqlapi: {
         protocol: 'https',

config/environments/test.js.example

@@ -97,7 +97,11 @@ var config = {
         // kept open by the server. The default is 50.
         max: 50,
         idleTimeoutMillis: 1, // idle time before dropping connection
-        reapIntervalMillis: 1 // time between cleanups
+        reapIntervalMillis: 1, // time between cleanups
+        slowPool: {
+            log: true, // whether a slow acquire must be logged or not
+            elapsedThreshold: 25 // the threshold to determine an slow acquire must be reported or not
+        }
     }
     ,sqlapi: {
         protocol: 'http',

docs/Map-API.md

@@ -51,7 +51,7 @@ The following API call creates a map of European countries that have a white fil
 {% highlight javascript %}
 // mapconfig.json
 {
-  "version": "0.0.1"
+  "version": "0.0.1",
   "name": "test",
   "auth": {
     "method": "open"
@@ -292,53 +292,52 @@ POST /api/v1/map/named
 
 #### Params
 
+<div class="code-title">template.json</div>
 {% highlight javascript %}
-// template.json 
 {
-  version: '0.0.1',
-  // there can be at most 1 template with the same name for any user 
-  // valid names start with a letter and only contains letter, numbers
-  // or underscores
-  name: 'template_name', 
-
-  auth: {
-   method: 'token', // or "open" (the default if no "method" is given)
-   valid_tokens: ['auth_token1','auth_token2'] // only (required and non empty) for 'token' method
+  "version": "0.0.1",
+  "name": "template_name",
+  "auth": {
+    "method": "token",
+    "valid_tokens": [
+      "auth_token1",
+      "auth_token2"
+    ]
   },
-
-  // Variables not listed here are not substituted
-  // Variable not provided at instantiation time trigger an error
-  // A default is required for optional variables
-  // Type specification is used for quoting, to avoid injections
-  // see template format section below
-  placeholders: {
-    color: {
-      type:'css_color',
-      default:'red'
+  "placeholders": {
+    "color": {
+      "type": "css_color",
+      "default": "red"
     },
-    cartodb_id: {
-      type:'number',
-      default: 1
+    "cartodb_id": {
+      "type": "number",
+      "default": 1
     }
   },
-
-  // the layer list definition
-  layergroup: {
-    // this is the MapConfig explained in anonymous maps
-    // see https://github.com/CartoDB/Windshaft/blob/0.19.1/doc/MapConfig-1.1.0.md)
+  "layergroup": {
     "version": "1.0.1",
-    "layers": [{
-      "type": "cartodb",
-      "options": {
-        "cartocss_version": "2.1.1", 
-        "cartocss": "#layer { polygon-fill: <%= color %>; }",
-        "sql": "select * from european_countries_e WHERE cartodb_id = <%= cartodb_id %>"
+    "layers": [
+      {
+        "type": "cartodb",
+        "options": {
+          "cartocss_version": "2.1.1",
+          "cartocss": "#layer { polygon-fill: <%= color %>; }",
+          "sql": "select * from european_countries_e WHERE cartodb_id = <%= cartodb_id %>"
+        }
       }
-    }]
+    ]
   }
 }
 {% endhighlight %}
 
+##### Arguments
+
+- **name**: there can be at most 1 template with the same name for any user valid names start with a letter and only contains letter, numbers or underscores
+- **auth**:
+  - **method** `"token"` or `"open"` (the default if no `"method"` is given)
+- **placeholders**: Variables not listed here are not substituted. Variable not provided at instantiation time trigger an error. A default is required for optional variables. Type specification is used for quoting, to avoid injections see template format section below.
+- **layergroup**: the layer list definition. This is the MapConfig explained in anonymous maps see https://github.com/CartoDB/Windshaft/blob/master/doc/MapConfig-1.1.0.md
+
 #### Template Format
 
 A templated `layergroup` allows using placeholders in the "cartocss" and "sql" elements of the "option" object in any "layer" of a layergroup configuration

docs/metrics.md

@@ -4,8 +4,6 @@ See [Windshaft metrics documentation](https://github.com/CartoDB/Windshaft/blob/
 
 The next list includes the API endpoints, each endpoint may have several inner timers, some of them are displayed within this list as subitems. Find the description for them in the Inner timers section.
 ## Timers
-- **windshaft-cartodb.get_infowindow**: time to retrieve an infowindow popup
-- **windshaft-cartodb.get_map_metadata**: time to retrieve metadata for embedded maps
 - **windshaft-cartodb.flush_cache**: time to flush the tile and sql cache
 - **windshaft-cartodb.get_template**: time to retrieve an specific template
 - **windshaft-cartodb.delete_template**: time to delete an specific template
@@ -30,8 +28,6 @@ Again, each inner timer may have several inner timers.
 - **authorizedByAPIKey**: time to authorize using an API KEY
 - **authorizedByCert**: time to authorize a request by a cert, see [signed map](https://github.com/CartoDB/Windshaft-cartodb/wiki/Signed-maps)
 - **authorizedBySigner**: time to authorize a request for a [signed map](https://github.com/CartoDB/Windshaft-cartodb/wiki/Signed-maps)
-- **cartoData.getTableGeometryType**: time to retrieve from redis the geom type for a given table
-- **cors**: time to set the CORS headers
 - **findLastUpdated**: time to retrieve the last update time for a list of tables, see *affectedTables*
 - **fingerPrint**: time to create a fingerprint for a signed map
 - **generateCacheChannel**: time to generate the headers for the cache channel based on the request, see *addCacheChannel*

lib/cartodb/api/query_tables_api.js

@@ -1,10 +1,12 @@
-var sqlApi = require('../sql/sql_api');
+var sqlApi = require('../sql/sql_api'),
+    PSQL   = require('cartodb-psql');
 
 function QueryTablesApi() {
 }
 
 var affectedTableRegexCache = {
     bbox: /!bbox!/g,
+    scale_denominator: /!scale_denominator!/g,
     pixel_width: /!pixel_width!/g,
     pixel_height: /!pixel_height!/g
 };
@@ -33,63 +35,87 @@ QueryTablesApi.prototype.getLastUpdatedTime = function (username, api_key, table
     });
 };
 
-QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, api_key, sql, callback) {
-    // Replace mapnik tokens
-    sql = sql
-        .replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)')
-        .replace(affectedTableRegexCache.pixel_width, '1')
-        .replace(affectedTableRegexCache.pixel_height, '1')
-    ;
+QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, options, sql, callback) {
 
-    // Pass to CDB_QueryTables
-    sql = 'SELECT CDB_QueryTables($windshaft$' + sql + '$windshaft$)';
+    var query = 'SELECT CDB_QueryTables($windshaft$' + prepareSql(sql) + '$windshaft$)';
 
-    // call sql api
-    sqlApi.query(username, api_key, sql, function(err, rows){
-        if (err){
-            var msg = err.message ? err.message : err;
-            callback(new Error('could not fetch source tables: ' + msg));
-            return;
-        }
-        var qtables = rows[0].cdb_querytables;
-        var tableNames = qtables.split(/^\{(.*)\}$/)[1];
-        tableNames = tableNames ? tableNames.split(',') : [];
-        callback(null, tableNames);
-    });
+    runQuery(username, options, query, handleAffectedTablesInQueryRows, callback);
 };
 
-QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (username, api_key, sql, callback) {
-    sql = sql
-        .replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)')
-        .replace(affectedTableRegexCache.pixel_width, '1')
-        .replace(affectedTableRegexCache.pixel_height, '1')
-    ;
+function handleAffectedTablesInQueryRows(err, rows, callback) {
+    if (err){
+        var msg = err.message ? err.message : err;
+        callback(new Error('could not fetch source tables: ' + msg));
+        return;
+    }
+    var qtables = rows[0].cdb_querytables;
+    var tableNames = qtables.split(/^\{(.*)\}$/)[1];
+    tableNames = tableNames ? tableNames.split(',') : [];
+    callback(null, tableNames);
+}
+
+QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (username, options, sql, callback) {
 
     var query = [
-        'SELECT',
-            'CDB_QueryTables($windshaft$' + sql + '$windshaft$) as tablenames,',
-        'EXTRACT(EPOCH FROM max(updated_at)) as max',
+        'WITH querytables AS (',
+            'SELECT * FROM CDB_QueryTables($windshaft$' + prepareSql(sql) + '$windshaft$) as tablenames',
+        ')',
+        'SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max',
         'FROM CDB_TableMetadata m',
-            'WHERE m.tabname = any (CDB_QueryTables($windshaft$' + sql + '$windshaft$)::regclass[])'
+        'WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])'
     ].join(' ');
 
-    sqlApi.query(username, api_key, query, function(err, rows){
-        if (err || rows.length === 0) {
-            var msg = err.message ? err.message : err;
-            callback(new Error('could not fetch affected tables and last updated time: ' + msg));
-            return;
-        }
-
-        var result = rows[0];
-
-        var tableNames = result.tablenames.split(/^\{(.*)\}$/)[1];
-        tableNames = tableNames ? tableNames.split(',') : [];
-
-        var lastUpdatedTime = result.max || 0;
-
-        callback(null, {
-            affectedTables: tableNames,
-            lastUpdatedTime: lastUpdatedTime * 1000
-        });
-    });
+    runQuery(username, options, query, handleAffectedTablesAndLastUpdatedTimeRows, callback);
 };
+
+function handleAffectedTablesAndLastUpdatedTimeRows(err, rows, callback) {
+    if (err || rows.length === 0) {
+        var msg = err.message ? err.message : err;
+        callback(new Error('could not fetch affected tables and last updated time: ' + msg));
+        return;
+    }
+
+    var result = rows[0];
+
+    var tableNames = result.tablenames.split(/^\{(.*)\}$/)[1];
+    tableNames = tableNames ? tableNames.split(',') : [];
+
+    var lastUpdatedTime = result.max || 0;
+
+    callback(null, {
+        affectedTables: tableNames,
+        lastUpdatedTime: lastUpdatedTime * 1000
+    });
+}
+
+
+function runQuery(username, options, query, queryHandler, callback) {
+    if (shouldQueryPostgresDirectly()) {
+        var psql = new PSQL(options);
+        psql.query(query, function(err, resultSet) {
+            var rows = resultSet.rows || [];
+            queryHandler(err, rows, callback);
+        });
+    } else {
+        sqlApi.query(username, options.api_key, query, function(err, rows) {
+            queryHandler(err, rows, callback);
+        });
+    }
+}
+
+
+function prepareSql(sql) {
+    return sql
+        .replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)')
+        .replace(affectedTableRegexCache.scale_denominator, '0')
+        .replace(affectedTableRegexCache.pixel_width, '1')
+        .replace(affectedTableRegexCache.pixel_height, '1')
+    ;
+}
+
+
+function shouldQueryPostgresDirectly() {
+    return global.environment
+        && global.environment.enabledFeatures
+        && global.environment.enabledFeatures.cdbQueryTablesFromPostgres;
+}

lib/cartodb/cartodb_windshaft.js

@@ -2,7 +2,7 @@
 var _ = require('underscore')
     , Step       = require('step')
     , Windshaft = require('windshaft')
-    , redisPool = require('redis-mpool')(global.environment.redis)
+    , redisPool = require('redis-mpool')(_.extend(global.environment.redis, {name: 'windshaft:cartodb'}))
     // TODO: instanciate cartoData with redisPool
     , cartoData  = require('cartodb-redis')(global.environment.redis)
     , SignedMaps = require('./signed_maps.js')
@@ -137,9 +137,6 @@ var CartodbWindshaft = function(serverOptions) {
      * Helper to allow access to the layer to be used in the maps infowindow popup.
      */
     ws.get(serverOptions.base_url + '/infowindow', function(req, res){
-        if ( req.profiler && req.profiler.statsd_client ) {
-          req.profiler.start('windshaft-cartodb.get_infowindow');
-        }
         ws.doCORS(res);
         Step(
             function(){
@@ -161,9 +158,6 @@ var CartodbWindshaft = function(serverOptions) {
      * Helper to allow access to metadata to be used in embedded maps.
      */
     ws.get(serverOptions.base_url + '/map_metadata', function(req, res){
-        if ( req.profiler && req.profiler.statsd_client ) {
-          req.profiler.start('windshaft-cartodb.get_map_metadata');
-        }
         ws.doCORS(res);
         Step(
             function(){
@@ -492,7 +486,6 @@ var CartodbWindshaft = function(serverOptions) {
     // Instantiate a template
     function instanciateTemplate(req, res, template_params, callback) {
       ws.doCORS(res);
-      if ( req.profiler ) req.profiler.done('cors');
       var that = this;
       var response = {};
       var template;

lib/cartodb/server_options.js

@@ -3,9 +3,8 @@ var   _          = require('underscore')
     , cartoData  = require('cartodb-redis')(global.environment.redis)
     , Cache      = require('./cache_validator')
     , QueryTablesApi   = require('./api/query_tables_api')
-    , mapnik     = require('mapnik')
     , crypto     = require('crypto')
-    , LZMA       = require('lzma/lzma_worker.js').LZMA
+    , LZMA       = require('lzma').LZMA;
   ;
 
 // This is for backward compatibility with 1.3.3
@@ -19,6 +18,8 @@ if ( _.isUndefined(global.environment.sqlapi.domain) ) {
 
 module.exports = function(){
 
+    var lzmaWorker = new LZMA();
+
     var queryTablesApi = new QueryTablesApi();
 
     var rendererConfig = _.defaults(global.environment.renderer || {}, {
@@ -48,12 +49,13 @@ module.exports = function(){
           },
           datasource: global.environment.postgres,
           cachedir: global.environment.millstone.cache_basedir,
-          mapnik_version: global.environment.mapnik_version || mapnik.versions.mapnik,
+          mapnik_version: global.environment.mapnik_version,
           mapnik_tile_format: global.environment.mapnik_tile_format || 'png',
           default_layergroup_ttl: global.environment.mapConfigTTL || 7200, 
           gc_prob: 0.01 // @deprecated since Windshaft-1.8.0
         },
         mapnik: {
+          poolSize: rendererConfig.poolSize,
           metatile: rendererConfig.metatile,
           bufferSize: rendererConfig.bufferSize
         },
@@ -75,14 +77,6 @@ module.exports = function(){
     // See http://github.com/CartoDB/Windshaft-cartodb/issues/161
     me.redis.unwatchOnRelease = false;
 
-    // Be nice and warn if configured mapnik version
-    // is != instaled mapnik version
-    if ( mapnik.versions.mapnik != me.grainstore.mapnik_version ) {
-      console.warn("WARNING: detected mapnik version ("
-        + mapnik.versions.mapnik + ") != configured mapnik version ("
-        + me.grainstore.mapnik_version + ")");
-    }
-
 /* This whole block is about generating X-Cache-Channel { */
 
     // TODO: review lifetime of elements of this cache
@@ -192,7 +186,14 @@ module.exports = function(){
                 if ( req.profiler ) req.profiler.done('getSignerMapKey');
                 key = data;
               }
-              queryTablesApi.getAffectedTablesInQuery(user, key, sql, this); // in addCacheChannel
+              queryTablesApi.getAffectedTablesInQuery(user, {
+                  user: req.params.dbuser,
+                  pass: req.params.dbpass,
+                  host: req.params.dbhost,
+                  port: req.params.dbport,
+                  dbname: req.params.dbname,
+                  api_key: key
+              }, sql, this); // in addCacheChannel
             },
             function finish(err, data) {
               next(err,data);
@@ -317,7 +318,14 @@ module.exports = function(){
 
         Step(
             function getAffectedTablesAndLastUpdatedTime() {
-                queryTablesApi.getAffectedTablesAndLastUpdatedTime(usr, key, sql, this);
+                queryTablesApi.getAffectedTablesAndLastUpdatedTime(usr, {
+                    user: req.params.dbuser,
+                    pass: req.params.dbpass,
+                    host: req.params.dbhost,
+                    port: req.params.dbport,
+                    dbname: req.params.dbname,
+                    api_key: key
+                }, sql, this);
             },
             function handleAffectedTablesAndLastUpdatedTime(err, result) {
                 if (req.profiler) req.profiler.done('queryTablesAndLastUpdated');
@@ -625,7 +633,7 @@ module.exports = function(){
           var lzma = (new Buffer(req.query.lzma, 'base64').toString('binary')).split('').map(function(c) { return c.charCodeAt(0) - 128 });
 
           // Decompress
-          LZMA.decompress(
+          lzmaWorker.decompress(
             lzma,
             function(result) {
               if (req.profiler) req.profiler.done('LZMA decompress');
@@ -711,7 +719,6 @@ module.exports = function(){
                 cartoData.getTableGeometryType(req.params.dbname, req.params.table, this);
             },
             function finishSetup(err, data){
-                if (req.profiler) req.profiler.done('cartoData.getTableGeometryType');
                 if ( err ) { callback(err, req); return; }
 
                 if (!_.isNull(data))

lib/cartodb/template_maps.js

@@ -1,10 +1,7 @@
-var crypto    = require('crypto');
-var Step      = require('step');
-var _         = require('underscore');
-
-// Templates in this hash (keyed as <username>@<template_name>)
-// are being worked on.
-var user_template_locks = {};
+var crypto  = require('crypto'),
+    Step    = require('step'),
+    _       = require('underscore'),
+    dot     = require('dot');
 
 // Class handling map templates
 //
@@ -38,17 +35,18 @@ function TemplateMaps(redis_pool, signed_maps, opts) {
   // 
   // We have the following datastores:
   //
-  //  1. User teplates: set of per-user map templates
+  //  1. User templates: set of per-user map templates
   //     NOTE: each template would have an associated auth
   //           reference, see signed_maps.js
 
   // User templates (HASH:tpl_id->tpl_val)
-  this.key_usr_tpl = "map_tpl|<%= owner %>";
+  this.key_usr_tpl = dot.template("map_tpl|{{=it.owner}}");
 
   // User template locks (HASH:tpl_id->ctime)
-  this.key_usr_tpl_lck = "map_tpl|<%= owner %>|locks";
+  this.key_usr_tpl_lck = dot.template("map_tpl|{{=it.owner}}|locks");
 
-};
+  this.lock_ttl = this.opts['lock_ttl'] || 5000;
+}
 
 var o = TemplateMaps.prototype;
 
@@ -97,36 +95,34 @@ o._redisCmd = function(redisFunc, redisArgs, callback) {
 
 // @param callback function(err, obtained)
 o._obtainTemplateLock = function(owner, tpl_id, callback) {
-  var usr_tpl_lck_key = _.template(this.key_usr_tpl_lck, {owner:owner});
-  var that = this;
-  var gotLock = false;
-  Step (
-    function obtainLock() {
-      var ctime = Date.now();
-      that._redisCmd('HSETNX', [usr_tpl_lck_key, tpl_id, ctime], this);
-    },
-    function checkLock(err, locked) {
-      if ( err ) throw err;
-      if ( ! locked ) {
-        // Already locked
-        // TODO: unlock if expired ?
-        throw new Error("Template '" + tpl_id + "' of user '" + owner + "' is locked");
-      }
-      return gotLock = true;
-    },
-    function finish(err) {
-      callback(err, gotLock);
-    }
-  );
+    var that = this,
+        lockKey = this.key_usr_tpl_lck({owner:owner});
+    Step (
+        function obtainLock() {
+            that._redisCmd('HGET', [lockKey, tpl_id], this);
+        },
+        function checkLock(err, lockTime) {
+            if (err) { throw err; }
+
+            var _newLockTime = Date.now();
+            if (!lockTime || ((_newLockTime - lockTime) > that.lock_ttl)) {
+                that._redisCmd('HSET', [lockKey, tpl_id, _newLockTime], this);
+            } else {
+                throw new Error("Template '" + tpl_id + "' of user '" + owner + "' is locked");
+            }
+        },
+        function finish(err, hsetValue) {
+            callback(err, !!hsetValue);
+        }
+    );
 };
 
 // @param callback function(err, deleted)
 o._releaseTemplateLock = function(owner, tpl_id, callback) {
-  var usr_tpl_lck_key = _.template(this.key_usr_tpl_lck, {owner:owner});
-  this._redisCmd('HDEL', [usr_tpl_lck_key, tpl_id], callback);
+    this._redisCmd('HDEL', [this.key_usr_tpl_lck({owner:owner}), tpl_id], callback);
 };
 
-o._reValidIdentifier = /^[a-zA-Z][0-9a-zA-Z_]*$/;
+var _reValidIdentifier = /^[a-zA-Z][0-9a-zA-Z_]*$/;
 o._checkInvalidTemplate = function(template) {
   if ( template.version != '0.0.1' ) {
     return new Error("Unsupported template version " + template.version);
@@ -135,22 +131,26 @@ o._checkInvalidTemplate = function(template) {
   if ( ! tplname ) {
     return new Error("Missing template name");
   }
-  if ( ! tplname.match(this._reValidIdentifier) ) {
+  if ( ! tplname.match(_reValidIdentifier) ) {
     return new Error("Invalid characters in template name '" + tplname + "'");
   }
 
-  var phold = template.placeholders;
-  for (var k in phold) {
-    if ( ! k.match(this._reValidIdentifier) ) {
-      return new Error("Invalid characters in placeholder name '" + k + "'");
-    }
-    if ( ! phold[k].hasOwnProperty('default') ) {
-      return new Error("Missing default for placeholder '" + k + "'");
-    }
-    if ( ! phold[k].hasOwnProperty('type') ) {
-      return new Error("Missing type for placeholder '" + k + "'");
-    }
-  };
+  var placeholders = template.placeholders || {};
+
+  var placeholderKeys = Object.keys(placeholders);
+  for (var i = 0, len = placeholderKeys.length; i < len; i++) {
+      var placeholderKey = placeholderKeys[i];
+
+      if (!placeholderKey.match(_reValidIdentifier)) {
+          return new Error("Invalid characters in placeholder name '" + placeholderKey + "'");
+      }
+      if ( ! placeholders[placeholderKey].hasOwnProperty('default') ) {
+          return new Error("Missing default for placeholder '" + placeholderKey + "'");
+      }
+      if ( ! placeholders[placeholderKey].hasOwnProperty('type') ) {
+          return new Error("Missing type for placeholder '" + placeholderKey + "'");
+      }
+  }
 
   // Check certificate validity
   var cert = this.getTemplateCertificate(template);
@@ -168,12 +168,11 @@ o._checkInvalidTemplate = function(template) {
 // SignedMaps.addCertificate or SignedMaps.authorizedByCert
 //
 o.getTemplateCertificate = function(template) {
-  var cert = {
-    version: '0.0.1',
-    template_id: template.name,
-    auth: template.auth
+  return {
+      version: '0.0.1',
+      template_id: template.name,
+      auth: template.auth
   };
-  return cert;
 };
 
 // Add a template
@@ -209,7 +208,7 @@ o.addTemplate = function(owner, template, callback) {
   //
   //
 
-  var usr_tpl_key = _.template(this.key_usr_tpl, {owner:owner});
+  var usr_tpl_key = this.key_usr_tpl({owner:owner});
   var gotLock = false;
   var that = this;
   var limit = that._userTemplateLimit();
@@ -293,7 +292,7 @@ o.addTemplate = function(owner, template, callback) {
 // @param callback function(err)
 //
 o.delTemplate = function(owner, tpl_id, callback) {
-  var usr_tpl_key = _.template(this.key_usr_tpl, {owner:owner});
+  var usr_tpl_key = this.key_usr_tpl({owner:owner});
   var gotLock = false;
   var that = this;
   Step(
@@ -402,7 +401,7 @@ o.updTemplate = function(owner, tpl_id, template, callback) {
     return;
   }
 
-  var usr_tpl_key = _.template(this.key_usr_tpl, {owner:owner});
+  var usr_tpl_key = this.key_usr_tpl({owner:owner});
   var gotLock = false;
   var that = this;
   Step(
@@ -496,8 +495,7 @@ o.updTemplate = function(owner, tpl_id, template, callback) {
 //        Returns a list of template identifiers
 // 
 o.listTemplates = function(owner, callback) {
-  var usr_tpl_key = _.template(this.key_usr_tpl, {owner:owner});
-  this._redisCmd('HKEYS', [ usr_tpl_key ], callback);
+    this._redisCmd('HKEYS', [ this.key_usr_tpl({owner:owner}) ], callback);
 };
 
 // Get a templates
@@ -511,17 +509,15 @@ o.listTemplates = function(owner, callback) {
 //        Return full template definition
 //
 o.getTemplate = function(owner, tpl_id, callback) {
-  var usr_tpl_key = _.template(this.key_usr_tpl, {owner:owner});
   var that = this;
   Step(
     function getTemplate() {
-      that._redisCmd('HGET', [ usr_tpl_key, tpl_id ], this);
+      that._redisCmd('HGET', [ that.key_usr_tpl({owner:owner}), tpl_id ], this);
     },
     function parseTemplate(err, tpl_val) {
       if ( err ) throw err;
-      var tpl = JSON.parse(tpl_val);
       // Should we strip auth_id ?
-      return tpl;
+      return JSON.parse(tpl_val);
     },
     function finish(err, tpl) {
       callback(err, tpl);
@@ -541,25 +537,22 @@ o.getTemplate = function(owner, tpl_id, callback) {
 //
 // @throws Error on malformed template or parameter
 //
-o._reNumber = /^([-+]?[\d\.]?\d+([eE][+-]?\d+)?)$/;
-o._reCSSColorName = /^[a-zA-Z]+$/;
-o._reCSSColorVal = /^#[0-9a-fA-F]{3,6}$/;
-o._replaceVars = function(str, params) {
+var _reNumber = /^([-+]?[\d\.]?\d+([eE][+-]?\d+)?)$/,
+    _reCSSColorName = /^[a-zA-Z]+$/,
+    _reCSSColorVal = /^#[0-9a-fA-F]{3,6}$/;
+
+_replaceVars = function(str, params) {
   //return _.template(str, params); // lazy way, possibly dangerous 
   // Construct regular expressions for each param
-  if ( ! params._re ) {
-    params._re = {};
-    for (var k in params) {
-      params._re[k] = RegExp("<%=\\s*" + k + "\\s*%>", "g");
-    }
-  }
-  for (var k in params) str = str.replace(params._re[k], params[k]);
-  return str;
+    Object.keys(params).forEach(function(k) {
+        str = str.replace(new RegExp("<%=\\s*" + k + "\\s*%>", "g"), params[k]);
+    });
+    return str;
 };
 o.instance = function(template, params) {
   var all_params = {};
-  var phold = template.placeholders;
-  for (var k in phold) {
+  var phold = template.placeholders || {};
+  Object.keys(phold).forEach(function(k) {
     var val = params.hasOwnProperty(k) ? params[k] : phold[k].default;
     var type = phold[k].type;
     // properly escape
@@ -573,7 +566,7 @@ o.instance = function(template, params) {
     }
     else if ( type === 'number' ) {
       // check it's a number
-      if ( typeof(val) !== 'number' && ! val.match(this._reNumber) ) {
+      if ( typeof(val) !== 'number' && ! val.match(_reNumber) ) {
         throw new Error("Invalid number value for template parameter '"
           + k + "': " + val);
       }
@@ -581,7 +574,7 @@ o.instance = function(template, params) {
     else if ( type === 'css_color' ) {
       // check it only contains letters or
       // starts with # and only contains hexdigits
-      if ( ! val.match(this._reCSSColorName) && ! val.match(this._reCSSColorVal) ) {
+      if ( ! val.match(_reCSSColorName) && ! val.match(_reCSSColorVal) ) {
         throw new Error("Invalid css_color value for template parameter '"
           + k + "': " + val);
       }
@@ -591,14 +584,14 @@ o.instance = function(template, params) {
       throw new Error("Invalid placeholder type '" + type + "'");
     }
     all_params[k] = val;
-  }
+  });
 
   // NOTE: we're deep-cloning the layergroup here
   var layergroup = JSON.parse(JSON.stringify(template.layergroup));
   for (var i=0; i<layergroup.layers.length; ++i) {
     var lyropt = layergroup.layers[i].options;
-    if ( lyropt.cartocss ) lyropt.cartocss = this._replaceVars(lyropt.cartocss, all_params);
-    if ( lyropt.sql) lyropt.sql = this._replaceVars(lyropt.sql, all_params);
+    if ( lyropt.cartocss ) lyropt.cartocss = _replaceVars(lyropt.cartocss, all_params);
+    if ( lyropt.sql) lyropt.sql = _replaceVars(lyropt.sql, all_params);
     // Anything else ?
   }
   return layergroup;

npm-shrinkwrap.json

@@ -1,29 +1,39 @@
 {
   "name": "windshaft-cartodb",
-  "version": "1.14.0",
+  "version": "1.17.2",
   "dependencies": {
     "node-varnish": {
       "version": "0.3.0",
-      "from": "http://github.com/Vizzuality/node-varnish/tarball/0.3.0"
+      "from": "https://github.com/Vizzuality/node-varnish/tarball/0.3.0"
     },
     "underscore": {
-      "version": "1.3.3"
+      "version": "1.6.0"
+    },
+    "dot": {
+      "version": "1.0.2"
     },
     "windshaft": {
-      "version": "0.23.0",
-      "from": "https://github.com/CartoDB/Windshaft/tarball/0.23.0",
+      "version": "0.27.2",
+      "from": "https://github.com/CartoDB/Windshaft/tarball/0.27.2",
       "dependencies": {
+        "chronograph": {
+          "version": "0.1.0",
+          "from": "git://github.com/CartoDB/chronographjs.git#0.1.0"
+        },
         "grainstore": {
-          "version": "0.19.0",
-          "from": "git://github.com/CartoDB/grainstore.git#0.19.0",
+          "version": "0.22.0",
+          "from": "https://github.com/CartoDB/grainstore/tarball/0.22.0",
           "dependencies": {
             "carto": {
               "version": "0.9.5-cdb2",
-              "from": "http://github.com/CartoDB/carto/tarball/0.9.5-cdb2",
+              "from": "https://github.com/CartoDB/carto/tarball/0.9.5-cdb2",
               "dependencies": {
                 "underscore": {
                   "version": "1.4.4"
                 },
+                "mapnik-reference": {
+                  "version": "5.0.9"
+                },
                 "xml2js": {
                   "version": "0.2.8",
                   "dependencies": {
@@ -45,14 +55,11 @@
                 }
               }
             },
-            "mapnik-reference": {
-              "version": "5.0.9"
-            },
             "millstone": {
               "version": "0.6.14",
               "dependencies": {
-                "underscore": {
-                  "version": "1.6.0"
+                "generic-pool": {
+                  "version": "2.0.4"
                 },
                 "request": {
                   "version": "2.34.0",
@@ -73,7 +80,7 @@
                       "version": "0.12.1",
                       "dependencies": {
                         "punycode": {
-                          "version": "1.2.4"
+                          "version": "1.3.1"
                         }
                       }
                     },
@@ -369,16 +376,19 @@
                   }
                 },
                 "zipfile": {
-                  "version": "0.5.2",
+                  "version": "0.5.3",
                   "dependencies": {
+                    "nan": {
+                      "version": "1.2.0"
+                    },
                     "node-pre-gyp": {
-                      "version": "0.5.8",
+                      "version": "0.5.19",
                       "dependencies": {
                         "nopt": {
-                          "version": "2.2.0",
+                          "version": "2.2.1",
                           "dependencies": {
                             "abbrev": {
-                              "version": "1.0.4"
+                              "version": "1.0.5"
                             }
                           }
                         },
@@ -391,7 +401,7 @@
                           }
                         },
                         "request": {
-                          "version": "2.34.0",
+                          "version": "2.36.0",
                           "dependencies": {
                             "qs": {
                               "version": "0.6.6"
@@ -399,15 +409,15 @@
                             "json-stringify-safe": {
                               "version": "5.0.0"
                             },
+                            "mime": {
+                              "version": "1.2.11"
+                            },
                             "forever-agent": {
                               "version": "0.5.2"
                             },
                             "node-uuid": {
                               "version": "1.4.1"
                             },
-                            "mime": {
-                              "version": "1.2.11"
-                            },
                             "tough-cookie": {
                               "version": "0.12.1",
                               "dependencies": {
@@ -433,7 +443,7 @@
                               }
                             },
                             "tunnel-agent": {
-                              "version": "0.3.0"
+                              "version": "0.4.0"
                             },
                             "http-signature": {
                               "version": "0.10.0",
@@ -475,7 +485,7 @@
                           }
                         },
                         "semver": {
-                          "version": "2.2.1"
+                          "version": "2.3.0"
                         },
                         "tar": {
                           "version": "0.1.19",
@@ -489,6 +499,9 @@
                             "fstream": {
                               "version": "0.1.25",
                               "dependencies": {
+                                "mkdirp": {
+                                  "version": "0.3.5"
+                                },
                                 "graceful-fs": {
                                   "version": "2.0.3"
                                 }
@@ -511,6 +524,9 @@
                             "fstream": {
                               "version": "0.1.25",
                               "dependencies": {
+                                "mkdirp": {
+                                  "version": "0.3.5"
+                                },
                                 "graceful-fs": {
                                   "version": "2.0.3"
                                 },
@@ -539,7 +555,7 @@
                               }
                             },
                             "readable-stream": {
-                              "version": "1.0.26-4",
+                              "version": "1.0.27-1",
                               "dependencies": {
                                 "core-util-is": {
                                   "version": "1.0.1"
@@ -561,16 +577,24 @@
                           }
                         },
                         "mkdirp": {
-                          "version": "0.3.5"
-                        },
-                        "rc": {
-                          "version": "0.3.4",
+                          "version": "0.5.0",
                           "dependencies": {
                             "minimist": {
                               "version": "0.0.8"
+                            }
+                          }
+                        },
+                        "rc": {
+                          "version": "0.4.0",
+                          "dependencies": {
+                            "minimist": {
+                              "version": "0.0.10"
                             },
                             "deep-extend": {
-                              "version": "0.2.8"
+                              "version": "0.2.10"
+                            },
+                            "strip-json-comments": {
+                              "version": "0.1.3"
                             },
                             "ini": {
                               "version": "1.1.0"
@@ -578,36 +602,39 @@
                           }
                         },
                         "rimraf": {
-                          "version": "2.2.6"
+                          "version": "2.2.8"
                         }
                       }
                     }
                   }
                 },
                 "sqlite3": {
-                  "version": "2.2.3",
+                  "version": "2.2.7",
                   "dependencies": {
+                    "nan": {
+                      "version": "1.1.2"
+                    },
                     "node-pre-gyp": {
-                      "version": "0.5.9",
+                      "version": "0.5.22",
                       "dependencies": {
                         "nopt": {
-                          "version": "2.2.0",
+                          "version": "3.0.1",
                           "dependencies": {
                             "abbrev": {
-                              "version": "1.0.4"
+                              "version": "1.0.5"
                             }
                           }
                         },
                         "npmlog": {
-                          "version": "0.0.6",
+                          "version": "0.1.1",
                           "dependencies": {
                             "ansi": {
-                              "version": "0.2.1"
+                              "version": "0.3.0"
                             }
                           }
                         },
                         "request": {
-                          "version": "2.34.0",
+                          "version": "2.39.0",
                           "dependencies": {
                             "qs": {
                               "version": "0.6.6"
@@ -615,41 +642,44 @@
                             "json-stringify-safe": {
                               "version": "5.0.0"
                             },
+                            "mime-types": {
+                              "version": "1.0.2"
+                            },
                             "forever-agent": {
                               "version": "0.5.2"
                             },
                             "node-uuid": {
                               "version": "1.4.1"
                             },
-                            "mime": {
-                              "version": "1.2.11"
-                            },
                             "tough-cookie": {
                               "version": "0.12.1",
                               "dependencies": {
                                 "punycode": {
-                                  "version": "1.2.4"
+                                  "version": "1.3.0"
                                 }
                               }
                             },
                             "form-data": {
-                              "version": "0.1.2",
+                              "version": "0.1.4",
                               "dependencies": {
                                 "combined-stream": {
-                                  "version": "0.0.4",
+                                  "version": "0.0.5",
                                   "dependencies": {
                                     "delayed-stream": {
                                       "version": "0.0.5"
                                     }
                                   }
                                 },
+                                "mime": {
+                                  "version": "1.2.11"
+                                },
                                 "async": {
-                                  "version": "0.2.10"
+                                  "version": "0.9.0"
                                 }
                               }
                             },
                             "tunnel-agent": {
-                              "version": "0.3.0"
+                              "version": "0.4.0"
                             },
                             "http-signature": {
                               "version": "0.10.0",
@@ -669,7 +699,7 @@
                               "version": "0.3.0"
                             },
                             "hawk": {
-                              "version": "1.0.0",
+                              "version": "1.1.1",
                               "dependencies": {
                                 "hoek": {
                                   "version": "0.9.1"
@@ -687,28 +717,31 @@
                             },
                             "aws-sign2": {
                               "version": "0.5.0"
+                            },
+                            "stringstream": {
+                              "version": "0.0.4"
                             }
                           }
                         },
                         "semver": {
-                          "version": "2.2.1"
+                          "version": "3.0.1"
                         },
                         "tar": {
-                          "version": "0.1.19",
+                          "version": "1.0.0",
                           "dependencies": {
-                            "inherits": {
-                              "version": "2.0.1"
-                            },
                             "block-stream": {
                               "version": "0.0.7"
                             },
                             "fstream": {
-                              "version": "0.1.25",
+                              "version": "1.0.1",
                               "dependencies": {
                                 "graceful-fs": {
-                                  "version": "2.0.3"
+                                  "version": "3.0.2"
                                 }
                               }
+                            },
+                            "inherits": {
+                              "version": "2.0.1"
                             }
                           }
                         },
@@ -725,10 +758,21 @@
                               "version": "0.7.4"
                             },
                             "fstream": {
-                              "version": "0.1.25",
+                              "version": "0.1.31",
                               "dependencies": {
                                 "graceful-fs": {
-                                  "version": "2.0.3"
+                                  "version": "3.0.2"
+                                },
+                                "inherits": {
+                                  "version": "2.0.1"
+                                }
+                              }
+                            },
+                            "tar": {
+                              "version": "0.1.20",
+                              "dependencies": {
+                                "block-stream": {
+                                  "version": "0.0.7"
                                 },
                                 "inherits": {
                                   "version": "2.0.1"
@@ -755,7 +799,7 @@
                               }
                             },
                             "readable-stream": {
-                              "version": "1.0.26-4",
+                              "version": "1.0.27-1",
                               "dependencies": {
                                 "core-util-is": {
                                   "version": "1.0.1"
@@ -777,16 +821,24 @@
                           }
                         },
                         "mkdirp": {
-                          "version": "0.3.5"
-                        },
-                        "rc": {
-                          "version": "0.3.4",
+                          "version": "0.5.0",
                           "dependencies": {
                             "minimist": {
                               "version": "0.0.8"
+                            }
+                          }
+                        },
+                        "rc": {
+                          "version": "0.5.0",
+                          "dependencies": {
+                            "minimist": {
+                              "version": "0.0.10"
                             },
                             "deep-extend": {
-                              "version": "0.2.8"
+                              "version": "0.2.11"
+                            },
+                            "strip-json-comments": {
+                              "version": "0.1.3"
                             },
                             "ini": {
                               "version": "1.1.0"
@@ -794,9 +846,12 @@
                           }
                         },
                         "rimraf": {
-                          "version": "2.2.6"
+                          "version": "2.2.8"
                         }
                       }
+                    },
+                    "set-immediate": {
+                      "version": "0.1.1"
                     }
                   }
                 },
@@ -821,9 +876,6 @@
             }
           }
         },
-        "generic-pool": {
-          "version": "2.0.4"
-        },
         "express": {
           "version": "2.5.11",
           "dependencies": {
@@ -846,14 +898,21 @@
             }
           }
         },
+        "mapnik": {
+          "version": "0.7.26-cdb1",
+          "from": "http://github.com/Vizzuality/node-mapnik/tarball/0.7.26-cdb1"
+        },
         "tilelive": {
-          "version": "4.4.3",
+          "version": "4.5.3",
           "dependencies": {
             "optimist": {
-              "version": "0.3.7",
+              "version": "0.6.1",
               "dependencies": {
                 "wordwrap": {
                   "version": "0.0.2"
+                },
+                "minimist": {
+                  "version": "0.0.10"
                 }
               }
             },
@@ -863,8 +922,14 @@
           }
         },
         "tilelive-mapnik": {
-          "version": "0.6.9",
+          "version": "0.6.11",
           "dependencies": {
+            "underscore": {
+              "version": "1.3.3"
+            },
+            "generic-pool": {
+              "version": "2.0.4"
+            },
             "mime": {
               "version": "1.2.11"
             },
@@ -908,17 +973,6 @@
           "version": "0.1.0",
           "from": "git://github.com/CartoDB/node-step-profiler.git#0.1.0"
         },
-        "pg": {
-          "version": "2.6.2",
-          "dependencies": {
-            "generic-pool": {
-              "version": "2.0.3"
-            },
-            "buffer-writer": {
-              "version": "1.0.0"
-            }
-          }
-        },
         "torque.js": {
           "version": "2.2.0"
         },
@@ -931,24 +985,45 @@
       "version": "0.0.5"
     },
     "request": {
-      "version": "2.9.202"
+      "version": "2.9.203"
     },
     "cartodb-redis": {
-      "version": "0.5.0",
-      "from": "git://github.com/CartoDB/node-cartodb-redis.git#0.5.0"
+      "version": "0.11.0",
+      "from": "https://github.com/CartoDB/node-cartodb-redis/tarball/0.11.0"
+    },
+    "cartodb-psql": {
+      "version": "0.4.0",
+      "from": "https://github.com/CartoDB/node-cartodb-psql/tarball/0.4.0",
+      "dependencies": {
+        "pg": {
+          "version": "2.6.2-cdb1",
+          "from": "git://github.com/CartoDB/node-postgres.git#2.6.2-cdb1",
+          "dependencies": {
+            "generic-pool": {
+              "version": "2.0.3"
+            },
+            "buffer-writer": {
+              "version": "1.0.0"
+            }
+          }
+        }
+      }
     },
     "redis-mpool": {
-      "version": "0.0.4",
-      "from": "http://github.com/CartoDB/node-redis-mpool/tarball/0.0.4",
+      "version": "0.1.0",
+      "from": "https://github.com/CartoDB/node-redis-mpool/tarball/0.1.0",
       "dependencies": {
         "generic-pool": {
-          "version": "2.0.4"
+          "version": "2.1.1"
+        },
+        "redis": {
+          "version": "0.12.1"
         },
         "hiredis": {
           "version": "0.1.17",
           "dependencies": {
             "bindings": {
-              "version": "1.2.0"
+              "version": "1.2.1"
             },
             "nan": {
               "version": "1.1.2"
@@ -957,21 +1032,17 @@
         }
       }
     },
-    "mapnik": {
-      "version": "0.7.26-cdb1",
-      "from": "http://github.com/Vizzuality/node-mapnik/tarball/0.7.26-cdb1"
-    },
     "lzma": {
-      "version": "1.2.3"
+      "version": "1.3.7"
     },
     "log4js": {
-      "version": "0.6.14",
+      "version": "0.6.21",
       "dependencies": {
         "async": {
-          "version": "0.1.15"
+          "version": "0.2.10"
         },
         "readable-stream": {
-          "version": "1.0.27-1",
+          "version": "1.0.32",
           "dependencies": {
             "core-util-is": {
               "version": "1.0.1"
@@ -980,7 +1051,7 @@
               "version": "0.0.1"
             },
             "string_decoder": {
-              "version": "0.10.25-1"
+              "version": "0.10.31"
             },
             "inherits": {
               "version": "2.0.1"
@@ -990,7 +1061,7 @@
       }
     },
     "rollbar": {
-      "version": "0.3.8",
+      "version": "0.3.13",
       "dependencies": {
         "node-uuid": {
           "version": "1.4.1"
@@ -1010,16 +1081,16 @@
       "version": "1.1.4"
     },
     "strftime": {
-      "version": "0.6.2"
+      "version": "0.8.2"
     },
     "mocha": {
-      "version": "1.14.0",
+      "version": "1.21.4",
       "dependencies": {
         "commander": {
           "version": "2.0.0"
         },
         "growl": {
-          "version": "1.7.0"
+          "version": "1.8.1"
         },
         "jade": {
           "version": "0.26.3",
@@ -1036,7 +1107,7 @@
           "version": "1.0.7"
         },
         "debug": {
-          "version": "1.0.2",
+          "version": "2.0.0",
           "dependencies": {
             "ms": {
               "version": "0.6.2"

package.json

@@ -1,7 +1,7 @@
 {
     "private": true,
     "name": "windshaft-cartodb",
-    "version": "1.14.0",
+    "version": "1.17.2",
     "description": "A map tile server for CartoDB",
     "keywords": [
       "cartodb"
@@ -22,23 +22,24 @@
       "Sandro Santilli <strk@vizzuality.com>"
     ],
     "dependencies": {
-        "node-varnish": "http://github.com/Vizzuality/node-varnish/tarball/0.3.0",
-        "underscore"  : "~1.3.3",
-        "windshaft": "https://github.com/CartoDB/Windshaft/tarball/0.23.0",
-        "step": "0.0.x",
-        "request": "2.9.202",
-        "cartodb-redis": "git://github.com/CartoDB/node-cartodb-redis.git#0.5.0",
-        "redis-mpool": "http://github.com/CartoDB/node-redis-mpool/tarball/0.0.4",
-        "mapnik": "http://github.com/Vizzuality/node-mapnik/tarball/0.7.26-cdb1",
-        "lzma": "~1.2.3",
-        "log4js": "~0.6.10",
-        "rollbar": "~0.3.1"
+        "node-varnish": "https://github.com/Vizzuality/node-varnish/tarball/0.3.0",
+        "underscore"  : "~1.6.0",
+        "dot": "~1.0.2",
+        "windshaft": "https://github.com/CartoDB/Windshaft/tarball/0.27.2",
+        "step": "~0.0.5",
+        "request": "~2.9.203",
+        "cartodb-redis": "https://github.com/CartoDB/node-cartodb-redis/tarball/0.11.0",
+        "cartodb-psql": "https://github.com/CartoDB/node-cartodb-psql/tarball/0.4.0",
+        "redis-mpool": "https://github.com/CartoDB/node-redis-mpool/tarball/0.1.0",
+        "lzma": "~1.3.7",
+        "log4js": "~0.6.17",
+        "rollbar": "~0.3.13"
     },
     "devDependencies": {
-        "mocha": "1.14.0",
-        "redis": "~0.8.3",
-        "strftime": "~0.6.0",
-        "semver": "~1.1.0"
+        "mocha": "~1.21.4",
+        "redis": "~0.8.6",
+        "strftime": "~0.8.2",
+        "semver": "~1.1.4"
     },
     "scripts": {
         "test": "make check"

test/acceptance/multilayer.js

@@ -4,7 +4,6 @@ var _           = require('underscore');
 var redis       = require('redis');
 var querystring = require('querystring');
 var semver      = require('semver');
-var mapnik      = require('mapnik');
 var Step        = require('step');
 var strftime    = require('strftime');
 var SQLAPIEmu   = require(__dirname + '/../support/SQLAPIEmu.js');
@@ -23,7 +22,11 @@ serverOptions = ServerOptions();
 var server = new CartodbWindshaft(serverOptions);
 server.setMaxListeners(0);
 
-suite('multilayer', function() {
+[true, false].forEach(function(cdbQueryTablesFromPostgresEnabledValue) {
+
+global.environment.enabledFeatures = {cdbQueryTablesFromPostgres: cdbQueryTablesFromPostgresEnabledValue};
+
+suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function() {
 
     var redis_client = redis.createClient(global.environment.redis.port);
     var sqlapi_server;
@@ -112,13 +115,12 @@ suite('multilayer', function() {
               var jsonquery = cc.substring(dbname.length+1);
               var sentquery = JSON.parse(jsonquery);
               var expectedQuery = [layergroup.layers[0].options.sql, ';', layergroup.layers[1].options.sql].join('');
-              assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$'
+              assert.equal(sentquery.q, 'WITH querytables AS ( SELECT * FROM CDB_QueryTables($windshaft$'
                 + expectedQuery
-                + '$windshaft$) as tablenames, EXTRACT(EPOCH FROM max(updated_at)) as max'
+                + '$windshaft$) as tablenames )'
+                + ' SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max'
                 + ' FROM CDB_TableMetadata m'
-                + ' WHERE m.tabname = any (CDB_QueryTables($windshaft$'
-                + expectedQuery
-                + '$windshaft$)::regclass[])');
+                + ' WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])');
 
               assert.imageEqualsFile(res.body, 'test/fixtures/test_table_0_0_0_multilayer1.png', IMAGE_EQUALS_HIGHER_TOLERANCE_PER_MIL,
                 function(err, similarity) {
@@ -155,7 +157,7 @@ suite('multilayer', function() {
               method: 'GET'
           }, {}, function(res) {
               assert.equal(res.statusCode, 200, res.body);
-              assert.equal(res.headers['content-type'], "text/javascript; charset=utf-8; charset=utf-8");
+              assert.equal(res.headers['content-type'], "application/json; charset=utf-8");
               assert.utfgridEqualsFile(res.body, 'test/fixtures/test_table_0_0_0_multilayer1.layer0.grid.json', 2,
                 function(err, similarity) {
                   next(err);
@@ -173,7 +175,7 @@ suite('multilayer', function() {
               method: 'GET'
           }, {}, function(res) {
               assert.equal(res.statusCode, 200, res.body);
-              assert.equal(res.headers['content-type'], "text/javascript; charset=utf-8; charset=utf-8");
+              assert.equal(res.headers['content-type'], "application/json; charset=utf-8");
               assert.utfgridEqualsFile(res.body, 'test/fixtures/test_table_0_0_0_multilayer1.layer1.grid.json', 2,
                 function(err, similarity) {
                   next(err);
@@ -395,13 +397,12 @@ suite('multilayer', function() {
                   .replace(/!bbox!/g, 'ST_MakeEnvelope(0,0,0,0)')
                   .replace(/!pixel_width!/g, '1')
                   .replace(/!pixel_height!/g, '1');
-              assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$'
+              assert.equal(sentquery.q, 'WITH querytables AS ( SELECT * FROM CDB_QueryTables($windshaft$'
                   + expectedQuery
-                  + '$windshaft$) as tablenames, EXTRACT(EPOCH FROM max(updated_at)) as max'
+                  + '$windshaft$) as tablenames )'
+                  + ' SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max'
                   + ' FROM CDB_TableMetadata m'
-                  + ' WHERE m.tabname = any (CDB_QueryTables($windshaft$'
-                  + expectedQuery
-                  + '$windshaft$)::regclass[])');
+                  + ' WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])');
 
               assert.imageEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.png', IMAGE_EQUALS_TOLERANCE_PER_MIL,
                 function(err, similarity) {
@@ -433,13 +434,12 @@ suite('multilayer', function() {
                   .replace('!bbox!', 'ST_MakeEnvelope(0,0,0,0)')
                   .replace('!pixel_width!', '1')
                   .replace('!pixel_height!', '1');
-              assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$'
-                + expectedQuery
-                + '$windshaft$) as tablenames, EXTRACT(EPOCH FROM max(updated_at)) as max'
-                  + ' FROM CDB_TableMetadata m'
-                  + ' WHERE m.tabname = any (CDB_QueryTables($windshaft$'
+              assert.equal(sentquery.q, 'WITH querytables AS ( SELECT * FROM CDB_QueryTables($windshaft$'
                   + expectedQuery
-                  + '$windshaft$)::regclass[])');
+                  + '$windshaft$) as tablenames )'
+                  + ' SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max'
+                  + ' FROM CDB_TableMetadata m'
+                  + ' WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])');
 
               assert.imageEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.png', IMAGE_EQUALS_TOLERANCE_PER_MIL,
                 function(err, similarity) {
@@ -458,7 +458,7 @@ suite('multilayer', function() {
               method: 'GET'
           }, {}, function(res) {
               assert.equal(res.statusCode, 200, res.body);
-              assert.equal(res.headers['content-type'], "text/javascript; charset=utf-8; charset=utf-8");
+              assert.equal(res.headers['content-type'], "application/json; charset=utf-8");
               assert.utfgridEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.grid.json', 2,
                 function(err, similarity) {
                   next(err);
@@ -476,7 +476,7 @@ suite('multilayer', function() {
               method: 'GET'
           }, {}, function(res) {
               assert.equal(res.statusCode, 200, res.body);
-              assert.equal(res.headers['content-type'], "text/javascript; charset=utf-8; charset=utf-8");
+              assert.equal(res.headers['content-type'], "application/json; charset=utf-8");
               assert.utfgridEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.grid.json', 2,
                 function(err, similarity) {
                   next(err);
@@ -752,7 +752,7 @@ suite('multilayer', function() {
               method: 'GET'
           }, {}, function(res) {
               assert.equal(res.statusCode, 200, res.body);
-              assert.equal(res.headers['content-type'], "text/javascript; charset=utf-8; charset=utf-8");
+              assert.equal(res.headers['content-type'], "application/json; charset=utf-8");
               next(err);
           });
         },
@@ -1345,3 +1345,4 @@ suite('multilayer', function() {
     
 });
 
+});

test/acceptance/server.js

@@ -4,7 +4,6 @@ var _           = require('underscore');
 var redis       = require('redis');
 var querystring = require('querystring');
 var semver      = require('semver');
-var mapnik      = require('mapnik');
 var Step        = require('step');
 var http        = require('http');
 var SQLAPIEmu   = require(__dirname + '/../support/SQLAPIEmu.js');
@@ -19,12 +18,17 @@ var serverOptions = require(__dirname + '/../../lib/cartodb/server_options')();
 var server = new CartodbWindshaft(serverOptions);
 server.setMaxListeners(0);
 
-suite('server', function() {
+[true, false].forEach(function(cdbQueryTablesFromPostgresEnabledValue) {
+
+global.environment.enabledFeatures = {cdbQueryTablesFromPostgres: cdbQueryTablesFromPostgresEnabledValue};
+
+suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function() {
+
 
     var redis_client = redis.createClient(global.environment.redis.port);
     var sqlapi_server;
 
-    var mapnik_version = global.environment.mapnik_version || mapnik.versions.mapnik;
+    var mapnik_version = global.environment.mapnik_version || server.getVersion().mapnik;
     var test_database = _.template(global.environment.postgres_auth_user, {user_id:1}) + '_db';
     var default_style;
     if ( semver.satisfies(mapnik_version, '<2.1.0') ) {
@@ -582,7 +586,7 @@ suite('server', function() {
             method: 'GET'
         },{
             status: 200,
-            headers: { 'Content-Type': 'text/javascript; charset=utf-8; charset=utf-8',
+            headers: { 'Content-Type': 'application/json; charset=utf-8',
                        'X-Cache-Channel': test_database+':gadm4' }
         }, function() { done(); });
     });
@@ -594,7 +598,7 @@ suite('server', function() {
             method: 'GET'
         },{
             status: 200,
-            headers: { 'Content-Type': 'text/javascript; charset=utf-8; charset=utf-8' }
+            headers: { 'Content-Type': 'application/json; charset=utf-8' }
         }, function() { done(); });
     });
     
@@ -606,7 +610,7 @@ suite('server', function() {
             method: 'GET'
         },{
             status: 200,
-            headers: { 'Content-Type': 'text/javascript; charset=utf-8; charset=utf-8' }
+            headers: { 'Content-Type': 'application/json; charset=utf-8' }
         }, function() { done(); });
     });
 
@@ -1389,3 +1393,4 @@ suite('server', function() {
     
 });
 
+});

test/acceptance/templates.js

@@ -3,7 +3,6 @@ var _           = require('underscore');
 var redis       = require('redis');
 var querystring = require('querystring');
 var semver      = require('semver');
-var mapnik      = require('mapnik');
 var Step        = require('step');
 var strftime    = require('strftime');
 var SQLAPIEmu   = require(__dirname + '/../support/SQLAPIEmu.js');
@@ -25,7 +24,11 @@ var serverOptions = ServerOptions();
 var server = new CartodbWindshaft(serverOptions);
 server.setMaxListeners(0);
 
-suite('template_api', function() {
+[true, false].forEach(function(cdbQueryTablesFromPostgresEnabledValue) {
+
+global.environment.enabledFeatures = {cdbQueryTablesFromPostgres: cdbQueryTablesFromPostgresEnabledValue};
+
+suite('template_api:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function() {
 
     var redis_client = redis.createClient(global.environment.redis.port);
     var sqlapi_server;
@@ -1948,3 +1951,4 @@ suite('template_api', function() {
     
 });
 
+});

test/support/assert.js

@@ -226,6 +226,8 @@ assert.response = function(server, req, res, msg){
             response.on('end', function(){
                 if (timer) clearTimeout(timer);
 
+                check();
+
                 // Assert response body
                 if (res.body !== undefined) {
                     var eql = res.body instanceof RegExp
@@ -271,7 +273,6 @@ assert.response = function(server, req, res, msg){
 
                 // Callback
                 callback(response);
-                check();
             });
         });
 

test/support/prepare_db.sh

@@ -86,7 +86,7 @@ if test x"$PREPARE_REDIS" = xyes; then
 
   cat <<EOF | redis-cli -p ${REDIS_PORT} -n 5
 HMSET rails:users:localhost id ${TESTUSERID} \
-                            database_name '${TEST_DB}' \
+                            database_name "${TEST_DB}" \
                             database_host localhost \
                             map_key 1234
 SADD rails:users:localhost:map_key 1235

test/support/test_helper.js

@@ -7,7 +7,9 @@
 
 var _ = require('underscore');
 var assert = require('assert');
-var LZMA  = require('lzma/lzma_worker.js').LZMA;
+var LZMA  = require('lzma').LZMA;
+
+var lzmaWorker = new LZMA();
 
 // set environment specific variables
 global.environment  = require(__dirname + '/../../config/environments/test');
@@ -16,7 +18,7 @@ process.env.NODE_ENV = 'test';
 
 // Utility function to compress & encode LZMA
 function lzma_compress_to_base64(payload, mode, callback) {
-  LZMA.compress(payload, mode, 
+  lzmaWorker.compress(payload, mode,
     function(ints) {
       ints = ints.map(function(c) { return String.fromCharCode(c + 128) }).join('')
       var base64 = new Buffer(ints, 'binary').toString('base64');

test/unit/cartodb/template_maps.test.js

@@ -12,6 +12,14 @@ suite('template_maps', function() {
   // configure redis pool instance to use in tests
   var redis_pool = RedisPool(global.environment.redis);
   var signed_maps = new SignedMaps(redis_pool);
+
+    var validTemplate = {
+        version:'0.0.1',
+        name: 'first',
+        auth: {},
+        layergroup: {}
+    };
+    var owner = 'me';
     
   test('does not accept template with unsupported version', function(done) {
     var tmap = new TemplateMaps(redis_pool, signed_maps);
@@ -502,5 +510,89 @@ suite('template_maps', function() {
       }
     );
   });
-    
+
+    var redisCmdFunc = TemplateMaps.prototype._redisCmd;
+
+    function runWithRedisStubbed(stubbedCommands, func) {
+        TemplateMaps.prototype._redisCmd = function(redisFunc, redisArgs, callback) {
+            redisFunc = redisFunc.toLowerCase();
+            if (stubbedCommands.hasOwnProperty(redisFunc)) {
+                callback(null, stubbedCommands[redisFunc]);
+            } else {
+                throw 'Unknown command';
+            }
+        };
+
+        func();
+
+        TemplateMaps.prototype._redisCmd = redisCmdFunc;
+    }
+
+    test('_obtainTemplateLock with no previous value, happy case', function(done) {
+        runWithRedisStubbed({hget: null, hset: 1}, function() {
+            var templateMaps = new TemplateMaps(redis_pool, signed_maps);
+
+            templateMaps._obtainTemplateLock(owner, validTemplate.name, function(err, gotLock) {
+                assert.ok(!err);
+                assert.ok(gotLock);
+                done();
+            });
+        });
+    });
+
+    test('_obtainTemplateLock no lock for non expired ttl, simulates obtaining two locks at same time', function(done) {
+        runWithRedisStubbed({hget: Date.now()}, function() {
+            var templateMaps = new TemplateMaps(redis_pool, signed_maps);
+
+            templateMaps._obtainTemplateLock(owner, validTemplate.name, function(err, gotLock) {
+                assert.ok(!!err);
+                assert.equal(gotLock, false);
+                done();
+            });
+        });
+    });
+
+
+    test('_obtainTemplateLock no lock for non expired ttl, last millisecond of valid ttl', function(done) {
+        var nowValue = Date.now(),
+            nowFunc = Date.now;
+        Date.now = function() {
+            return nowValue;
+        };
+        var lockTtl = 1000;
+        runWithRedisStubbed({hget: Date.now() - lockTtl, hset: true}, function() {
+            var templateMaps = new TemplateMaps(redis_pool, signed_maps, {lock_ttl: lockTtl});
+
+            templateMaps._obtainTemplateLock(owner, validTemplate.name, function(err, gotLock) {
+                assert.ok(!!err);
+                assert.equal(gotLock, false);
+
+                Date.now = nowFunc;
+
+                done();
+            });
+        });
+    });
+
+    test('_obtainTemplateLock gets lock for expired ttl, first millisecond of invalid ttl', function(done) {
+        var nowValue = Date.now(),
+            nowFunc = Date.now;
+        Date.now = function() {
+            return nowValue;
+        };
+        var lockTtl = 1000;
+        runWithRedisStubbed({hget: Date.now() - lockTtl - 1, hset: true}, function() {
+            var templateMaps = new TemplateMaps(redis_pool, signed_maps, {lock_ttl: lockTtl});
+
+            templateMaps._obtainTemplateLock(owner, validTemplate.name, function(err, gotLock) {
+                assert.ok(!err);
+                assert.ok(gotLock);
+
+                Date.now = nowFunc;
+
+                done();
+            });
+        });
+    });
+
 });