Release 1.8.3

Closes #168

NEWS.md

@@ -1,3 +1,17 @@
+1.8.3 -- 2014-02-27
+-------------------
+
+Enhancements:
+
+ - Upgrades windshaft to 0.19.1 with many performance improvements,
+   See node_modules/windshaft/NEWS 
+ - Improve speed of instanciating a map (#147, #159, #165)
+ - Give meaningful error on attempts to use map tokens
+   with attribute service (#156)
+ - Reduce sql-api communication timeout, and allow overriding (#167)
+   [ new sqlapi.timeout directive, defaults to 100 ms ]
+ - Do not query CDB_TableMetadata for queries affected by no tables (#16)
+
 1.8.2 -- 2014-02-25
 -------------------
 

config/environments/development.js.example

@@ -106,7 +106,10 @@ var config = {
         // Maximum lenght of SQL query for GET
         // requests. Longer queries will be sent
         // using POST. Defaults to 2048
-        max_get_sql_length: 2048
+        max_get_sql_length: 2048,
+        // Maximum time to wait for a response,
+        // in milliseconds. Defaults to 100.
+        timeout: 100
     }
     ,varnish: {
         host: 'localhost',

config/environments/production.js.example

@@ -100,7 +100,10 @@ var config = {
         // Maximum lenght of SQL query for GET
         // requests. Longer queries will be sent
         // using POST. Defaults to 2048
-        max_get_sql_length: 2048
+        max_get_sql_length: 2048,
+        // Maximum time to wait for a response,
+        // in milliseconds. Defaults to 100.
+        timeout: 100
     }
     ,varnish: {
         host: 'localhost',

config/environments/staging.js.example

@@ -100,7 +100,10 @@ var config = {
         // Maximum lenght of SQL query for GET
         // requests. Longer queries will be sent
         // using POST. Defaults to 2048
-        max_get_sql_length: 2048
+        max_get_sql_length: 2048,
+        // Maximum time to wait for a response,
+        // in milliseconds. Defaults to 100.
+        timeout: 100
     }
     ,varnish: {
         host: 'localhost',

config/environments/test.js.example

@@ -102,7 +102,10 @@ var config = {
         // Maximum lenght of SQL query for GET
         // requests. Longer queries will be sent
         // using POST. Defaults to 2048
-        max_get_sql_length: 2048
+        max_get_sql_length: 2048,
+        // Maximum time to wait for a response,
+        // in milliseconds. Defaults to 100.
+        timeout: 100
     }
     ,varnish: {
         host: '',

lib/cartodb/server_options.js

@@ -116,13 +116,15 @@ module.exports = function(){
       //
       var maxSockets = global.environment.maxConnections || 128;
       var maxGetLen = api.max_get_sql_length || 2048;
+      var maxSQLTime = api.timeout || 100; // 1/10 of a second by default
       var reqSpec = {
             url:sqlapi,
             json:true,
             headers:{host: sqlapihostname}
             // http://nodejs.org/api/http.html#http_agent_maxsockets
             ,pool:{maxSockets:maxSockets}
-            //,timeout:100
+            // timeout in milliseconds
+            ,timeout:maxSQLTime
       }
       if ( sql.length > maxGetLen ) {
         reqSpec.method = 'POST';
@@ -389,23 +391,34 @@ module.exports = function(){
         var key    = req.params.map_key || req.params.api_key;
 
         var cacheKey = dbName + ':' + token;
+        var tabNames;
 
-        me.affectedTables(usr, key, sql, function(err, tableNames) {
+        Step(
+          function getTables() {
+            me.affectedTables(usr, key, sql, this);
+          },
+          function getLastupdated(err, tableNames) {
             if (req.profiler) req.profiler.done('affectedTables');
-
-            if ( err ) { done(err); return; }
+            if ( err ) throw err;
             var cacheChannel = me.buildCacheChannel(dbName,tableNames);
             // store for caching from me.afterLayergroupCreate
             me.channelCache[cacheKey] = cacheChannel;
             // find last updated
-            me.findLastUpdated(usr, key, tableNames, function(err, lastUpdated) {
-              if (req.profiler) req.profiler.done('findLastUpdated');
-              if ( err ) { done(err); return; }
-              response.layergroupid = response.layergroupid + ':' + lastUpdated; // use epoch
-              response.last_updated = new Date(lastUpdated).toISOString(); // TODO: use ISO format
-              done(null);
-            });
-        });
+            if ( ! tableNames.length ) return 0; // skip for no affected tables
+            tabNames = tableNames;
+            me.findLastUpdated(usr, key, tableNames, this);
+          },
+          function(err, lastUpdated) {
+            if ( err ) throw err;
+            if (req.profiler && tabNames) req.profiler.done('findLastUpdated');
+            response.layergroupid = response.layergroupid + ':' + lastUpdated; // use epoch
+            response.last_updated = new Date(lastUpdated).toISOString(); 
+            return null;
+          },
+          function finish(err) {
+            done(err);
+          }
+        );
     };
 
 /* X-Cache-Channel generation } */

npm-shrinkwrap.json

@@ -1,6 +1,6 @@
 {
   "name": "windshaft-cartodb",
-  "version": "1.8.2",
+  "version": "1.8.3",
   "dependencies": {
     "node-varnish": {
       "version": "0.2.0",
@@ -10,8 +10,8 @@
       "version": "1.3.3"
     },
     "windshaft": {
-      "version": "0.19.0",
-      "from": "http://github.com/CartoDB/Windshaft/tarball/0.19.0-rc1",
+      "version": "0.19.1",
+      "from": "http://github.com/CartoDB/Windshaft/tarball/0.19.1",
       "dependencies": {
         "grainstore": {
           "version": "0.18.0",
@@ -191,7 +191,7 @@
                               }
                             },
                             "readable-stream": {
-                              "version": "1.0.25-1",
+                              "version": "1.0.26",
                               "dependencies": {
                                 "string_decoder": {
                                   "version": "0.10.25-1"
@@ -220,15 +220,10 @@
                           }
                         },
                         "rc": {
-                          "version": "0.3.3",
+                          "version": "0.3.4",
                           "dependencies": {
-                            "optimist": {
-                              "version": "0.3.7",
-                              "dependencies": {
-                                "wordwrap": {
-                                  "version": "0.0.2"
-                                }
-                              }
+                            "minimist": {
+                              "version": "0.0.8"
                             },
                             "deep-extend": {
                               "version": "0.2.8"
@@ -258,7 +253,7 @@
                       "version": "0.0.2"
                     },
                     "minimist": {
-                      "version": "0.0.7"
+                      "version": "0.0.8"
                     }
                   }
                 }
@@ -308,7 +303,7 @@
           }
         },
         "tilelive-mapnik": {
-          "version": "0.6.5",
+          "version": "0.6.7",
           "dependencies": {
             "eio": {
               "version": "0.2.2"
@@ -349,7 +344,7 @@
                   "version": "0.0.2"
                 },
                 "minimist": {
-                  "version": "0.0.7"
+                  "version": "0.0.8"
                 }
               }
             }

package.json

@@ -1,7 +1,7 @@
 {
     "private": true,
     "name": "windshaft-cartodb",
-    "version": "1.8.2",
+    "version": "1.8.3",
     "description": "A map tile server for CartoDB",
     "keywords": [
       "cartodb"
@@ -24,7 +24,7 @@
     "dependencies": {
         "node-varnish": "http://github.com/Vizzuality/node-varnish/tarball/v0.2.0",
         "underscore"  : "~1.3.3",
-        "windshaft"  : "http://github.com/CartoDB/Windshaft/tarball/0.19.0",
+        "windshaft"  : "http://github.com/CartoDB/Windshaft/tarball/0.19.1",
         "step": "0.0.x",
         "request": "2.9.202",
         "cartodb-redis": "~0.3.0",

test/acceptance/multilayer.js

@@ -1128,6 +1128,47 @@ suite('multilayer', function() {
       );
     });
 
+    // See https://github.com/CartoDB/Windshaft-cartodb/issues/167
+    test("lack of response from sql-api will result in a timeout", function(done) {
+
+      var layergroup =  {
+        version: '1.0.0',
+        layers: [
+           { options: {
+               sql: "select *, 'SQLAPINOANSWER' from test_table",
+               cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', 
+               cartocss_version: '2.1.0'
+             } }
+        ]
+      };
+
+      Step(
+        function do_post()
+        {
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup',
+              method: 'POST',
+              headers: {host: 'localhost', 'Content-Type': 'application/json' },
+              data: JSON.stringify(layergroup)
+          }, {}, function(res, err) { next(err, res); });
+        },
+        function check_post(err, res) {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body);
+          var parsed = JSON.parse(res.body);
+          assert.ok(parsed.errors, 'Missing "errors" in response: ' + JSON.stringify(parsed));
+          assert.equal(parsed.errors.length, 1);
+          var msg = parsed.errors[0];
+          assert.equal(msg, 'Error: could not fetch source tables: ETIMEDOUT');
+          return null;
+        },
+        function finish(err) {
+          done(err);
+        }
+      );
+    });
+
 
     suiteTeardown(function(done) {
 

test/support/SQLAPIEmu.js

@@ -42,6 +42,9 @@ o.prototype.handleQuery = function(query, res) {
     if ( query.q.match('SQLAPIERROR') ) {
       res.statusCode = 400;
       res.write(JSON.stringify({'error':'Some error occurred'}));
+    } else if ( query.q.match('SQLAPINOANSWER') ) {
+      console.log("SQLAPIEmulator will never respond, on request");
+      return;
     } else if ( query.q.match('EPOCH.* as max') ) {
       // This is the structure of the known query sent by tiler
       var row = {