Merge commit '4738b880a6c29a6d10dda3ad178f35a54bd576d3'

Conflicts:
	NEWS.md
	package.json

HOWTO_RELEASE

@@ -1,13 +1,20 @@
-1. Ensure proper version in package.json 
-2. Ensure NEWS section exists for the new version, review it, add release date
-3. Drop npm-shrinkwrap.json
-4. Run npm install
-5. Test (make check or npm test), fix if broken before proceeding
-6. Run npm shrinkwrap
-7. Set "from" in npm-shrinkwrap.json for known packages
-   (windshaft, node-varnish, grainstore...)
-8. Commit package.json, npm-shrinwrap.json, NEWS
-9. git tag -a Major.Minor.Patch # use NEWS section as content
-10. Announce
-11. Stub NEWS/package for next version
+1. Test (make clean all check), fix if broken before proceeding
+2. Ensure proper version in package.json 
+3. Ensure NEWS section exists for the new version, review it, add release date
+4. Drop npm-shrinkwrap.json
+5. Run npm shrinkwrap to recreate npm-shrinkwrap.json
+6. Commit package.json, npm-shrinwrap.json, NEWS
+7. git tag -a Major.Minor.Patch # use NEWS section as content
+8. Announce on cartodb@googlegroups.com
+9. Stub NEWS/package for next version
 
+Versions:
+
+Bugfix releases increment Patch component of version.
+Feature releases increment Minor and set Patch to zero.
+If backward compatibility is broken, increment Major and
+set to zero Minor and Patch.
+
+Branches named 'b<Major>.<Minor>' are kept for any critical
+fix that might need to be shipped before next feature release
+is ready.

NEWS.md

@@ -1,5 +1,40 @@
-1.10.0 -- 2014-MM-DD
--------------------
+1.11.1 -- 2014-05-07
+--------------------
+
+Enhancements:
+
+ - Upgrade Windshaft to 0.21.0, see
+ http://github.com/CartoDB/Windshaft/blob/0.21.0/NEWS
+
+1.11.0 -- 2014-04-28
+--------------------
+
+New features:
+
+ - Add support for log_filename directive
+ - Reopen log file on SIGHUP, for better logrotate integration
+
+Enhancements:
+
+ - Set default PostgreSQL application name to "cartodb_tiler"
+
+1.10.2 -- 2014-04-08
+--------------------
+
+Bug fixes:
+
+ - Fix show_style tool broken since 1.8.1
+ - Fix X-Cache-Channel of tiles accessed via signed token (#188)
+
+1.10.1 -- 2014-03-21
+--------------------
+
+Bug fixes:
+
+ - Do not cache non-success jsonp responses (#186)
+
+1.10.0 -- 2014-03-20
+--------------------
 
 New features:
 

app.js

@@ -7,6 +7,10 @@
  * environments: [development, production]
  */
 
+var path = require('path'),
+    fs = require('fs')
+;
+
 
 if ( process.argv[2] ) ENV = process.argv[2];
 else if ( process.env['NODE_ENV'] ) ENV = process.env['NODE_ENV'];
@@ -21,22 +25,35 @@ if (ENV != 'development' && ENV != 'production' && ENV != 'staging' ){
     process.exit(1);
 }
 
-var _ = require('underscore')
-    , Step       = require('step')
-    ;
+var _ = require('underscore');
+
 // set environment specific variables
-global.settings     = require(__dirname + '/config/settings');
 global.environment  = require(__dirname + '/config/environments/' + ENV);
-_.extend(global.settings, global.environment);
 
 global.log4js = require('log4js')
 log4js_config = {
-  appenders: [
-    { type: "console", layout: { type:'basic' } }
-  ],
+  appenders: [],
   replaceConsole:true
 };
 
+if ( global.environment.log_filename ) {
+  var logdir = path.dirname(global.environment.log_filename);
+  // See cwd inlog4js.configure call below
+  logdir = path.resolve(__dirname, logdir);
+  if ( ! fs.existsSync(logdir) ) {
+    console.error("Log filename directory does not exist: " + logdir);
+    process.exit(1);
+  }
+  console.log("Logs will be written to " + global.environment.log_filename);
+  log4js_config.appenders.push(
+    { type: "file", filename: global.environment.log_filename }
+  );
+} else {
+  log4js_config.appenders.push(
+    { type: "console", layout: { type:'basic' } }
+  );
+}
+
 if ( global.environment.rollbar ) {
   log4js_config.appenders.push({
     type: __dirname + "/lib/cartodb/log4js_rollbar.js",
@@ -44,7 +61,7 @@ if ( global.environment.rollbar ) {
   });
 }
 
-log4js.configure(log4js_config);
+log4js.configure(log4js_config, { cwd: __dirname });
 global.logger = log4js.getLogger();
 
 // Include cartodb_windshaft only _after_ the "global" variable is set
@@ -80,6 +97,11 @@ process.on('SIGUSR2', function() {
   ws.dumpCacheStats();
 });
 
+process.on('SIGHUP', function() {
+  log4js.configure(log4js_config);
+  console.log('Log files reloaded');
+});
+
 process.on('uncaughtException', function(err) {
   logger.error('Uncaught exception: ' + err.stack); 
 });

config/environments/development.js.example

@@ -31,11 +31,15 @@ var config = {
     // to be able to navigate the map without a reload ?
     // Defaults to 7200 (2 hours)
     ,mapConfigTTL: 7200
-    // idle socket timeout, in miliseconds 
+    // idle socket timeout, in milliseconds 
     ,socket_timeout: 600000
     ,enable_cors: true
     ,cache_enabled: false
     ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
+    // If log_filename is given logs will be written
+    // there, in append mode. Otherwise stdout is used (default).
+    // Log file will be re-opened on receiving the HUP signal
+    ,log_filename: 'logs/node-windshaft.log'
     // Templated database username for authorized user
     // Supported labels: 'user_id' (read from redis)
     ,postgres_auth_user: 'development_cartodb_user_<%= user_id %>'
@@ -58,6 +62,14 @@ var config = {
         */
         row_limit: 65535,
         simplify_geometries: true,
+        /*
+         * Set persist_connection to false if you want
+         * database connections to be closed on renderer
+         * expiration (1 minute after last use).
+         * Setting to true (the default) would never
+         * close any connection for the server's lifetime
+         */
+        persist_connection: false,
         max_size: 500
     }
     ,mapnik_version: undefined
@@ -84,9 +96,9 @@ var config = {
         // Max number of connections in each pool.
         // Users will be put on a queue when the limit is hit.
         // Set to maxConnection to have no possible queues.
-        // There are currently 3 pools involved in serving
+        // There are currently 2 pools involved in serving
         // windshaft-cartodb requests so multiply this number
-        // by 3 to know how many possible connections will be
+        // by 2 to know how many possible connections will be
         // kept open by the server. The default is 50.
         max: 50,
         idleTimeoutMillis: 1, // idle time before dropping connection

config/environments/production.js.example

@@ -31,11 +31,15 @@ var config = {
     // to be able to navigate the map without a reload ?
     // Defaults to 7200 (2 hours)
     ,mapConfigTTL: 7200
-    // idle socket timeout, in miliseconds
+    // idle socket timeout, in milliseconds
     ,socket_timeout: 600000 
     ,enable_cors: true
     ,cache_enabled: true
     ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
+    // If log_filename is given logs will be written
+    // there, in append mode. Otherwise stdout is used (default).
+    // Log file will be re-opened on receiving the HUP signal
+    ,log_filename: 'logs/node-windshaft.log'
     // Templated database username for authorized user
     // Supported labels: 'user_id' (read from redis)
     ,postgres_auth_user: 'cartodb_user_<%= user_id %>'
@@ -51,6 +55,14 @@ var config = {
         port: 6432,
         extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
         row_limit: 65535,
+        /*
+         * Set persist_connection to false if you want
+         * database connections to be closed on renderer
+         * expiration (1 minute after last use).
+         * Setting to true (the default) would never
+         * close any connection for the server's lifetime
+         */
+        persist_connection: false,
         simplify_geometries: true,
         max_size: 500
     }
@@ -78,9 +90,9 @@ var config = {
         // Max number of connections in each pool.
         // Users will be put on a queue when the limit is hit.
         // Set to maxConnection to have no possible queues.
-        // There are currently 3 pools involved in serving
+        // There are currently 2 pools involved in serving
         // windshaft-cartodb requests so multiply this number
-        // by 3 to know how many possible connections will be
+        // by 2 to know how many possible connections will be
         // kept open by the server. The default is 50.
         max: 50,
         idleTimeoutMillis: 30000, // idle time before dropping connection

config/environments/staging.js.example

@@ -31,11 +31,15 @@ var config = {
     // to be able to navigate the map without a reload ?
     // Defaults to 7200 (2 hours)
     ,mapConfigTTL: 7200
-    // idle socket timeout, in miliseconds
+    // idle socket timeout, in milliseconds
     ,socket_timeout: 600000 
     ,enable_cors: true
     ,cache_enabled: true
     ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms (:res[X-Tiler-Profiler]) -> :res[Content-Type]'
+    // If log_filename is given logs will be written
+    // there, in append mode. Otherwise stdout is used (default).
+    // Log file will be re-opened on receiving the HUP signal
+    ,log_filename: 'logs/node-windshaft.log'
     // Templated database username for authorized user
     // Supported labels: 'user_id' (read from redis)
     ,postgres_auth_user: 'cartodb_staging_user_<%= user_id %>'
@@ -52,6 +56,14 @@ var config = {
         extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
         row_limit: 65535,
         simplify_geometries: true,
+        /*
+         * Set persist_connection to false if you want
+         * database connections to be closed on renderer
+         * expiration (1 minute after last use).
+         * Setting to true (the default) would never
+         * close any connection for the server's lifetime
+         */
+        persist_connection: false,
         max_size: 500
     }
     ,mapnik_version: undefined
@@ -78,9 +90,9 @@ var config = {
         // Max number of connections in each pool.
         // Users will be put on a queue when the limit is hit.
         // Set to maxConnection to have no possible queues.
-        // There are currently 3 pools involved in serving
+        // There are currently 2 pools involved in serving
         // windshaft-cartodb requests so multiply this number
-        // by 3 to know how many possible connections will be
+        // by 2 to know how many possible connections will be
         // kept open by the server. The default is 50.
         max: 50,
         idleTimeoutMillis: 30000, // idle time before dropping connection

config/environments/test.js.example

@@ -31,11 +31,15 @@ var config = {
     // to be able to navigate the map without a reload ?
     // Defaults to 7200 (2 hours)
     ,mapConfigTTL: 7200
-    // idle socket timeout, in miliseconds
+    // idle socket timeout, in milliseconds
     ,socket_timeout: 600000 
     ,enable_cors: true
     ,cache_enabled: false
     ,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
+    // If log_filename is given logs will be written
+    // there, in append mode. Otherwise stdout is used (default).
+    // Log file will be re-opened on receiving the HUP signal
+    //,log_filename: 'logs/node-windshaft.log'
     // Templated database username for authorized user
     // Supported labels: 'user_id' (read from redis)
     ,postgres_auth_user: 'test_windshaft_cartodb_user_<%= user_id %>'
@@ -52,6 +56,14 @@ var config = {
         extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
         row_limit: 65535,
         simplify_geometries: true,
+        /*
+         * Set persist_connection to false if you want
+         * database connections to be closed on renderer
+         * expiration (1 minute after last use).
+         * Setting to true (the default) would never
+         * close any connection for the server's lifetime
+         */
+        persist_connection: false,
         max_size: 500
     }
     ,mapnik_version: ''
@@ -78,9 +90,9 @@ var config = {
         // Max number of connections in each pool.
         // Users will be put on a queue when the limit is hit.
         // Set to maxConnection to have no possible queues.
-        // There are currently 3 pools involved in serving
+        // There are currently 2 pools involved in serving
         // windshaft-cartodb requests so multiply this number
-        // by 3 to know how many possible connections will be
+        // by 2 to know how many possible connections will be
         // kept open by the server. The default is 50.
         max: 50,
         idleTimeoutMillis: 1, // idle time before dropping connection

config/settings.js

@@ -1 +0,0 @@
-module.exports.oneDay   = 86400000;

lib/cartodb/cartodb_windshaft.js

@@ -11,6 +11,9 @@ var _ = require('underscore')
     , os = require('os')
 ;
 
+if ( ! process.env['PGAPPNAME'] )
+  process.env['PGAPPNAME']='cartodb_tiler';
+
 var CartodbWindshaft = function(serverOptions) {
    var debug = global.environment.debug;
 
@@ -76,9 +79,14 @@ var CartodbWindshaft = function(serverOptions) {
       var that = this;
       var thatArgs = arguments;
       var statusCode;
-      if ( args.length > 2 ) statusCode = args[2];
-      else {
-        statusCode = args[1] || 200;
+      if ( res._windshaftStatusCode ) {
+        // Added by our override of sendError
+        statusCode = res._windshaftStatusCode;
+      } else {
+        if ( args.length > 2 ) statusCode = args[2];
+        else {
+          statusCode = args[1] || 200;
+        }
       }
       var req = res.req;
       Step (
@@ -117,6 +125,14 @@ var CartodbWindshaft = function(serverOptions) {
       );
     };
 
+    var ws_sendError = ws.sendError;
+    ws.sendError = function() {
+      var res = arguments[0];
+      var statusCode = arguments[2];
+      res._windshaftStatusCode = statusCode;
+      ws_sendError.apply(this, arguments);
+    };
+
     /**
      * Helper to allow access to the layer to be used in the maps infowindow popup.
      */

lib/cartodb/server_options.js

@@ -285,8 +285,31 @@ module.exports = function(){
             }
             return [req.params.table];
           }
-          var username   = that.userByReq(req);
-          me.affectedTables(username, req.params.map_key, sql, this);
+          var user, key;
+          var next = this;
+          Step (
+            function findUserKey() {
+              if ( req.params.hasOwnProperty('_authorizedBySigner') ) {
+                user = req.params._authorizedBySigner;
+                cartoData.getUserMapKey(user, this);
+              } else {
+                user = that.userByReq(req);
+                key = req.params.map_key || req.params.api_key;
+                return null;
+              }
+            },
+            function getAffected(err, data) {
+              if ( err ) throw err;
+              if ( data ) {
+                if ( req.profiler ) req.profiler.done('getSignerMapKey');
+                key = data;
+              }
+              me.affectedTables(user, key, sql, this); // in addCacheChannel
+            },
+            function finish(err, data) {
+              next(err,data);
+            }
+          );
         },
         function buildCacheChannel(err, tableNames) {
           if ( err ) throw err;
@@ -407,7 +430,7 @@ module.exports = function(){
 
         Step(
           function getTables() {
-            me.affectedTables(usr, key, sql, this);
+            me.affectedTables(usr, key, sql, this); // in afterLayergroupCreate
           },
           function getLastupdated(err, tableNames) {
             if (req.profiler) req.profiler.done('affectedTables');
@@ -671,6 +694,7 @@ module.exports = function(){
                 }
 
                 // Authorized by "signed_by" !
+                _.extend(req.params, { _authorizedBySigner: signed_by });
                 that.setDBAuth(signed_by, req.params, function(err) {
                   if (req.profiler) req.profiler.done('setDBAuth');
                   callback(err, true); // authorized (or error)

npm-shrinkwrap.json

@@ -1,6 +1,6 @@
 {
   "name": "windshaft-cartodb",
-  "version": "1.10.0",
+  "version": "1.11.1",
   "dependencies": {
     "node-varnish": {
       "version": "0.3.0",
@@ -11,7 +11,7 @@
     },
     "windshaft": {
       "version": "0.20.0",
-      "from": "http://github.com/CartoDB/Windshaft/tarball/0.20.0",
+      "from": "http://github.com/CartoDB/Windshaft/tarball/0.21.0",
       "dependencies": {
         "grainstore": {
           "version": "0.18.1",
@@ -469,14 +469,14 @@
         }
       }
     },
-    "strftime": {
-      "version": "0.6.2"
+    "redis": {
+      "version": "0.8.6"
     },
     "semver": {
       "version": "1.1.4"
     },
-    "redis": {
-      "version": "0.8.6"
+    "strftime": {
+      "version": "0.6.2"
     },
     "mocha": {
       "version": "1.14.0",

package.json

@@ -1,7 +1,7 @@
 {
     "private": true,
     "name": "windshaft-cartodb",
-    "version": "1.10.0",
+    "version": "1.11.1",
     "description": "A map tile server for CartoDB",
     "keywords": [
       "cartodb"
@@ -24,7 +24,7 @@
     "dependencies": {
         "node-varnish": "http://github.com/Vizzuality/node-varnish/tarball/0.3.0",
         "underscore"  : "~1.3.3",
-        "windshaft"  : "http://github.com/CartoDB/Windshaft/tarball/0.20.0",
+        "windshaft"  : "http://github.com/CartoDB/Windshaft/tarball/0.21.0",
         "step": "0.0.x",
         "request": "2.9.202",
         "cartodb-redis": "~0.3.0",

run_tests.sh

@@ -5,6 +5,8 @@ OPT_CREATE_PGSQL=yes # create the PostgreSQL test environment
 OPT_DROP_REDIS=yes   # drop the redis test environment
 OPT_DROP_PGSQL=yes   # drop the PostgreSQL test environment
 
+export PGAPPNAME=cartodb_tiler_tester
+
 cd $(dirname $0)
 BASEDIR=$(pwd)
 cd -

test/acceptance/server.js

@@ -620,6 +620,24 @@ suite('server', function() {
         });
     });
 
+    // See http://github.com/CartoDB/Windshaft-cartodb/issues/186
+    test("get'ing the grid of a private table should fail when unauthenticated (jsonp)",
+    function(done) {
+        assert.response(server, {
+            headers: {host: 'localhost'},
+            url: '/tiles/test_table_private_1/6/31/24.grid.json?callback=x',
+            method: 'GET'
+        },{}, function(res) {
+          // It's forbidden, but jsonp calls for status = 200
+          assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+          // Still, we do NOT want to add caching headers here
+          // See https://github.com/CartoDB/Windshaft-cartodb/issues/186
+          assert.ok(!res.headers.hasOwnProperty('cache-control'),
+            "Unexpected Cache-Control: " + res.headers['cache-control']);
+          done();
+        });
+    });
+
     // See http://github.com/Vizzuality/Windshaft-cartodb/issues/55
     test("get'ing grid of private table should fail on unknown username",
     function(done) {
@@ -871,8 +889,8 @@ suite('server', function() {
     // See https://github.com/CartoDB/Windshaft-cartodb/issues/89
     test("get'ing a tile with a user-specific database password",  function(done){
         var style = querystring.stringify({style: test_style_black_200, style_version: '2.0.0'});
-        var backupDBPass = global.settings.postgres_auth_pass;
-        global.settings.postgres_auth_pass = '<%= user_password %>';
+        var backupDBPass = global.environment.postgres_auth_pass;
+        global.environment.postgres_auth_pass = '<%= user_password %>';
         Step (
           function() {
             var next = this;
@@ -899,7 +917,7 @@ suite('server', function() {
               return null
           },
           function finish(err) {
-            global.settings.postgres_auth_pass = backupDBPass;
+            global.environment.postgres_auth_pass = backupDBPass;
             done(err);
           }
         );

test/acceptance/templates.js

@@ -20,7 +20,8 @@ var helper = require(__dirname + '/../support/test_helper');
 var windshaft_fixtures = __dirname + '/../../node_modules/windshaft/test/fixtures';
 
 var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft');
-var serverOptions = require(__dirname + '/../../lib/cartodb/server_options')();
+var ServerOptions = require(__dirname + '/../../lib/cartodb/server_options');
+var serverOptions = ServerOptions();
 var server = new CartodbWindshaft(serverOptions);
 server.setMaxListeners(0);
 
@@ -1167,12 +1168,37 @@ suite('template_api', function() {
           assert.response(server, get_request, {},
             function(res) { next(null, res); });
         },
-        function checkTile(err, res) {
+        function checkTile_fetchOnRestart(err, res) {
           if ( err ) throw err;
           assert.equal(res.statusCode, 200, 
             'Unexpected error for authorized instance: '
             + res.statusCode + ' -- ' + res.body);
           assert.equal(res.headers['content-type'], "application/json; charset=utf-8");
+          var cc = res.headers['x-cache-channel'];
+          assert.ok(cc);
+          assert.ok(cc.match, /ciao/, cc);
+          // hack simulating restart...
+          serverOptions = ServerOptions(); // need to clean channel cache
+          server = new CartodbWindshaft(serverOptions);
+          var get_request = {
+              url: '/tiles/layergroup/' + layergroupid + ':cb1/0/0/0/1.json.torque?auth_token=valid1',
+              method: 'GET',
+              headers: {host: 'localhost' },
+              encoding: 'binary'
+          }
+          var next = this;
+          assert.response(server, get_request, {},
+            function(res) { next(null, res); });
+        },
+        function checkCacheChannel(err, res) {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 200, 
+            'Unexpected error for authorized instance: '
+            + res.statusCode + ' -- ' + res.body);
+          assert.equal(res.headers['content-type'], "application/json; charset=utf-8");
+          var cc = res.headers['x-cache-channel'];
+          assert.ok(cc, "Missing X-Cache-Channel on fetch-after-restart");
+          assert.ok(cc.match, /ciao/, cc);
           return null;
         },
         function deleteTemplate(err)

test/support/test_helper.js

@@ -10,9 +10,7 @@ var assert = require('assert');
 var LZMA  = require('lzma/lzma_worker.js').LZMA;
 
 // set environment specific variables
-global.settings     = require(__dirname + '/../../config/settings');
 global.environment  = require(__dirname + '/../../config/environments/test');
-_.extend(global.settings, global.environment);
 process.env.NODE_ENV = 'test';
 
 

tools/examples/mapconfig_slow.js

@@ -0,0 +1,11 @@
+{"version":"1.0.1",
+	"layers":[{
+    "type":"cartodb",
+	  "options":{
+      "sql":"select 1 as id, ST_Transform(ST_SetSRID(ST_MakePoint(x/1000,x/2000),4326),3857) as the_geom_webmercator FROM generate_series(-170000,170000) x",
+	    "cartocss":"#style{ marker-width: 12;}",
+	    "cartocss_version":"2.1.1",
+      "Interactivity":"id"
+    }
+  }]
+}

tools/show_style

@@ -38,7 +38,8 @@ if ( ! username ) usage(me, 1);
 console.log("Using environment " + ENV);
 
 global.environment = require('../config/environments/' + ENV);
-var serverOptions = require('../lib/cartodb/server_options'); // _after_ setting global.environment
+// _after_ setting global.environment
+var serverOptions = require('../lib/cartodb/server_options')();
 
 var client;
 var dbname;