CDB-3256 Prepares 1.12.1 release

Conflicts:
	NEWS.md
	package.json

.travis.yml

@@ -1,15 +1,27 @@
 before_install:
+  - sudo mv /etc/apt/sources.list.d/pgdg-source.list* /tmp
+  - sudo apt-get -qq purge postgis* postgresql*
+  - sudo apt-add-repository --yes ppa:cartodb/postgresql-9.3
+  - sudo apt-add-repository --yes ppa:cartodb/gis
+  - sudo rm -Rf /var/lib/postgresql /etc/postgresql
   - sudo apt-add-repository --yes ppa:mapnik/v2.1.0
-  - sudo apt-get update -q
-  - sudo apt-get install -q libmapnik-dev
+  - sudo apt-get update
+  - sudo apt-get install -y postgresql-9.3-postgis-2.1
+  - sudo apt-get install -y postgresql-contrib-9.3
+  - sudo apt-get install -y libmapnik-dev
+  - sudo apt-get install -y gdal-bin
+  - echo -e "local\tall\tall\ttrust\nhost\tall\tall\t127.0.0.1/32\ttrust\nhost\tall\tall\t::1/128\ttrust" |sudo tee /etc/postgresql/9.3/main/pg_hba.conf
+  - sudo service postgresql restart
   - createdb template_postgis
   - psql -c "CREATE EXTENSION postgis" template_postgis
+
+before_script:
   # Tell npm to use known registrars:
   # see http://blog.npmjs.org/post/78085451721/npms-self-signed-certificate-is-no-more
   - npm config set ca ""
 
 env:
-  - NPROCS=1 JOBS=1
+  - NPROCS=1 JOBS=1 PGUSER=postgres
 
 language: node_js
 node_js:

HOWTO_RELEASE

@@ -1,13 +1,20 @@
-1. Ensure proper version in package.json 
-2. Ensure NEWS section exists for the new version, review it, add release date
-3. Drop npm-shrinkwrap.json
-4. Run npm install
-5. Test (make check or npm test), fix if broken before proceeding
-6. Run npm shrinkwrap
-7. Set "from" in npm-shrinkwrap.json for known packages
-   (windshaft, node-varnish, grainstore...)
-8. Commit package.json, npm-shrinwrap.json, NEWS
-9. git tag -a Major.Minor.Patch # use NEWS section as content
-10. Announce
-11. Stub NEWS/package for next version
+1. Test (make clean all check), fix if broken before proceeding
+2. Ensure proper version in package.json 
+3. Ensure NEWS section exists for the new version, review it, add release date
+4. Drop npm-shrinkwrap.json
+5. Run npm shrinkwrap to recreate npm-shrinkwrap.json
+6. Commit package.json, npm-shrinwrap.json, NEWS
+7. git tag -a Major.Minor.Patch # use NEWS section as content
+8. Announce on cartodb@googlegroups.com
+9. Stub NEWS/package for next version
 
+Versions:
+
+Bugfix releases increment Patch component of version.
+Feature releases increment Minor and set Patch to zero.
+If backward compatibility is broken, increment Major and
+set to zero Minor and Patch.
+
+Branches named 'b<Major>.<Minor>' are kept for any critical
+fix that might need to be shipped before next feature release
+is ready.

NEWS.md

@@ -1,8 +1,40 @@
+1.12.1 -- 2014-06-24
+--------------------
+
+Enhancements:
+ - Caches layergroup and sets X-Cache-Channel in GET requests also in named maps
+
+1.12.0 -- 2014-06-24
+--------------------
+
+New features:
+ - Caches layergroup and sets X-Cache-Channel in GET requests
+
+1.11.1 -- 2014-05-07
+--------------------
+
+Enhancements:
+
+ - Upgrade Windshaft to 0.21.0, see
+ http://github.com/CartoDB/Windshaft/blob/0.21.0/NEWS
+
+1.11.0 -- 2014-04-28
+--------------------
+
+New features:
+
+ - Add support for log_filename directive
+ - Reopen log file on SIGHUP, for better logrotate integration
+
+Enhancements:
+
+ - Set default PostgreSQL application name to "cartodb_tiler"
+
 1.10.2 -- 2014-04-08
 --------------------
 
 Bug fixes:
- 
+
  - Fix show_style tool broken since 1.8.1
  - Fix X-Cache-Channel of tiles accessed via signed token (#188)
 
@@ -14,7 +46,7 @@ Bug fixes:
  - Do not cache non-success jsonp responses (#186)
 
 1.10.0 -- 2014-03-20
--------------------
+--------------------
 
 New features:
 

app.js

@@ -7,6 +7,10 @@
  * environments: [development, production]
  */
 
+var path = require('path'),
+    fs = require('fs')
+;
+
 
 if ( process.argv[2] ) ENV = process.argv[2];
 else if ( process.env['NODE_ENV'] ) ENV = process.env['NODE_ENV'];
@@ -21,22 +25,35 @@ if (ENV != 'development' && ENV != 'production' && ENV != 'staging' ){
     process.exit(1);
 }
 
-var _ = require('underscore')
-    , Step       = require('step')
-    ;
+var _ = require('underscore');
+
 // set environment specific variables
-global.settings     = require(__dirname + '/config/settings');
 global.environment  = require(__dirname + '/config/environments/' + ENV);
-_.extend(global.settings, global.environment);
 
 global.log4js = require('log4js')
 log4js_config = {
-  appenders: [
-    { type: "console", layout: { type:'basic' } }
-  ],
+  appenders: [],
   replaceConsole:true
 };
 
+if ( global.environment.log_filename ) {
+  var logdir = path.dirname(global.environment.log_filename);
+  // See cwd inlog4js.configure call below
+  logdir = path.resolve(__dirname, logdir);
+  if ( ! fs.existsSync(logdir) ) {
+    console.error("Log filename directory does not exist: " + logdir);
+    process.exit(1);
+  }
+  console.log("Logs will be written to " + global.environment.log_filename);
+  log4js_config.appenders.push(
+    { type: "file", filename: global.environment.log_filename }
+  );
+} else {
+  log4js_config.appenders.push(
+    { type: "console", layout: { type:'basic' } }
+  );
+}
+
 if ( global.environment.rollbar ) {
   log4js_config.appenders.push({
     type: __dirname + "/lib/cartodb/log4js_rollbar.js",
@@ -44,7 +61,7 @@ if ( global.environment.rollbar ) {
   });
 }
 
-log4js.configure(log4js_config);
+log4js.configure(log4js_config, { cwd: __dirname });
 global.logger = log4js.getLogger();
 
 // Include cartodb_windshaft only _after_ the "global" variable is set
@@ -80,6 +97,11 @@ process.on('SIGUSR2', function() {
   ws.dumpCacheStats();
 });
 
+process.on('SIGHUP', function() {
+  log4js.configure(log4js_config);
+  console.log('Log files reloaded');
+});
+
 process.on('uncaughtException', function(err) {
   logger.error('Uncaught exception: ' + err.stack); 
 });

config/environments/development.js.example

@@ -31,11 +31,15 @@ var config = {
     // to be able to navigate the map without a reload ?
     // Defaults to 7200 (2 hours)
     ,mapConfigTTL: 7200
-    // idle socket timeout, in miliseconds 
+    // idle socket timeout, in milliseconds 
     ,socket_timeout: 600000
     ,enable_cors: true
     ,cache_enabled: false
     ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
+    // If log_filename is given logs will be written
+    // there, in append mode. Otherwise stdout is used (default).
+    // Log file will be re-opened on receiving the HUP signal
+    ,log_filename: 'logs/node-windshaft.log'
     // Templated database username for authorized user
     // Supported labels: 'user_id' (read from redis)
     ,postgres_auth_user: 'development_cartodb_user_<%= user_id %>'
@@ -58,6 +62,14 @@ var config = {
         */
         row_limit: 65535,
         simplify_geometries: true,
+        /*
+         * Set persist_connection to false if you want
+         * database connections to be closed on renderer
+         * expiration (1 minute after last use).
+         * Setting to true (the default) would never
+         * close any connection for the server's lifetime
+         */
+        persist_connection: false,
         max_size: 500
     }
     ,mapnik_version: undefined
@@ -84,9 +96,9 @@ var config = {
         // Max number of connections in each pool.
         // Users will be put on a queue when the limit is hit.
         // Set to maxConnection to have no possible queues.
-        // There are currently 3 pools involved in serving
+        // There are currently 2 pools involved in serving
         // windshaft-cartodb requests so multiply this number
-        // by 3 to know how many possible connections will be
+        // by 2 to know how many possible connections will be
         // kept open by the server. The default is 50.
         max: 50,
         idleTimeoutMillis: 1, // idle time before dropping connection

config/environments/production.js.example

@@ -31,11 +31,15 @@ var config = {
     // to be able to navigate the map without a reload ?
     // Defaults to 7200 (2 hours)
     ,mapConfigTTL: 7200
-    // idle socket timeout, in miliseconds
+    // idle socket timeout, in milliseconds
     ,socket_timeout: 600000 
     ,enable_cors: true
     ,cache_enabled: true
     ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
+    // If log_filename is given logs will be written
+    // there, in append mode. Otherwise stdout is used (default).
+    // Log file will be re-opened on receiving the HUP signal
+    ,log_filename: 'logs/node-windshaft.log'
     // Templated database username for authorized user
     // Supported labels: 'user_id' (read from redis)
     ,postgres_auth_user: 'cartodb_user_<%= user_id %>'
@@ -51,6 +55,14 @@ var config = {
         port: 6432,
         extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
         row_limit: 65535,
+        /*
+         * Set persist_connection to false if you want
+         * database connections to be closed on renderer
+         * expiration (1 minute after last use).
+         * Setting to true (the default) would never
+         * close any connection for the server's lifetime
+         */
+        persist_connection: false,
         simplify_geometries: true,
         max_size: 500
     }
@@ -78,9 +90,9 @@ var config = {
         // Max number of connections in each pool.
         // Users will be put on a queue when the limit is hit.
         // Set to maxConnection to have no possible queues.
-        // There are currently 3 pools involved in serving
+        // There are currently 2 pools involved in serving
         // windshaft-cartodb requests so multiply this number
-        // by 3 to know how many possible connections will be
+        // by 2 to know how many possible connections will be
         // kept open by the server. The default is 50.
         max: 50,
         idleTimeoutMillis: 30000, // idle time before dropping connection

config/environments/staging.js.example

@@ -31,11 +31,15 @@ var config = {
     // to be able to navigate the map without a reload ?
     // Defaults to 7200 (2 hours)
     ,mapConfigTTL: 7200
-    // idle socket timeout, in miliseconds
+    // idle socket timeout, in milliseconds
     ,socket_timeout: 600000 
     ,enable_cors: true
     ,cache_enabled: true
     ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms (:res[X-Tiler-Profiler]) -> :res[Content-Type]'
+    // If log_filename is given logs will be written
+    // there, in append mode. Otherwise stdout is used (default).
+    // Log file will be re-opened on receiving the HUP signal
+    ,log_filename: 'logs/node-windshaft.log'
     // Templated database username for authorized user
     // Supported labels: 'user_id' (read from redis)
     ,postgres_auth_user: 'cartodb_staging_user_<%= user_id %>'
@@ -52,6 +56,14 @@ var config = {
         extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
         row_limit: 65535,
         simplify_geometries: true,
+        /*
+         * Set persist_connection to false if you want
+         * database connections to be closed on renderer
+         * expiration (1 minute after last use).
+         * Setting to true (the default) would never
+         * close any connection for the server's lifetime
+         */
+        persist_connection: false,
         max_size: 500
     }
     ,mapnik_version: undefined
@@ -78,9 +90,9 @@ var config = {
         // Max number of connections in each pool.
         // Users will be put on a queue when the limit is hit.
         // Set to maxConnection to have no possible queues.
-        // There are currently 3 pools involved in serving
+        // There are currently 2 pools involved in serving
         // windshaft-cartodb requests so multiply this number
-        // by 3 to know how many possible connections will be
+        // by 2 to know how many possible connections will be
         // kept open by the server. The default is 50.
         max: 50,
         idleTimeoutMillis: 30000, // idle time before dropping connection

config/environments/test.js.example

@@ -31,11 +31,15 @@ var config = {
     // to be able to navigate the map without a reload ?
     // Defaults to 7200 (2 hours)
     ,mapConfigTTL: 7200
-    // idle socket timeout, in miliseconds
+    // idle socket timeout, in milliseconds
     ,socket_timeout: 600000 
     ,enable_cors: true
     ,cache_enabled: false
     ,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
+    // If log_filename is given logs will be written
+    // there, in append mode. Otherwise stdout is used (default).
+    // Log file will be re-opened on receiving the HUP signal
+    //,log_filename: 'logs/node-windshaft.log'
     // Templated database username for authorized user
     // Supported labels: 'user_id' (read from redis)
     ,postgres_auth_user: 'test_windshaft_cartodb_user_<%= user_id %>'
@@ -52,6 +56,14 @@ var config = {
         extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
         row_limit: 65535,
         simplify_geometries: true,
+        /*
+         * Set persist_connection to false if you want
+         * database connections to be closed on renderer
+         * expiration (1 minute after last use).
+         * Setting to true (the default) would never
+         * close any connection for the server's lifetime
+         */
+        persist_connection: false,
         max_size: 500
     }
     ,mapnik_version: ''
@@ -78,9 +90,9 @@ var config = {
         // Max number of connections in each pool.
         // Users will be put on a queue when the limit is hit.
         // Set to maxConnection to have no possible queues.
-        // There are currently 3 pools involved in serving
+        // There are currently 2 pools involved in serving
         // windshaft-cartodb requests so multiply this number
-        // by 3 to know how many possible connections will be
+        // by 2 to know how many possible connections will be
         // kept open by the server. The default is 50.
         max: 50,
         idleTimeoutMillis: 1, // idle time before dropping connection

config/settings.js

@@ -1 +0,0 @@
-module.exports.oneDay   = 86400000;

docs/Template-maps.md

@@ -49,16 +49,17 @@ certificate that would be used to sign any instance of the template.
 ```js
 // template.json 
 {
-  version: '0.0.1',
+  version: "0.0.1",
   // there can be at most 1 template with the same name for any user 
   // valid names start with a letter and only contains letter, numbers
   // or underscores
-  name: 'template_name', 
+  name: "template_name", 
   // embedded authorization certificate
   auth: {
    // See https://github.com/CartoDB/Windshaft-cartodb/wiki/Signed-maps
-   method: 'token', // or "open" (the default if no "method" is given)
-   valid_tokens: ['auth_token1','auth_token2'] // only (required and non empty) for 'token' method
+   method: "token", // or "open" (the default if no "method" is given)
+   // only (required and non empty) for "token" method
+   valid_tokens: ["auth_token1","auth_token2"]
   },
   // Variables not listed here are not substituted
   // Variable not provided at instantiation time trigger an error
@@ -66,11 +67,11 @@ certificate that would be used to sign any instance of the template.
   // Type specification is used for quoting, to avoid injections
   placeholders: {
       color: {
-        type:'css_color',
-        default:'red'
+        type:"css_color",
+        default:"red"
       },
       cartodb_id: {
-        type:'number',
+        type:"number",
         default: 1
       }
   },

docs/metrics.md

@@ -0,0 +1,50 @@
+Windshaft-cartodb metrics
+=========================
+See [Windshaft metrics documentation](https://github.com/CartoDB/Windshaft/blob/master/doc/metrics.md) to understand the full picture.
+
+The next list includes the API endpoints, each endpoint may have several inner timers, some of them are displayed within this list as subitems. Find the description for them in the Inner timers section.
+## Timers
+- **windshaft-cartodb.get_infowindow**: time to retrieve an infowindow popup
+- **windshaft-cartodb.get_map_metadata**: time to retrieve metadata for embedded maps
+- **windshaft-cartodb.flush_cache**: time to flush the tile and sql cache
+- **windshaft-cartodb.get_template**: time to retrieve an specific template
+- **windshaft-cartodb.delete_template**: time to delete an specific template
+- **windshaft-cartodb.get_template_list**: time to retrieve the list of owned templates
+- **windshaft-cartodb.instance_template_post**: time to create a template via HTTP POST
+- **windshaft-cartodb.instance_template_get**: time to create a template via HTTP GET
+    + TemplateMaps_instance
+    + createLayergroup
+
+There are some endpoints that are not being tracked:
+- Adding a template
+- Updating a template
+
+### Inner timers
+Again, each inner timer may have several inner timers.
+
+- **addCacheChannel**: time to add X-Cache-Channel header based on table last modifications
+- **LZMA decompress**: time to decompress request params with LZMA
+- **TemplateMaps_instance**: time to retrieve a map template instance, see *getTemplate* and *authorizedByCert*
+- **affectedTables**: time to check what are the affected tables for adding the cache channel, see *addCacheChannel*
+- **authorize**: time to authorize a request, see *authorizedByAPIKey*, *authorizedByCert*, *authorizedBySigner*
+- **authorizedByAPIKey**: time to authorize using an API KEY
+- **authorizedByCert**: time to authorize a request by a cert, see [signed map](https://github.com/CartoDB/Windshaft-cartodb/wiki/Signed-maps)
+- **authorizedBySigner**: time to authorize a request for a [signed map](https://github.com/CartoDB/Windshaft-cartodb/wiki/Signed-maps)
+- **cartoData.getTableGeometryType**: time to retrieve from redis the geom type for a given table
+- **cors**: time to set the CORS headers
+- **findLastUpdated**: time to retrieve the last update time for a list of tables, see *affectedTables*
+- **fingerPrint**: time to create a fingerprint for a signed map
+- **generateCacheChannel**: time to generate the headers for the cache channel based on the request, see *addCacheChannel*
+- **getSignerMapKey**: time to retrieve from redis the authorized key for a signed map
+- **getTablePrivacy**: time to retrieve from redis the privacy of a table
+- **getTemplate**: time to retrieve from redis the template for a map
+- **getUserMapKey**: time to retrieve from redis the user key for a map
+- **incMapviewCount**: time to incremenent in redis the map views
+- **mapStore_load**: time to retrieve from redis a map configuration
+- **req2params.setup**: time to prepare the params from a request, see *req2params* in Windshaft documentation
+- **setDBAuth**: time to retrieve from redis and set db user and db password from a user
+- **setDBConn**: time to retrieve from redis and set db host and db name from a user
+- **setDBParams**: time to prepare all db params to be able to connect/query a database, see *setDBAuth* and *setDBConn*
+- **signMap**: time to sign in redis layergroup for a map, see signed maps
+- **tablePrivacy_getUserDBName**: time to retrieve from redis the database for a user
+

lib/cartodb/cartodb_windshaft.js

@@ -11,6 +11,9 @@ var _ = require('underscore')
     , os = require('os')
 ;
 
+if ( ! process.env['PGAPPNAME'] )
+  process.env['PGAPPNAME']='cartodb_tiler';
+
 var CartodbWindshaft = function(serverOptions) {
    var debug = global.environment.debug;
 
@@ -554,6 +557,8 @@ var CartodbWindshaft = function(serverOptions) {
           if ( err ) throw err;
           layergroup = instance;
           fakereq = { query: {}, params: {}, headers: _.clone(req.headers),
+            method: req.method,
+            res: res,
             profiler: req.profiler
           };
           ws.setDBParams(cdbuser, fakereq.params, this);

lib/cartodb/server_options.js

@@ -438,6 +438,17 @@ module.exports = function(){
             var cacheChannel = me.buildCacheChannel(dbName,tableNames);
             // store for caching from me.afterLayergroupCreate
             me.channelCache[cacheKey] = cacheChannel;
+            if (req.res && req.method == 'GET') {
+                var res = req.res;
+                if ( req.query && req.query.cache_policy == 'persist' ) {
+                    res.header('Cache-Control', 'public,max-age=31536000'); // 1 year
+                } else {
+                    var ttl = global.environment.varnish.ttl || 86400;
+                    res.header('Cache-Control', 'public,max-age='+ttl+',must-revalidate');
+                }
+                res.header('Last-Modified', (new Date()).toUTCString());
+                res.header('X-Cache-Channel', cacheChannel);
+            }
             // find last updated
             if ( ! tableNames.length ) return 0; // skip for no affected tables
             tabNames = tableNames;

npm-shrinkwrap.json

@@ -1,6 +1,6 @@
 {
   "name": "windshaft-cartodb",
-  "version": "1.10.2",
+  "version": "1.12.1",
   "dependencies": {
     "node-varnish": {
       "version": "0.3.0",
@@ -10,8 +10,8 @@
       "version": "1.3.3"
     },
     "windshaft": {
-      "version": "0.20.0",
-      "from": "http://github.com/CartoDB/Windshaft/tarball/0.20.0",
+      "version": "0.21.0",
+      "from": "http://github.com/CartoDB/Windshaft/tarball/0.21.0",
       "dependencies": {
         "grainstore": {
           "version": "0.18.1",
@@ -38,23 +38,23 @@
                       "version": "0.0.2"
                     },
                     "minimist": {
-                      "version": "0.0.8"
+                      "version": "0.0.10"
                     }
                   }
                 }
               }
             },
             "mapnik-reference": {
-              "version": "5.0.7"
+              "version": "5.0.9"
             },
             "millstone": {
-              "version": "0.6.11",
+              "version": "0.6.14",
               "dependencies": {
                 "underscore": {
-                  "version": "1.5.2"
+                  "version": "1.6.0"
                 },
                 "request": {
-                  "version": "2.26.0",
+                  "version": "2.34.0",
                   "dependencies": {
                     "qs": {
                       "version": "0.6.6"
@@ -65,6 +65,33 @@
                     "forever-agent": {
                       "version": "0.5.2"
                     },
+                    "node-uuid": {
+                      "version": "1.4.1"
+                    },
+                    "tough-cookie": {
+                      "version": "0.12.1",
+                      "dependencies": {
+                        "punycode": {
+                          "version": "1.2.4"
+                        }
+                      }
+                    },
+                    "form-data": {
+                      "version": "0.1.4",
+                      "dependencies": {
+                        "combined-stream": {
+                          "version": "0.0.5",
+                          "dependencies": {
+                            "delayed-stream": {
+                              "version": "0.0.5"
+                            }
+                          }
+                        },
+                        "async": {
+                          "version": "0.9.0"
+                        }
+                      }
+                    },
                     "tunnel-agent": {
                       "version": "0.3.0"
                     },
@@ -82,6 +109,9 @@
                         }
                       }
                     },
+                    "oauth-sign": {
+                      "version": "0.3.0"
+                    },
                     "hawk": {
                       "version": "1.0.0",
                       "dependencies": {
@@ -99,50 +129,252 @@
                         }
                       }
                     },
-                    "aws-sign": {
-                      "version": "0.3.0"
+                    "aws-sign2": {
+                      "version": "0.5.0"
+                    }
+                  }
+                },
+                "srs": {
+                  "version": "0.4.3",
+                  "dependencies": {
+                    "nan": {
+                      "version": "1.2.0"
                     },
-                    "oauth-sign": {
-                      "version": "0.3.0"
-                    },
-                    "cookie-jar": {
-                      "version": "0.3.0"
-                    },
-                    "node-uuid": {
-                      "version": "1.4.1"
-                    },
-                    "form-data": {
-                      "version": "0.1.2",
+                    "node-pre-gyp": {
+                      "version": "0.5.17",
                       "dependencies": {
-                        "combined-stream": {
-                          "version": "0.0.4",
+                        "nopt": {
+                          "version": "2.2.1",
                           "dependencies": {
-                            "delayed-stream": {
-                              "version": "0.0.5"
+                            "abbrev": {
+                              "version": "1.0.5"
                             }
                           }
                         },
-                        "async": {
-                          "version": "0.2.10"
+                        "npmlog": {
+                          "version": "0.0.6",
+                          "dependencies": {
+                            "ansi": {
+                              "version": "0.2.1"
+                            }
+                          }
+                        },
+                        "request": {
+                          "version": "2.36.0",
+                          "dependencies": {
+                            "qs": {
+                              "version": "0.6.6"
+                            },
+                            "json-stringify-safe": {
+                              "version": "5.0.0"
+                            },
+                            "mime": {
+                              "version": "1.2.11"
+                            },
+                            "forever-agent": {
+                              "version": "0.5.2"
+                            },
+                            "node-uuid": {
+                              "version": "1.4.1"
+                            },
+                            "tough-cookie": {
+                              "version": "0.12.1",
+                              "dependencies": {
+                                "punycode": {
+                                  "version": "1.2.4"
+                                }
+                              }
+                            },
+                            "form-data": {
+                              "version": "0.1.2",
+                              "dependencies": {
+                                "combined-stream": {
+                                  "version": "0.0.4",
+                                  "dependencies": {
+                                    "delayed-stream": {
+                                      "version": "0.0.5"
+                                    }
+                                  }
+                                },
+                                "async": {
+                                  "version": "0.2.10"
+                                }
+                              }
+                            },
+                            "tunnel-agent": {
+                              "version": "0.4.0"
+                            },
+                            "http-signature": {
+                              "version": "0.10.0",
+                              "dependencies": {
+                                "assert-plus": {
+                                  "version": "0.1.2"
+                                },
+                                "asn1": {
+                                  "version": "0.1.11"
+                                },
+                                "ctype": {
+                                  "version": "0.5.2"
+                                }
+                              }
+                            },
+                            "oauth-sign": {
+                              "version": "0.3.0"
+                            },
+                            "hawk": {
+                              "version": "1.0.0",
+                              "dependencies": {
+                                "hoek": {
+                                  "version": "0.9.1"
+                                },
+                                "boom": {
+                                  "version": "0.4.2"
+                                },
+                                "cryptiles": {
+                                  "version": "0.2.2"
+                                },
+                                "sntp": {
+                                  "version": "0.2.4"
+                                }
+                              }
+                            },
+                            "aws-sign2": {
+                              "version": "0.5.0"
+                            }
+                          }
+                        },
+                        "semver": {
+                          "version": "2.3.0"
+                        },
+                        "tar": {
+                          "version": "0.1.19",
+                          "dependencies": {
+                            "inherits": {
+                              "version": "2.0.1"
+                            },
+                            "block-stream": {
+                              "version": "0.0.7"
+                            },
+                            "fstream": {
+                              "version": "0.1.25",
+                              "dependencies": {
+                                "mkdirp": {
+                                  "version": "0.3.5"
+                                },
+                                "graceful-fs": {
+                                  "version": "2.0.3"
+                                }
+                              }
+                            }
+                          }
+                        },
+                        "tar-pack": {
+                          "version": "2.0.0",
+                          "dependencies": {
+                            "uid-number": {
+                              "version": "0.0.3"
+                            },
+                            "once": {
+                              "version": "1.1.1"
+                            },
+                            "debug": {
+                              "version": "0.7.4"
+                            },
+                            "fstream": {
+                              "version": "0.1.25",
+                              "dependencies": {
+                                "mkdirp": {
+                                  "version": "0.3.5"
+                                },
+                                "graceful-fs": {
+                                  "version": "2.0.3"
+                                },
+                                "inherits": {
+                                  "version": "2.0.1"
+                                }
+                              }
+                            },
+                            "fstream-ignore": {
+                              "version": "0.0.7",
+                              "dependencies": {
+                                "minimatch": {
+                                  "version": "0.2.14",
+                                  "dependencies": {
+                                    "lru-cache": {
+                                      "version": "2.5.0"
+                                    },
+                                    "sigmund": {
+                                      "version": "1.0.0"
+                                    }
+                                  }
+                                },
+                                "inherits": {
+                                  "version": "2.0.1"
+                                }
+                              }
+                            },
+                            "readable-stream": {
+                              "version": "1.0.27-1",
+                              "dependencies": {
+                                "core-util-is": {
+                                  "version": "1.0.1"
+                                },
+                                "isarray": {
+                                  "version": "0.0.1"
+                                },
+                                "string_decoder": {
+                                  "version": "0.10.25-1"
+                                },
+                                "inherits": {
+                                  "version": "2.0.1"
+                                }
+                              }
+                            },
+                            "graceful-fs": {
+                              "version": "1.2.3"
+                            }
+                          }
+                        },
+                        "mkdirp": {
+                          "version": "0.5.0",
+                          "dependencies": {
+                            "minimist": {
+                              "version": "0.0.8"
+                            }
+                          }
+                        },
+                        "rc": {
+                          "version": "0.4.0",
+                          "dependencies": {
+                            "minimist": {
+                              "version": "0.0.10"
+                            },
+                            "deep-extend": {
+                              "version": "0.2.10"
+                            },
+                            "strip-json-comments": {
+                              "version": "0.1.3"
+                            },
+                            "ini": {
+                              "version": "1.1.0"
+                            }
+                          }
+                        },
+                        "rimraf": {
+                          "version": "2.2.8"
                         }
                       }
                     }
                   }
                 },
-                "srs": {
-                  "version": "0.3.11"
-                },
                 "zipfile": {
-                  "version": "0.4.3"
-                },
-                "sqlite3": {
-                  "version": "2.2.0",
+                  "version": "0.5.2",
                   "dependencies": {
                     "node-pre-gyp": {
-                      "version": "0.2.6",
+                      "version": "0.5.8",
                       "dependencies": {
                         "nopt": {
-                          "version": "2.1.2",
+                          "version": "2.2.0",
                           "dependencies": {
                             "abbrev": {
                               "version": "1.0.4"
@@ -157,8 +389,92 @@
                             }
                           }
                         },
+                        "request": {
+                          "version": "2.34.0",
+                          "dependencies": {
+                            "qs": {
+                              "version": "0.6.6"
+                            },
+                            "json-stringify-safe": {
+                              "version": "5.0.0"
+                            },
+                            "forever-agent": {
+                              "version": "0.5.2"
+                            },
+                            "node-uuid": {
+                              "version": "1.4.1"
+                            },
+                            "mime": {
+                              "version": "1.2.11"
+                            },
+                            "tough-cookie": {
+                              "version": "0.12.1",
+                              "dependencies": {
+                                "punycode": {
+                                  "version": "1.2.4"
+                                }
+                              }
+                            },
+                            "form-data": {
+                              "version": "0.1.2",
+                              "dependencies": {
+                                "combined-stream": {
+                                  "version": "0.0.4",
+                                  "dependencies": {
+                                    "delayed-stream": {
+                                      "version": "0.0.5"
+                                    }
+                                  }
+                                },
+                                "async": {
+                                  "version": "0.2.10"
+                                }
+                              }
+                            },
+                            "tunnel-agent": {
+                              "version": "0.3.0"
+                            },
+                            "http-signature": {
+                              "version": "0.10.0",
+                              "dependencies": {
+                                "assert-plus": {
+                                  "version": "0.1.2"
+                                },
+                                "asn1": {
+                                  "version": "0.1.11"
+                                },
+                                "ctype": {
+                                  "version": "0.5.2"
+                                }
+                              }
+                            },
+                            "oauth-sign": {
+                              "version": "0.3.0"
+                            },
+                            "hawk": {
+                              "version": "1.0.0",
+                              "dependencies": {
+                                "hoek": {
+                                  "version": "0.9.1"
+                                },
+                                "boom": {
+                                  "version": "0.4.2"
+                                },
+                                "cryptiles": {
+                                  "version": "0.2.2"
+                                },
+                                "sntp": {
+                                  "version": "0.2.4"
+                                }
+                              }
+                            },
+                            "aws-sign2": {
+                              "version": "0.5.0"
+                            }
+                          }
+                        },
                         "semver": {
-                          "version": "2.1.0"
+                          "version": "2.2.1"
                         },
                         "tar": {
                           "version": "0.1.19",
@@ -208,6 +524,9 @@
                                 "minimatch": {
                                   "version": "0.2.14",
                                   "dependencies": {
+                                    "lru-cache": {
+                                      "version": "2.5.0"
+                                    },
                                     "sigmund": {
                                       "version": "1.0.0"
                                     }
@@ -219,10 +538,19 @@
                               }
                             },
                             "readable-stream": {
-                              "version": "1.0.26-2",
+                              "version": "1.0.26-4",
                               "dependencies": {
+                                "core-util-is": {
+                                  "version": "1.0.1"
+                                },
+                                "isarray": {
+                                  "version": "0.0.1"
+                                },
                                 "string_decoder": {
                                   "version": "0.10.25-1"
+                                },
+                                "inherits": {
+                                  "version": "2.0.1"
                                 }
                               }
                             },
@@ -231,22 +559,225 @@
                             }
                           }
                         },
-                        "aws-sdk": {
-                          "version": "2.0.0-rc9",
+                        "mkdirp": {
+                          "version": "0.3.5"
+                        },
+                        "rc": {
+                          "version": "0.3.4",
                           "dependencies": {
-                            "xml2js": {
-                              "version": "0.2.4",
+                            "minimist": {
+                              "version": "0.0.8"
+                            },
+                            "deep-extend": {
+                              "version": "0.2.8"
+                            },
+                            "ini": {
+                              "version": "1.1.0"
+                            }
+                          }
+                        },
+                        "rimraf": {
+                          "version": "2.2.6"
+                        }
+                      }
+                    }
+                  }
+                },
+                "sqlite3": {
+                  "version": "2.2.3",
+                  "dependencies": {
+                    "node-pre-gyp": {
+                      "version": "0.5.9",
+                      "dependencies": {
+                        "nopt": {
+                          "version": "2.2.0",
+                          "dependencies": {
+                            "abbrev": {
+                              "version": "1.0.4"
+                            }
+                          }
+                        },
+                        "npmlog": {
+                          "version": "0.0.6",
+                          "dependencies": {
+                            "ansi": {
+                              "version": "0.2.1"
+                            }
+                          }
+                        },
+                        "request": {
+                          "version": "2.34.0",
+                          "dependencies": {
+                            "qs": {
+                              "version": "0.6.6"
+                            },
+                            "json-stringify-safe": {
+                              "version": "5.0.0"
+                            },
+                            "forever-agent": {
+                              "version": "0.5.2"
+                            },
+                            "node-uuid": {
+                              "version": "1.4.1"
+                            },
+                            "mime": {
+                              "version": "1.2.11"
+                            },
+                            "tough-cookie": {
+                              "version": "0.12.1",
                               "dependencies": {
-                                "sax": {
-                                  "version": "0.6.0"
+                                "punycode": {
+                                  "version": "1.2.4"
                                 }
                               }
                             },
-                            "xmlbuilder": {
-                              "version": "0.4.2"
+                            "form-data": {
+                              "version": "0.1.2",
+                              "dependencies": {
+                                "combined-stream": {
+                                  "version": "0.0.4",
+                                  "dependencies": {
+                                    "delayed-stream": {
+                                      "version": "0.0.5"
+                                    }
+                                  }
+                                },
+                                "async": {
+                                  "version": "0.2.10"
+                                }
+                              }
+                            },
+                            "tunnel-agent": {
+                              "version": "0.3.0"
+                            },
+                            "http-signature": {
+                              "version": "0.10.0",
+                              "dependencies": {
+                                "assert-plus": {
+                                  "version": "0.1.2"
+                                },
+                                "asn1": {
+                                  "version": "0.1.11"
+                                },
+                                "ctype": {
+                                  "version": "0.5.2"
+                                }
+                              }
+                            },
+                            "oauth-sign": {
+                              "version": "0.3.0"
+                            },
+                            "hawk": {
+                              "version": "1.0.0",
+                              "dependencies": {
+                                "hoek": {
+                                  "version": "0.9.1"
+                                },
+                                "boom": {
+                                  "version": "0.4.2"
+                                },
+                                "cryptiles": {
+                                  "version": "0.2.2"
+                                },
+                                "sntp": {
+                                  "version": "0.2.4"
+                                }
+                              }
+                            },
+                            "aws-sign2": {
+                              "version": "0.5.0"
                             }
                           }
                         },
+                        "semver": {
+                          "version": "2.2.1"
+                        },
+                        "tar": {
+                          "version": "0.1.19",
+                          "dependencies": {
+                            "inherits": {
+                              "version": "2.0.1"
+                            },
+                            "block-stream": {
+                              "version": "0.0.7"
+                            },
+                            "fstream": {
+                              "version": "0.1.25",
+                              "dependencies": {
+                                "graceful-fs": {
+                                  "version": "2.0.3"
+                                }
+                              }
+                            }
+                          }
+                        },
+                        "tar-pack": {
+                          "version": "2.0.0",
+                          "dependencies": {
+                            "uid-number": {
+                              "version": "0.0.3"
+                            },
+                            "once": {
+                              "version": "1.1.1"
+                            },
+                            "debug": {
+                              "version": "0.7.4"
+                            },
+                            "fstream": {
+                              "version": "0.1.25",
+                              "dependencies": {
+                                "graceful-fs": {
+                                  "version": "2.0.3"
+                                },
+                                "inherits": {
+                                  "version": "2.0.1"
+                                }
+                              }
+                            },
+                            "fstream-ignore": {
+                              "version": "0.0.7",
+                              "dependencies": {
+                                "minimatch": {
+                                  "version": "0.2.14",
+                                  "dependencies": {
+                                    "lru-cache": {
+                                      "version": "2.5.0"
+                                    },
+                                    "sigmund": {
+                                      "version": "1.0.0"
+                                    }
+                                  }
+                                },
+                                "inherits": {
+                                  "version": "2.0.1"
+                                }
+                              }
+                            },
+                            "readable-stream": {
+                              "version": "1.0.26-4",
+                              "dependencies": {
+                                "core-util-is": {
+                                  "version": "1.0.1"
+                                },
+                                "isarray": {
+                                  "version": "0.0.1"
+                                },
+                                "string_decoder": {
+                                  "version": "0.10.25-1"
+                                },
+                                "inherits": {
+                                  "version": "2.0.1"
+                                }
+                              }
+                            },
+                            "graceful-fs": {
+                              "version": "1.2.3"
+                            }
+                          }
+                        },
+                        "mkdirp": {
+                          "version": "0.3.5"
+                        },
                         "rc": {
                           "version": "0.3.4",
                           "dependencies": {
@@ -281,7 +812,7 @@
                       "version": "0.0.2"
                     },
                     "minimist": {
-                      "version": "0.0.8"
+                      "version": "0.0.10"
                     }
                   }
                 }
@@ -299,7 +830,7 @@
               "version": "1.9.2",
               "dependencies": {
                 "formidable": {
-                  "version": "1.0.14"
+                  "version": "1.0.15"
                 }
               }
             },
@@ -331,11 +862,8 @@
           }
         },
         "tilelive-mapnik": {
-          "version": "0.6.8",
+          "version": "0.6.9",
           "dependencies": {
-            "eio": {
-              "version": "0.2.2"
-            },
             "mime": {
               "version": "1.2.11"
             },
@@ -344,9 +872,6 @@
             }
           }
         },
-        "lru-cache": {
-          "version": "2.3.1"
-        },
         "carto": {
           "version": "0.9.5-cdb3",
           "from": "http://github.com/CartoDB/carto/tarball/0.9.5-cdb3",
@@ -355,7 +880,7 @@
               "version": "1.4.4"
             },
             "mapnik-reference": {
-              "version": "5.0.7"
+              "version": "5.0.9"
             },
             "xml2js": {
               "version": "0.2.8",
@@ -372,7 +897,7 @@
                   "version": "0.0.2"
                 },
                 "minimist": {
-                  "version": "0.0.8"
+                  "version": "0.0.10"
                 }
               }
             }
@@ -382,14 +907,6 @@
           "version": "0.0.1",
           "from": "git://github.com/CartoDB/node-step-profiler.git#0.0.1"
         },
-        "underscore.string": {
-          "version": "1.1.6",
-          "dependencies": {
-            "underscore": {
-              "version": "1.1.7"
-            }
-          }
-        },
         "pg": {
           "version": "2.6.2",
           "dependencies": {
@@ -426,10 +943,13 @@
           "version": "2.0.4"
         },
         "hiredis": {
-          "version": "0.1.16",
+          "version": "0.1.17",
           "dependencies": {
             "bindings": {
-              "version": "1.1.1"
+              "version": "1.2.0"
+            },
+            "nan": {
+              "version": "1.1.2"
             }
           }
         }
@@ -443,29 +963,41 @@
       "version": "1.2.3"
     },
     "log4js": {
-      "version": "0.6.10",
+      "version": "0.6.14",
       "dependencies": {
         "async": {
           "version": "0.1.15"
         },
         "readable-stream": {
-          "version": "1.0.26",
+          "version": "1.0.27-1",
           "dependencies": {
+            "core-util-is": {
+              "version": "1.0.1"
+            },
+            "isarray": {
+              "version": "0.0.1"
+            },
             "string_decoder": {
               "version": "0.10.25-1"
+            },
+            "inherits": {
+              "version": "2.0.1"
             }
           }
         }
       }
     },
     "rollbar": {
-      "version": "0.3.1",
+      "version": "0.3.8",
       "dependencies": {
         "node-uuid": {
           "version": "1.4.1"
         },
         "lru-cache": {
           "version": "2.2.4"
+        },
+        "json-stringify-safe": {
+          "version": "5.0.0"
         }
       }
     },
@@ -502,7 +1034,12 @@
           "version": "1.0.7"
         },
         "debug": {
-          "version": "0.7.4"
+          "version": "1.0.2",
+          "dependencies": {
+            "ms": {
+              "version": "0.6.2"
+            }
+          }
         },
         "mkdirp": {
           "version": "0.3.5"
@@ -522,7 +1059,7 @@
               }
             },
             "graceful-fs": {
-              "version": "2.0.2"
+              "version": "2.0.3"
             },
             "inherits": {
               "version": "2.0.1"

package.json

@@ -1,7 +1,7 @@
 {
     "private": true,
     "name": "windshaft-cartodb",
-    "version": "1.10.2",
+    "version": "1.12.1",
     "description": "A map tile server for CartoDB",
     "keywords": [
       "cartodb"
@@ -24,7 +24,7 @@
     "dependencies": {
         "node-varnish": "http://github.com/Vizzuality/node-varnish/tarball/0.3.0",
         "underscore"  : "~1.3.3",
-        "windshaft"  : "http://github.com/CartoDB/Windshaft/tarball/0.20.0",
+        "windshaft"  : "http://github.com/CartoDB/Windshaft/tarball/0.21.0",
         "step": "0.0.x",
         "request": "2.9.202",
         "cartodb-redis": "~0.3.0",

run_tests.sh

@@ -5,6 +5,8 @@ OPT_CREATE_PGSQL=yes # create the PostgreSQL test environment
 OPT_DROP_REDIS=yes   # drop the redis test environment
 OPT_DROP_PGSQL=yes   # drop the PostgreSQL test environment
 
+export PGAPPNAME=cartodb_tiler_tester
+
 cd $(dirname $0)
 BASEDIR=$(pwd)
 cd -

test/acceptance/multilayer.js

@@ -226,9 +226,7 @@ suite('multilayer', function() {
     });
 
 
-    // See https://github.com/CartoDB/Windshaft-cartodb/issues/176
-    // NOTE: another test like this is in templates.js
-    test("get creation requests no cache", function(done) {
+    test("get creation requests has cache", function(done) {
 
       var layergroup =  {
         version: '1.0.0',
@@ -257,7 +255,7 @@ suite('multilayer', function() {
           assert.equal(res.statusCode, 200, res.body);
           var parsedBody = JSON.parse(res.body);
           expected_token = parsedBody.layergroupid.split(':')[0];
-          helper.checkNoCache(res);
+          helper.checkCache(res);
           return null;
         },
         function finish(err) {
@@ -279,6 +277,49 @@ suite('multilayer', function() {
       );
     });
 
+    test("get creation has no cache if sql is bogus", function(done) {
+        var layergroup =  {
+            version: '1.0.0',
+            layers: [
+                { options: {
+                    sql: 'select bogus(0,0) as the_geom_webmercator',
+                    cartocss: '#layer { polygon-fill: red; }',
+                    cartocss_version: '2.0.1'
+                } }
+            ]
+        };
+        assert.response(server, {
+            url: '/tiles/layergroup?config=' + encodeURIComponent(JSON.stringify(layergroup)),
+            method: 'GET',
+            headers: {host: 'localhost'}
+        }, {}, function(res) {
+            assert.notEqual(res.statusCode, 200);
+            helper.checkNoCache(res);
+            done();
+        });
+    });
+
+    test("get creation has no cache if cartocss is not valid", function(done) {
+        var layergroup =  {
+            version: '1.0.0',
+            layers: [
+                { options: {
+                    sql: 'select cartodb_id, ST_Translate(the_geom_webmercator, 5e6, 0) as the_geom_webmercator from test_table limit 2',
+                    cartocss: '#layer { invalid-rule:red; }',
+                    cartocss_version: '2.0.1'
+                } }
+            ]
+        };
+        assert.response(server, {
+            url: '/tiles/layergroup?config=' + encodeURIComponent(JSON.stringify(layergroup)),
+            method: 'GET',
+            headers: {host: 'localhost'}
+        }, {}, function(res) {
+            assert.notEqual(res.statusCode, 200);
+            helper.checkNoCache(res);
+            done();
+        });
+    });
 
     test("layergroup can hold substitution tokens", function(done) {
 

test/acceptance/server.js

@@ -889,8 +889,8 @@ suite('server', function() {
     // See https://github.com/CartoDB/Windshaft-cartodb/issues/89
     test("get'ing a tile with a user-specific database password",  function(done){
         var style = querystring.stringify({style: test_style_black_200, style_version: '2.0.0'});
-        var backupDBPass = global.settings.postgres_auth_pass;
-        global.settings.postgres_auth_pass = '<%= user_password %>';
+        var backupDBPass = global.environment.postgres_auth_pass;
+        global.environment.postgres_auth_pass = '<%= user_password %>';
         Step (
           function() {
             var next = this;
@@ -917,7 +917,7 @@ suite('server', function() {
               return null
           },
           function finish(err) {
-            global.settings.postgres_auth_pass = backupDBPass;
+            global.environment.postgres_auth_pass = backupDBPass;
             done(err);
           }
         );

test/acceptance/templates.js

@@ -1593,7 +1593,7 @@ suite('template_api', function() {
           if ( err ) throw err;
           assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
           // See https://github.com/CartoDB/Windshaft-cartodb/issues/176
-          helper.checkNoCache(res);
+          helper.checkCache(res);
           return null;
         },
         function finish(err) {

test/support/test_helper.js

@@ -10,9 +10,7 @@ var assert = require('assert');
 var LZMA  = require('lzma/lzma_worker.js').LZMA;
 
 // set environment specific variables
-global.settings     = require(__dirname + '/../../config/settings');
 global.environment  = require(__dirname + '/../../config/environments/test');
-_.extend(global.settings, global.environment);
 process.env.NODE_ENV = 'test';
 
 
@@ -39,8 +37,21 @@ function checkNoCache(res) {
 }
 
 
-module.exports = {
-  lzma_compress_to_base64: lzma_compress_to_base64,
-  checkNoCache: checkNoCache
+/**
+ * Check that the response headers do not request caching
+ * @see checkNoCache
+ * @param res
+ */
+function checkCache(res) {
+    assert.ok(res.headers.hasOwnProperty('x-cache-channel'));
+    assert.ok(res.headers.hasOwnProperty('cache-control'));
+    assert.ok(res.headers.hasOwnProperty('last-modified'));
 }
 
+
+module.exports = {
+  lzma_compress_to_base64: lzma_compress_to_base64,
+  checkNoCache: checkNoCache,
+  checkCache: checkCache
+};
+

tools/examples/mapconfig_slow.js

@@ -0,0 +1,11 @@
+{"version":"1.0.1",
+	"layers":[{
+    "type":"cartodb",
+	  "options":{
+      "sql":"select 1 as id, ST_Transform(ST_SetSRID(ST_MakePoint(x/1000,x/2000),4326),3857) as the_geom_webmercator FROM generate_series(-170000,170000) x",
+	    "cartocss":"#style{ marker-width: 12;}",
+	    "cartocss_version":"2.1.1",
+      "Interactivity":"id"
+    }
+  }]
+}