updated NEW for v1.1.0

Ensures all errors are logged

.gitignore

@@ -1,2 +1,3 @@
 node_modules
-.idea
+config/environments/*.js
+.idea

Makefile

@@ -4,5 +4,8 @@ all:
 clean:
 	rm -rf node_modules/*
 
-check:
+config/environments/test.js: config/environments/test.js.example
+	./configure
+
+check: config/environments/test.js
 	./run_tests.sh

NEWS.md

@@ -0,0 +1,27 @@
+1.1.0 (30/10/12)
+-----
+* Add /version entry point
+* CartoCSS versioning
+ * Include version in GET /style response
+ * Support version and convert parameters in POST /style request
+ * Autodetect target mapnik version and let config override it
+ * Add tools/reset_styles script to batch-reset (and optionally convert) styles
+* Configurable logging format (#4)
+* Detailed error on missing user metadata 
+* Properly handle unauthenticated requests for metadata
+* Accept "api_key" in addition to "map_key",
+  both in query_string and POST body (#38)
+* Add ./configure script
+* Allow listening on host IP
+* Replaced environment configs by .example ones
+* Fixed some issues with cluster2
+
+1.0.0 (03/10/12)
+-----
+* Migrated to node 0.8.x.
+
+0.9.0 (25/09/12)
+-----
+* External resources in CartoCSS
+* Added X-Cache-Channel header in all the tiler GET requests
+* Small fixes

README.md

@@ -1,21 +1,47 @@
 Windshaft-CartoDB
 ==================
 
-NOTE: requires node-0.4.x
+NOTE: requires node-0.8.x
 
 This is the CartoDB map tiler. It extends Windshaft with some extra
 functionality and custom filters for authentication
 
 * reads dbname from subdomain and cartodb redis for pretty tile urls
-* configures windshaft to publish cartodb_id as the interactivity layer
+* configures windshaft to publish ``cartodb_id`` as the interactivity layer
 * gets the default geometry type from the cartodb redis store
 * allows tiles to be styled individually
 * provides a link to varnish high speed cache
 * provides a infowindow endpoint for windshaft
-* provides a map_metadata endpoint for windshaft
+* provides a ``map_metadata`` endpoint for windshaft
 
-Install
--------
+Requirements
+------------
+
+ [core]
+ - node-0.6.x+
+ - PostgreSQL-8.3+
+ - PostGIS-1.5.0+
+ - Redis 2.2.0+ (http://www.redis.io)
+ - Mapnik 2.0 or 2.1 
+
+ [for cache control]
+ - CartoDB-SQL-API 1.0.0+
+ - CartoDB 0.9.5+ (for ``CDB_QueryTables``)
+ - Varnish (https://www.varnish-cache.org)
+
+Configure
+---------
+
+Create the config/environments/<env>.js files (there are .example files
+to start from). You can optionally use the ./configure script for this,
+see ```./configure --help``` to see available options.
+
+Look at lib/cartodb/server_options.js for more on config
+
+Build/install
+-------------
+
+To fetch and build all node-based dependencies, run:
 
 ```
 git clone
@@ -28,22 +54,18 @@ happen to have startup errors you may need to force rebuilding those
 modules. At any time just wipe out the node_modules/ directory and run
 ```npm install``` again.
 
-Configure
----------
-
-Edit config/environments/<env>.js files
-
-Look at lib/cartodb/server_options for more on config
 
 Run
 ---
 
 ```
-node app.js [development | staging | production]
+node app.js <env> 
 ```
 
-Note that caches are kept in redis. If you're not seeing what
-you expect there may be out-of-sync records in there.
+Where <env> is the name of a configuration file under config/environments/.
+
+Note that caches are kept in redis. If you're not seeing what you expect
+there may be out-of-sync records in there.
 Take a look: http://redis.io/commands
 
 

app.js

@@ -33,5 +33,7 @@ var Windshaft = require('windshaft');
 var serverOptions = require('./lib/cartodb/server_options');
 
 ws = CartodbWindshaft(serverOptions);
-ws.listen(global.environment.port);
-console.log("Windshaft tileserver started on port " + global.environment.port);
+ws.listen(global.environment.port, global.environment.host);
+ws.on('listening', function() {
+  console.log("Windshaft tileserver started on " + global.environment.host + ':' + global.environment.port);
+});

cluster.js

@@ -7,7 +7,7 @@
  * environments: [development, production]
  */
 
-var cluster = require('cluster');
+var Cluster = require('cluster2');
 
 // sanity check
 var ENV = process.argv[2]
@@ -34,12 +34,22 @@ var cartoData = require('./lib/cartodb/carto_data');
 var Windshaft = require('windshaft');
 var serverOptions = require('./lib/cartodb/server_options');
 
-ws = CartodbWindshaft(serverOptions);
-cluster(ws)
-  .use(cluster.logger('logs'))
-  .use(cluster.stats())
-  .use(cluster.pidfiles('pids'))
-  .set('workers', 1)
-  .listen(global.environment.port, global.environment.host);
+var ws = CartodbWindshaft(serverOptions);
+
+//.use(cluster.logger('logs'))
+//.use(cluster.stats())
+//.use(cluster.pidfiles('pids'))
+var cluster = new Cluster({
+  port: global.environment.port,
+  host: global.environment.host,
+  monPort: global.environment.port+1,
+  monHost: global.environment.host,
+  noWorkers: 1 
+});
+
+cluster.listen(function(cb) {
+  cb(ws);
+}, function() {
+  console.log("Windshaft tileserver started on port " + global.environment.port);
+});
 
-console.log("Windshaft tileserver started on port " + global.environment.port);

config/environments/development.js.example

@@ -4,6 +4,7 @@ var config = {
     ,host: '127.0.0.1'
     ,enable_cors: true
     ,cache_enabled: false
+    ,log_format: '[:date] :req[X-Real-IP] \033[90m:method\033[0m \033[36m:req[Host]:url\033[0m \033[90m:status :response-time ms -> :res[Content-Type]\033[0m'
     ,postgres_auth_user: 'development_cartodb_user_<%= user_id %>'
     ,postgres: {
         type: "postgis",
@@ -18,6 +19,7 @@ var config = {
         */
         simplify: true
     }
+    ,mapnik_version: undefined
     ,millstone: {
         cache_basedir: '/tmp/cdb-tiler-dev/millstone-dev'
     }

config/environments/production.js.example

@@ -4,6 +4,7 @@ var config = {
     ,host: '127.0.0.1'
     ,enable_cors: true
     ,cache_enabled: true
+    ,log_format: '[:date] :req[X-Real-IP] \033[90m:method\033[0m \033[36m:req[Host]:url\033[0m \033[90m:status :response-time ms -> :res[Content-Type]\033[0m'
     ,postgres_auth_user: 'cartodb_user_<%= user_id %>'
     ,postgres: {
         user: "publicuser",
@@ -12,6 +13,7 @@ var config = {
         extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188",
         simplify: true
     }
+    ,mapnik_version: undefined
     ,millstone: {
         cache_basedir: '/home/ubuntu/tile_assets/'
     }

config/environments/staging.js.example

@@ -4,6 +4,7 @@ var config = {
     ,host: '127.0.0.1'
     ,enable_cors: true
     ,cache_enabled: true
+    ,log_format: '[:date] :req[X-Real-IP] \033[90m:method\033[0m \033[36m:req[Host]:url\033[0m \033[90m:status :response-time ms -> :res[Content-Type]\033[0m'
     ,postgres_auth_user: 'cartodb_staging_user_<%= user_id %>'
     ,postgres: {
         user: "publicuser",
@@ -12,6 +13,7 @@ var config = {
         extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188",
         simplify: true
     }
+    ,mapnik_version: undefined
     ,millstone: {
         cache_basedir: '/home/ubuntu/tile_assets/'
     }

config/environments/test.js.example

@@ -4,6 +4,7 @@ var config = {
     ,host: '127.0.0.1'
     ,enable_cors: true
     ,cache_enabled: false
+    ,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type]'
     ,postgres_auth_user: 'test_cartodb_user_<%= user_id %>'
     ,postgres: {
         user: "publicuser",
@@ -13,6 +14,7 @@ var config = {
         extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188",
         simplify: true
     }
+    ,mapnik_version: '2.0.2'
     ,millstone: {
         cache_basedir: '/tmp/cdb-tiler-test/millstone'
     }

configure

@@ -0,0 +1,55 @@
+#!/bin/sh
+
+#
+# This script creates config/environments/*.js files using
+# config/environments/*.js.example files as input and performing
+# settings substitutions.
+#
+# It relies on a known format of the .js.example files which haven't
+# been made easier to parse to still let humans copy them manually and
+# do further editing or leave them as such to get the same setup as before
+# the introduction of this script.
+#
+# The script is a work in progress. Available switches are printed
+# by invoking with the --help switch. More switches will be added
+# as the need/request for them arises.
+#
+# --strk(2012-07-23)
+#
+
+usage() {
+  echo "Usage: $0 [OPTION]"
+  echo 
+  echo "Configuration:"
+  echo "  --help                 display this help and exit"
+  echo "  --with-pgport=NUM      access PostgreSQL server on TCP port NUM"
+}
+
+PGPORT=5432
+
+while test -n "$1"; do
+  case "$1" in
+    --help|-h)
+      usage
+      exit 0
+      ;;
+    --with-pgport=*)
+      PGPORT=`echo "$1" | cut -d= -f2`
+      ;;
+    *)
+      echo "Unknown option '$1'" >&2
+      usage >&2
+      exit 1
+  esac
+  shift
+done
+
+echo "PGPORT: $PGPORT"
+
+# TODO: allow specifying configuration settings !
+for f in config/environments/*.example; do
+  o=`dirname "$f"`/`basename "$f" .example`
+  echo "Writing $o"
+  # See http://austinmatzko.com/2008/04/26/sed-multi-line-search-and-replace/
+  sed -n "1h;1!H;\${;g;s/\(,postgres: {[^}]*port: *'\?\)[^',]*\('\?,\)/\1$PGPORT\2/;p;}" < "$f" > "$o"
+done

lib/cartodb/carto_data.js

@@ -27,14 +27,20 @@ module.exports = function() {
      * Get the database name for this particular subdomain/username
      *
      * @param req - standard express req object. importantly contains host information
-     * @param callback
+     * @param callback - gets called with args(err, dbname) 
      */
     me.getDatabase = function(req, callback) {
         // strip subdomain from header host
         var username = req.headers.host.split('.')[0]
         var redisKey = _.template(this.user_key, {username: username});
 
-        this.retrieve(this.user_metadata_db, redisKey, 'database_name', callback);
+        this.retrieve(this.user_metadata_db, redisKey, 'database_name', function(err, dbname) {
+          if ( err ) callback(err, null);
+          else if ( dbname === null ) {
+            callback(new Error("missing " + username + "'s dbname in redis (try CARTODB/script/restore_redis)"), null);
+          }
+          else callback(err, dbname);
+        });
     };
 
 
@@ -50,7 +56,13 @@ module.exports = function() {
         var username = req.headers.host.split('.')[0];
         var redisKey = _.template(this.user_key, {username: username});
 
-        this.retrieve(this.user_metadata_db, redisKey, 'id', callback);
+        this.retrieve(this.user_metadata_db, redisKey, 'id', function(err, dbname) {
+          if ( err ) callback(err, null);
+          else if ( dbname === null ) {
+            callback(new Error("missing " + username + "'s dbuser in redis (try CARTODB/script/restore_redis)"), null);
+          }
+          else callback(err, dbname);
+        });
     };
 
     /**
@@ -65,7 +77,13 @@ module.exports = function() {
         var redisKey = "rails:users:" + username; 
         this.retrieve(this.user_metadata_db, redisKey, "map_key", function(err, val) {
             var valid = 0;
-            if ( val && val == req.query.map_key ) valid = 1;
+            if ( val ) {
+              if ( val == req.query.map_key ) valid = 1;
+              else if ( val == req.query.api_key ) valid = 1;
+              // check also in request body 
+              else if ( req.body && req.body.map_key && val == req.body.map_key ) valid = 1;
+              else if ( req.body && req.body.api_key && val == req.body.api_key ) valid = 1;
+            }
             callback(err, valid);
         });
     };
@@ -182,6 +200,8 @@ module.exports = function() {
     };
 
     // Redis Hash lookup
+    // @param callback will be invoked with args (err, reply)
+    //                 note that reply is null when the key is missing
     me.retrieve = function(db, redisKey, hashKey, callback) {
         this.redisCmd(db,'HGET',[redisKey, hashKey], callback);
     };

lib/cartodb/cartodb_windshaft.js

@@ -6,17 +6,6 @@ var _ = require('underscore')
 
 var CartodbWindshaft = function(serverOptions) {
 
-    // set the cache chanel info to invalidate the cache on the frontend server
-    serverOptions.afterTileRender = function(req, res, tile, headers, callback) {
-        var ttl = global.environment.varnish.ttl || 86400;
-        Cache.generateCacheChannel(req, function(channel){
-            res.header('X-Cache-Channel', channel);
-            res.header('Last-Modified', new Date().toUTCString());
-            res.header('Cache-Control', 'no-cache,max-age='+ttl+',must-revalidate, public');
-            callback(null, tile, headers);
-        });
-    };
-
     if(serverOptions.cache_enabled) {
         console.log("cache invalidation enabled, varnish on ", serverOptions.varnish_host, ' ', serverOptions.varnish_port);
         Cache.init(serverOptions.varnish_host, serverOptions.varnish_port);
@@ -48,7 +37,8 @@ var CartodbWindshaft = function(serverOptions) {
             },
             function(err, data){
                 if (err){
-                    res.send({error: err.message}, 500);
+                    ws.sendError(res, {error: err.message}, 500, 'GET INFOWINDOW');
+                    //res.send({error: err.message}, 500);
                 } else {
                     res.send({infowindow: data}, 200);
                 }
@@ -68,7 +58,8 @@ var CartodbWindshaft = function(serverOptions) {
             },
             function(err, data){
                 if (err){
-                    res.send(err.message, 500);
+                    ws.sendError(res, {error: err.message}, 500, 'GET MAP_METADATA');
+                    //res.send(err.message, 500);
                 } else {
                     res.send({map_metadata: data}, 200);
                 }
@@ -84,11 +75,12 @@ var CartodbWindshaft = function(serverOptions) {
         ws.doCORS(res);
         Step(
             function(){
-                serverOptions.flushCache(req, Cache, this);
+                serverOptions.flushCache(req, serverOptions.cache_enabled ? Cache : null, this);
             },
             function(err, data){
                 if (err){
-                    res.send(500);
+                    ws.sendError(res, {error: err.message}, 500, 'DELETE CACHE');
+                    //res.send(500);
                 } else {
                     res.send({status: 'ok'}, 200);
                 }

lib/cartodb/server_options.js

@@ -1,22 +1,56 @@
 var   _          = require('underscore')
     , Step       = require('step')
-    , cartoData  = require('./carto_data');
+    , cartoData  = require('./carto_data')
+    , Cache = require('./cache_validator')
+    , mapnik     = require('mapnik')
+  ;
 
 module.exports = function(){
     var me = {
         base_url: '/tiles/:table',
         grainstore: {
           datasource: global.environment.postgres,
-          cachedir: global.environment.millstone.cache_basedir
+          cachedir: global.environment.millstone.cache_basedir,
+          mapnik_version: global.environment.mapnik_version || mapnik.versions.mapnik
         },
         redis: global.environment.redis,
         enable_cors: global.environment.enable_cors,
         varnish_host: global.environment.varnish.host,
         varnish_port: global.environment.varnish.port,
         cache_enabled: global.environment.cache_enabled,
-        log_format: '[:date] :req[X-Real-IP] \033[90m:method\033[0m \033[36m:req[Host]:url\033[0m \033[90m:status :response-time ms -> :res[Content-Type]\033[0m'
+        log_format: global.environment.log_format
     };
 
+    // Be nice and warn if configured mapnik version
+    // is != instaled mapnik version
+    if ( mapnik.versions.mapnik != me.grainstore.mapnik_version ) {
+      console.warn("WARNING: detected mapnik version ("
+        + mapnik.versions.mapnik + ") != configured mapnik version ("
+        + me.grainstore.mapnik_version + ")");
+    }
+
+    // Set the cache chanel info to invalidate the cache on the frontend server
+    //
+    // @param req The request object.
+    //            The function will have no effect unless req.res exists.
+    //            It is expected that req.params contains 'table' and 'dbname'
+    //
+    // @param cb function(err, channel) will be called when ready.
+    //           the channel parameter will be null if nothing was added
+    //
+    me.addCacheChannel = function(req, cb) {
+        // skip non-GET requests, or requests for which there's no response
+        if ( req.method != 'GET' || ! req.res ) { cb(null, null); return; }
+        var res = req.res;
+        var ttl = global.environment.varnish.ttl || 86400;
+        Cache.generateCacheChannel(req, function(channel){
+            res.header('X-Cache-Channel', channel);
+            res.header('Last-Modified', new Date().toUTCString());
+            res.header('Cache-Control', 'no-cache,max-age='+ttl+',must-revalidate, public');
+            cb(null, channel); // add last-modified too ?
+        });
+    }
+
     /**
      * Whitelist input and get database name & default geometry type from
      * subdomain/user metadata held in CartoDB Redis
@@ -26,7 +60,7 @@ module.exports = function(){
     me.req2params = function(req, callback){
 
         // Whitelist query parameters and attach format
-        var good_query = ['sql', 'geom_type', 'cache_buster','callback', 'interactivity', 'map_key', 'style'];
+        var good_query = ['sql', 'geom_type', 'cache_buster','callback', 'interactivity', 'map_key', 'api_key', 'style'];
         var bad_query  = _.difference(_.keys(req.query), good_query);
 
         _.each(bad_query, function(key){ delete req.query[key]; });
@@ -45,6 +79,8 @@ module.exports = function(){
           callback(null, xml);
         }
 
+        var that = this;
+
         Step(
             function getPrivacy(){
                 cartoData.authorize(req, this);
@@ -66,10 +102,14 @@ module.exports = function(){
                 cartoData.getGeometryType(req, this);
             },
             function finishSetup(err, data){
+                if ( err ) { callback(err, req); return; }
+
                 if (!_.isNull(data))
                     _.extend(req.params, {geom_type: data});
 
-                callback(err, req);
+                that.addCacheChannel(req, function(err, chan) {
+                  callback(err, req);
+                });
             }
         );
     };
@@ -106,8 +146,8 @@ module.exports = function(){
                 that.req2params(req, this);
             },
             function(err, data){
-                if (err) throw err;
-                cartoData.getMapMetadata(data, callback);
+                if (err) callback(err, null);
+                else cartoData.getMapMetadata(data, callback);
             }
         );
     };
@@ -126,7 +166,9 @@ module.exports = function(){
             },
             function(err, data){
                 if (err) throw err;
-                Cache.invalidate_db(req.params.dbname, req.params.table);
+                if(Cache) {
+                  Cache.invalidate_db(req.params.dbname, req.params.table);
+                }
                 callback(null, true);
             }
         );

npm-shrinkwrap.json

@@ -0,0 +1,346 @@
+{
+  "name": "windshaft-cartodb",
+  "version": "1.1.0",
+  "dependencies": {
+    "cluster2": {
+      "version": "0.3.5-cdb02",
+      "from": "git://github.com/CartoDB/cluster2.git#cdb_production",
+      "dependencies": {
+        "express": {
+          "version": "2.5.11",
+          "dependencies": {
+            "connect": {
+              "version": "1.9.2",
+              "dependencies": {
+                "formidable": {
+                  "version": "1.0.11"
+                }
+              }
+            },
+            "mime": {
+              "version": "1.2.4"
+            },
+            "qs": {
+              "version": "0.4.2"
+            },
+            "mkdirp": {
+              "version": "0.3.0"
+            }
+          }
+        },
+        "ejs": {
+          "version": "0.8.3"
+        },
+        "npm": {
+          "version": "1.1.62",
+          "dependencies": {
+            "semver": {
+              "version": "1.0.14"
+            },
+            "ini": {
+              "version": "1.0.4"
+            },
+            "slide": {
+              "version": "1.1.3"
+            },
+            "abbrev": {
+              "version": "1.0.3"
+            },
+            "graceful-fs": {
+              "version": "1.1.14"
+            },
+            "minimatch": {
+              "version": "0.2.6"
+            },
+            "nopt": {
+              "version": "2.0.0"
+            },
+            "rimraf": {
+              "version": "2.0.2"
+            },
+            "request": {
+              "version": "2.9.203",
+              "from": "git://github.com/isaacs/request"
+            },
+            "which": {
+              "version": "1.0.5"
+            },
+            "tar": {
+              "version": "0.1.13"
+            },
+            "fstream": {
+              "version": "0.1.19"
+            },
+            "block-stream": {
+              "version": "0.0.6"
+            },
+            "inherits": {
+              "version": "1.0.0",
+              "from": "git://github.com/isaacs/inherits"
+            },
+            "mkdirp": {
+              "version": "0.3.4"
+            },
+            "read": {
+              "version": "1.0.4",
+              "dependencies": {
+                "mute-stream": {
+                  "version": "0.0.3"
+                }
+              }
+            },
+            "lru-cache": {
+              "version": "2.0.4"
+            },
+            "node-gyp": {
+              "version": "0.6.11"
+            },
+            "fstream-npm": {
+              "version": "0.1.2",
+              "dependencies": {
+                "fstream-ignore": {
+                  "version": "0.0.5"
+                }
+              }
+            },
+            "uid-number": {
+              "version": "0.0.3"
+            },
+            "archy": {
+              "version": "0.0.2"
+            },
+            "chownr": {
+              "version": "0.0.1"
+            },
+            "npmlog": {
+              "version": "0.0.2"
+            },
+            "ansi": {
+              "version": "0.1.2"
+            },
+            "npm-registry-client": {
+              "version": "0.2.7"
+            },
+            "read-package-json": {
+              "version": "0.1.5"
+            },
+            "read-installed": {
+              "version": "0.0.2"
+            },
+            "glob": {
+              "version": "3.1.12"
+            },
+            "init-package-json": {
+              "version": "0.0.5",
+              "dependencies": {
+                "promzard": {
+                  "version": "0.2.0"
+                }
+              }
+            },
+            "osenv": {
+              "version": "0.0.3"
+            },
+            "lockfile": {
+              "version": "0.2.1"
+            },
+            "retry": {
+              "version": "0.6.0"
+            },
+            "couch-login": {
+              "version": "0.1.12"
+            },
+            "once": {
+              "version": "1.1.1"
+            },
+            "npmconf": {
+              "version": "0.0.16",
+              "dependencies": {
+                "config-chain": {
+                  "version": "1.1.2",
+                  "dependencies": {
+                    "proto-list": {
+                      "version": "1.2.2"
+                    }
+                  }
+                }
+              }
+            },
+            "opener": {
+              "version": "1.3.0"
+            }
+          }
+        }
+      }
+    },
+    "node-varnish": {
+      "version": "0.1.1"
+    },
+    "underscore": {
+      "version": "1.3.3"
+    },
+    "grainstore": {
+      "version": "0.9.1",
+      "dependencies": {
+        "carto": {
+          "version": "0.9.2",
+          "dependencies": {
+            "mapnik-reference": {
+              "version": "5.0.0"
+            },
+            "xml2js": {
+              "version": "0.1.14",
+              "dependencies": {
+                "sax": {
+                  "version": "0.4.2"
+                }
+              }
+            }
+          }
+        },
+        "millstone": {
+          "version": "0.5.10",
+          "dependencies": {
+            "request": {
+              "version": "2.11.4",
+              "dependencies": {
+                "form-data": {
+                  "version": "0.0.3",
+                  "dependencies": {
+                    "combined-stream": {
+                      "version": "0.0.3",
+                      "dependencies": {
+                        "delayed-stream": {
+                          "version": "0.0.5"
+                        }
+                      }
+                    },
+                    "async": {
+                      "version": "0.1.9"
+                    }
+                  }
+                },
+                "mime": {
+                  "version": "1.2.7"
+                }
+              }
+            },
+            "srs": {
+              "version": "0.2.16"
+            },
+            "zipfile": {
+              "version": "0.3.2"
+            },
+            "sqlite3": {
+              "version": "2.1.5"
+            },
+            "mime": {
+              "version": "1.2.7"
+            },
+            "mkdirp": {
+              "version": "0.3.4"
+            }
+          }
+        }
+      }
+    },
+    "windshaft": {
+      "version": "0.6.2",
+      "dependencies": {
+        "express": {
+          "version": "2.5.11",
+          "dependencies": {
+            "connect": {
+              "version": "1.9.2",
+              "dependencies": {
+                "formidable": {
+                  "version": "1.0.11"
+                }
+              }
+            },
+            "mime": {
+              "version": "1.2.4"
+            },
+            "qs": {
+              "version": "0.4.2"
+            },
+            "mkdirp": {
+              "version": "0.3.0"
+            }
+          }
+        },
+        "tilelive": {
+          "version": "4.3.1",
+          "dependencies": {
+            "optimist": {
+              "version": "0.3.5",
+              "dependencies": {
+                "wordwrap": {
+                  "version": "0.0.2"
+                }
+              }
+            },
+            "sphericalmercator": {
+              "version": "1.0.2"
+            }
+          }
+        },
+        "tilelive-mapnik": {
+          "version": "0.3.3-dev",
+          "from": "git://github.com/Vizzuality/tilelive-mapnik.git#7df70554",
+          "dependencies": {
+            "eio": {
+              "version": "0.1.0"
+            },
+            "sphericalmercator": {
+              "version": "1.0.2"
+            }
+          }
+        }
+      }
+    },
+    "step": {
+      "version": "0.0.5"
+    },
+    "generic-pool": {
+      "version": "1.0.12"
+    },
+    "redis": {
+      "version": "0.7.2"
+    },
+    "hiredis": {
+      "version": "0.1.14"
+    },
+    "request": {
+      "version": "2.9.202"
+    },
+    "mapnik": {
+      "version": "0.7.14"
+    },
+    "mocha": {
+      "version": "1.2.1",
+      "dependencies": {
+        "commander": {
+          "version": "0.6.1"
+        },
+        "growl": {
+          "version": "1.5.1"
+        },
+        "jade": {
+          "version": "0.26.3",
+          "dependencies": {
+            "mkdirp": {
+              "version": "0.3.0"
+            }
+          }
+        },
+        "diff": {
+          "version": "1.0.2"
+        },
+        "debug": {
+          "version": "0.7.0"
+        }
+      }
+    }
+  }
+}

package.json

@@ -1,7 +1,7 @@
 {
     "private": true,
     "name": "windshaft-cartodb",
-    "version": "0.1.0",
+    "version": "1.1.0",
     "description": "A map tile server for CartoDB",
     "url": "https://github.com/Vizzuality/Windshaft-cartodb",
     "licenses": [{
@@ -18,17 +18,17 @@
       "email": "simon@vizzuality.com"
     },
     "dependencies": {
-        "connect": "1.8.7",
-        "cluster": "0.6.4",
+        "cluster2": "git://github.com/CartoDB/cluster2.git#cdb_production",
         "node-varnish": "0.1.1",
-        "underscore"  : "1.1.x",
-        "grainstore"  : "~0.6.2",
-        "windshaft"  : "~0.4.14",
+        "underscore"  : "~1.3.3",
+        "grainstore"  : "~0.9.1",
+        "windshaft"  : "~0.6.2",
         "step": "0.0.x",
         "generic-pool": "1.0.x",
         "redis": "0.7.2",
-        "hiredis": "~0.1.12",
-        "request": "2.9.202"
+        "hiredis": "~0.1.14",
+        "request": "2.9.202",
+        "mapnik": "~0.7.14"
     },
     "devDependencies": {
         "mocha": "1.2.1"

run_tests.sh

@@ -27,7 +27,7 @@ echo "port ${REDIS_PORT}" | redis-server - > test.log &
 PID_REDIS=$!
 
 echo "Preparing the database"
-cd test; sh prepare_db.sh >> test.log || die "database preparation failure (see test.log)"; cd -;
+cd test/support; sh prepare_db.sh >> test.log || die "database preparation failure (see test.log)"; cd -;
 
 PATH=node_modules/.bin/:$PATH
 

test/acceptance/cache_validator.js

@@ -1,31 +1,8 @@
 var assert      = require('../support/assert');
-var net = require('net');
 require(__dirname + '/../support/test_helper');
 var CacheValidator = require(__dirname + '/../../lib/cartodb/cache_validator');
-var tests = module.exports = {};
 
-function VarnishEmu(on_cmd_recieved, test_callback) {
-    var self = this;
-    var welcome_msg = 'hi, im a varnish emu, right?';
-
-    self.commands_recieved = [];
-
-    var server = net.createServer(function (socket) {
-      var command = '';
-      socket.write("200 " + welcome_msg.length + "\n");
-      socket.write(welcome_msg);
-      socket.on('data', function(data) {        
-        self.commands_recieved.push(data);
-        on_cmd_recieved && on_cmd_recieved(self.commands_recieved);
-        socket.write('200 0\n');
-      });
-    });
-    server.listen(1337, "127.0.0.1");
-    
-    server.on('listening', function(){
-      test_callback();
-    });    
-}
+var VarnishEmu = require('../support/VarnishEmu');
 
 suite('cache_validator', function() {
 

test/acceptance/server.js

@@ -1,6 +1,7 @@
 var assert      = require('../support/assert');
 var tests       = module.exports = {};
 var _           = require('underscore');
+var redis       = require('redis');
 var querystring = require('querystring');
 require(__dirname + '/../support/test_helper');
 
@@ -11,6 +12,11 @@ server.setMaxListeners(0);
 
 suite('server', function() {
 
+    var redis_client = redis.createClient(global.environment.redis.port);
+    
+    suiteSetup(function(){
+    });
+
     /////////////////////////////////////////////////////////////////////////////////
     //
     // GET UNSUPPORTED
@@ -35,26 +41,33 @@ suite('server', function() {
     
     test("get'ing blank style returns default style", function(done){
         assert.response(server, {
-            headers: {host: 'vizzuality.localhost.lan'},
+            headers: {host: 'localhost'},
             url: '/tiles/my_table/style',
             method: 'GET'
         },{
             status: 200,
-            body: '{"style":"#my_table {marker-fill: #FF6600;marker-opacity: 1;marker-width: 8;marker-line-color: white;marker-line-width: 3;marker-line-opacity: 0.9;marker-placement: point;marker-type: ellipse;marker-allow-overlap: true;}"}'
-        }, function() { done(); });
+            headers: { 'X-Cache-Channel': 'cartodb_test_user_1_db:my_table' },
+        }, function(res) {
+            var parsed = JSON.parse(res.body);
+            assert.equal(parsed.style, "#my_table {marker-fill: #FF6600;marker-opacity: 1;marker-width: 8;marker-line-color: white;marker-line-width: 3;marker-line-opacity: 0.9;marker-placement: point;marker-type: ellipse;marker-allow-overlap: true;}");
+            assert.equal(parsed.version, '2.0.0');
+            done();
+        });
     });
 
     // See https://github.com/Vizzuality/Windshaft-cartodb/issues/43
     test("get'ing style of private table should fail when unauthenticated",
     function(done) {
         assert.response(server, {
-            headers: {host: 'vizzuality.localhost.lan'},
+            headers: {host: 'localhost'},
             url: '/tiles/test_table_private_1/style',
             method: 'GET'
         },{
         }, function(res) {
           // FIXME: should be 401 Unauthorized
           assert.equal(res.statusCode, 500, res.body);
+          assert.deepEqual(JSON.parse(res.body),
+            {error: 'Sorry, you are unauthorized (permission denied)'});
           done();
         });
     });
@@ -62,13 +75,15 @@ suite('server', function() {
     test("get'ing style of private table should succeed when authenticated",
     function(done) {
         assert.response(server, {
-            headers: {host: 'vizzuality.localhost.lan'},
+            headers: {host: 'localhost'},
             url: '/tiles/test_table_private_1/style?map_key=1234',
             method: 'GET'
         },{
         }, function(res) {
           assert.equal(res.statusCode, 200, res.body);
-          assert.deepEqual(res.body, '{"style":"#test_table_private_1 {marker-fill: #FF6600;marker-opacity: 1;marker-width: 8;marker-line-color: white;marker-line-width: 3;marker-line-opacity: 0.9;marker-placement: point;marker-type: ellipse;marker-allow-overlap: true;}"}');
+          var parsed = JSON.parse(res.body);
+          assert.equal(parsed.style, "#test_table_private_1 {marker-fill: #FF6600;marker-opacity: 1;marker-width: 8;marker-line-color: white;marker-line-width: 3;marker-line-opacity: 0.9;marker-placement: point;marker-type: ellipse;marker-allow-overlap: true;}");
+          assert.equal(parsed.version, '2.0.0');
           done();
         });
     });
@@ -81,7 +96,7 @@ suite('server', function() {
     
     test("post'ing no style returns 400 with errors", function(done){
         assert.response(server, {
-            headers: {host: 'vizzuality.localhost.lan'},
+            headers: {host: 'localhost'},
             url: '/tiles/my_table/style',
             method: 'POST'
         },{
@@ -94,11 +109,23 @@ suite('server', function() {
         assert.response(server, {
             url: '/tiles/my_table3/style?map_key=1234',
             method: 'POST',
-            headers: {host: 'vizzuality.localhost.lan', 'Content-Type': 'application/x-www-form-urlencoded' },
+            headers: {host: 'localhost', 'Content-Type': 'application/x-www-form-urlencoded' },
             data: querystring.stringify({style: '#my_table3{backgxxxxxround-color:#fff;}'})
         },{
             status: 500, // FIXME: should be 400 !
-            body: JSON.stringify(['style.mss:1:11 Unrecognized rule: backgxxxxxround-color'])
+            body: /Unrecognized rule: backgxxxxxround-color/
+        }, function() { done(); });
+    });
+
+    test("post'ing unparseable style returns 400 with error", function(done){
+        assert.response(server, {
+            url: '/tiles/my_table3/style?map_key=1234',
+            method: 'POST',
+            headers: {host: 'localhost', 'Content-Type': 'application/x-www-form-urlencoded' },
+            data: querystring.stringify({style: '#my_table3{'})
+        },{
+            status: 500, // FIXME: should be 400 !
+            body: /Missing closing/
         }, function() { done(); });
     });
     
@@ -106,19 +133,24 @@ suite('server', function() {
         assert.response(server, {
             url: '/tiles/my_table4/style?map_key=1234',
             method: 'POST',
-            headers: {host: 'vizzuality.localhost.lan', 'Content-Type': 'application/x-www-form-urlencoded' },
+            headers: {host: 'localhost', 'Content-Type': 'application/x-www-form-urlencoded' },
             data: querystring.stringify({style: '#my_table4{backgxxxxxround-color:#fff;foo:bar}'})
         },{
             status: 500, // FIXME: should be 400 !
-            body: JSON.stringify([ 'style.mss:1:11 Unrecognized rule: backgxxxxxround-color', 'style.mss:1:38 Unrecognized rule: foo' ])
-        }, function() { done(); });
+        }, function(res) {
+          var parsed = JSON.parse(res.body);
+          assert.equal(parsed.length, 2);
+          assert.ok( RegExp(/Unrecognized rule: backgxxxxxround-color/).test(parsed[0]) );
+          assert.ok( RegExp(/Unrecognized rule: foo/).test(parsed[1]) );
+          done();
+        });
     });
 
     test("post'ing good style returns 200", function(done){
         assert.response(server, {
             url: '/tiles/my_table5/style?map_key=1234',
             method: 'POST',
-            headers: {host: 'vizzuality.localhost.lan', 'Content-Type': 'application/x-www-form-urlencoded' },
+            headers: {host: 'localhost', 'Content-Type': 'application/x-www-form-urlencoded' },
             data: querystring.stringify({style: 'Map {background-color:#fff;}'})
         },{
         }, function(res) {
@@ -126,12 +158,26 @@ suite('server', function() {
             done();
         });
     });
-    
+
+    // See https://github.com/Vizzuality/Windshaft-cartodb/issues/38
+    test("post'ing good style with auth passed as api_key returns 200", function(done){
+        assert.response(server, {
+            url: '/tiles/my_table5/style?api_key=1234',
+            method: 'POST',
+            headers: {host: 'localhost', 'Content-Type': 'application/x-www-form-urlencoded' },
+            data: querystring.stringify({style: 'Map {background-color:#fff;}'})
+        },{}, function(res) {
+            assert.equal(res.statusCode, 200, res.body);
+            done();
+        });
+    });
+
+    // See https://github.com/Vizzuality/cartodb-management/issues/155
     test("post'ing good style with no authentication returns an error", function(done){
         assert.response(server, {
             url: '/tiles/my_table5/style?map_key=1234',
             method: 'POST',
-            headers: {host: 'vizzuality.localhost.lan', 'Content-Type': 'application/x-www-form-urlencoded' },
+            headers: {host: 'localhost', 'Content-Type': 'application/x-www-form-urlencoded' },
             data: querystring.stringify({style: 'Map {background-color:#fff;}'})
         },{
         }, function(res) {
@@ -139,7 +185,7 @@ suite('server', function() {
           assert.response(server, {
               url: '/tiles/my_table5/style',
               method: 'POST',
-              headers: {host: 'vizzuality.localhost.lan', 'Content-Type': 'application/x-www-form-urlencoded' },
+              headers: {host: 'localhost', 'Content-Type': 'application/x-www-form-urlencoded' },
               data: querystring.stringify({style: 'Map {background-color:#aaa;}'})
           },{}, function(res) {
             // FIXME: should be 401 Unauthorized
@@ -147,13 +193,17 @@ suite('server', function() {
             assert.ok(res.body.indexOf('map state cannot be changed by unauthenticated request') != -1, res.body);
 
             assert.response(server, {
-                headers: {host: 'vizzuality.localhost.lan'},
+                headers: {host: 'localhost'},
                 url: '/tiles/my_table5/style',
                 method: 'GET'
             },{
                 status: 200,
-                body: JSON.stringify({style: 'Map {background-color:#fff;}'})
-            }, function() { done(); });
+            }, function(res) {
+              var parsed = JSON.parse(res.body);
+              assert.equal(parsed.style, 'Map {background-color:#fff;}');
+              //assert.equal(parsed.version, '2.0.0');
+              done();
+            });
 
           });
         });
@@ -164,7 +214,7 @@ suite('server', function() {
         assert.response(server, {
             url: '/tiles/my_table5/style?map_key=1234',
             method: 'POST',
-            headers: {host: 'vizzuality.localhost.lan', 'Content-Type': 'application/x-www-form-urlencoded' },
+            headers: {host: 'localhost', 'Content-Type': 'application/x-www-form-urlencoded' },
             data: querystring.stringify({style: style})
         },{
         }, function(res) { 
@@ -172,13 +222,17 @@ suite('server', function() {
             assert.equal(res.statusCode, 200, res.body);
 
             assert.response(server, {
-                headers: {host: 'vizzuality.localhost.lan'},
+                headers: {host: 'localhost'},
                 url: '/tiles/my_table5/style',
                 method: 'GET'
             },{
                 status: 200,
-                body: JSON.stringify({style: style})
-            }, function() { done(); });
+            }, function(res) {
+              var parsed = JSON.parse(res.body);
+              assert.equal(parsed.style, style);
+              //assert.equal(parsed.version, '2.0.0');
+              done();
+            });
 
         });
     
@@ -196,20 +250,24 @@ suite('server', function() {
         assert.response(server, {
             url: '/tiles/my_table5/style',
             method: 'DELETE',
-            headers: {host: 'vizzuality'},
+            headers: {host: 'localhost'},
         },{}, function(res) { 
           // FIXME: should be 401 Unauthorized
           assert.equal(res.statusCode, 500, res.body);
           assert.ok(res.body.indexOf('map state cannot be changed by unauthenticated request') != -1, res.body);
           // check that the style wasn't really deleted !
           assert.response(server, {
-              headers: {host: 'vizzuality'},
+              headers: {host: 'localhost'},
               url: '/tiles/my_table5/style?map_key=1234',
               method: 'GET'
           },{
               status: 200,
-              body: JSON.stringify({style: 'Map {background-color:#fff;}'})
-          }, function() { done(); });
+          }, function(res) {
+              var parsed = JSON.parse(res.body);
+              assert.equal(parsed.style, 'Map {background-color:#fff;}');
+              //assert.equal(parsed.version, '2.0.0');
+              done();
+          });
         });
     });
 
@@ -219,13 +277,13 @@ suite('server', function() {
         assert.response(server, {
             url: '/tiles/my_table5/style?map_key=1234',
             method: 'DELETE',
-            headers: {host: 'vizzuality'},
+            headers: {host: 'localhost'},
         },{}, function(res) { 
         assert.equal(res.statusCode, 200, res.body);
 
             // Retrive style with authenticated request
             assert.response(server, {
-                headers: {host: 'vizzuality'},
+                headers: {host: 'localhost'},
                 url: '/tiles/my_table5/style?map_key=1234',
                 method: 'GET'
             },{}, function(res) {
@@ -234,7 +292,7 @@ suite('server', function() {
 
               // Now retrive style with unauthenticated request
               assert.response(server, {
-                  headers: {host: 'vizzuality'},
+                  headers: {host: 'localhost'},
                   url: '/tiles/my_table5/style',
                   method: 'GET'
               }, {}, function(res) {
@@ -248,6 +306,18 @@ suite('server', function() {
         });
     });
 
+    // See https://github.com/Vizzuality/Windshaft-cartodb/issues/38
+    test("delete'ing style with api_key is accepted", function(done){
+        assert.response(server, {
+            url: '/tiles/my_table5/style?api_key=1234',
+            method: 'DELETE',
+            headers: {host: 'localhost'},
+        },{}, function(res) { 
+          assert.equal(res.statusCode, 200, res.body);
+          done();
+        });
+    });
+
     /////////////////////////////////////////////////////////////////////////////////
     //
     // GET INFOWINDOW
@@ -256,18 +326,19 @@ suite('server', function() {
     
     test("get'ing blank infowindow returns blank", function(done){
         assert.response(server, {
-            headers: {host: 'vizzuality.localhost.lan'},
+            headers: {host: 'localhost'},
             url: '/tiles/my_tablez/infowindow',
             method: 'GET'
         },{
             status: 200,
+            headers: { 'X-Cache-Channel': 'cartodb_test_user_1_db:my_tablez' },
             body: '{"infowindow":null}'
         }, function() { done(); });
     });
     
     test("get'ing blank infowindow with callback returns blank with callback", function(done){
         assert.response(server, {
-            headers: {host: 'vizzuality.localhost.lan'},
+            headers: {host: 'localhost'},
             url: '/tiles/my_tablez/infowindow?callback=simon',
             method: 'GET'
         },{
@@ -279,7 +350,7 @@ suite('server', function() {
     
     test("get'ing completed infowindow with callback returns information with callback", function(done){
         assert.response(server, {
-            headers: {host: 'vizzuality.localhost.lan'},
+            headers: {host: 'localhost'},
             url: '/tiles/my_table/infowindow?callback=simon',
             method: 'GET'
         },{
@@ -291,7 +362,7 @@ suite('server', function() {
     test("get'ing infowindow of private table should fail when unauthenticated",
     function(done) {
         assert.response(server, {
-            headers: {host: 'vizzuality.localhost.lan'},
+            headers: {host: 'localhost'},
             url: '/tiles/test_table_private_1/infowindow',
             method: 'GET'
         },{}, function(res) {
@@ -304,7 +375,7 @@ suite('server', function() {
     test("get'ing infowindow of private table should succeed when authenticated",
     function(done) {
         assert.response(server, {
-            headers: {host: 'vizzuality.localhost.lan'},
+            headers: {host: 'localhost'},
             url: '/tiles/test_table_private_1/infowindow?map_key=1234',
             method: 'GET'
         },{}, function(res) {
@@ -321,18 +392,19 @@ suite('server', function() {
 
     test("get'ing a json with default style should return an grid", function(done){
         assert.response(server, {
-            headers: {host: 'vizzuality.localhost.lan'},
+            headers: {host: 'localhost'},
             url: '/tiles/gadm4/6/31/24.grid.json',
             method: 'GET'
         },{
             status: 200,
-            headers: { 'Content-Type': 'text/javascript; charset=utf-8; charset=utf-8' }
+            headers: { 'Content-Type': 'text/javascript; charset=utf-8; charset=utf-8',
+                       'X-Cache-Channel': 'cartodb_test_user_1_db:gadm4' }
         }, function() { done(); });
     });
     
     test("get'ing a json with default style should return an grid", function(done){
         assert.response(server, {
-            headers: {host: 'vizzuality.localhost.lan'},
+            headers: {host: 'localhost'},
             url: '/tiles/gadm4/6/31/24.grid.json',
             method: 'GET'
         },{
@@ -344,7 +416,7 @@ suite('server', function() {
     test("get'ing a json with default style and sql should return a constrained grid", function(done){
         var sql = querystring.stringify({sql: "SELECT * FROM gadm4 WHERE codineprov = '08'"})
         assert.response(server, {
-            headers: {host: 'vizzuality.localhost.lan'},
+            headers: {host: 'localhost'},
             url: '/tiles/gadm4/6/31/24.grid.json?' + sql,
             method: 'GET'
         },{
@@ -356,7 +428,7 @@ suite('server', function() {
     test("get'ing the grid of a private table should fail when unauthenticated",
     function(done) {
         assert.response(server, {
-            headers: {host: 'vizzuality.localhost.lan'},
+            headers: {host: 'localhost'},
             url: '/tiles/test_table_private_1/6/31/24.grid.json',
             method: 'GET'
         },{}, function(res) {
@@ -369,7 +441,7 @@ suite('server', function() {
     test("get'ing the grid of a private table should succeed when authenticated",
     function(done) {
         assert.response(server, {
-            headers: {host: 'vizzuality.localhost.lan'},
+            headers: {host: 'localhost'},
             url: '/tiles/test_table_private_1/6/31/24.grid.json?map_key=1234',
             method: 'GET'
         },{}, function(res) {
@@ -386,19 +458,19 @@ suite('server', function() {
     
     test("get'ing a tile with default style should return an image", function(done){
         assert.response(server, {
-            headers: {host: 'vizzuality.localhost.lan'},
+            headers: {host: 'localhost'},
             url: '/tiles/gadm4/6/31/24.png?geom_type=polygon',
             method: 'GET'
         },{
             status: 200,
-            headers: { 'Content-Type': 'image/png' }
+            headers: { 'Content-Type': 'image/png', 'X-Cache-Channel': 'cartodb_test_user_1_db:gadm4' }
         }, function() { done(); });
     });
     
     test("get'ing a tile with default style and sql should return a constrained image", function(done){
         var sql = querystring.stringify({sql: "SELECT * FROM gadm4 WHERE codineprov = '08'"});
         assert.response(server, {
-            headers: {host: 'vizzuality.localhost.lan'},
+            headers: {host: 'localhost'},
             url: '/tiles/gadm4/6/31/24.png?' + sql,
             method: 'GET'
         },{
@@ -411,7 +483,7 @@ suite('server', function() {
     test("get'ing a tile with default style and complex sql should return a constrained image", function(done){
         var sql = querystring.stringify({sql: "SELECT * FROM gadm4 WHERE  codineprov = '08' AND codccaa > 60"})
         assert.response(server, {
-            headers: {host: 'vizzuality.localhost.lan'},
+            headers: {host: 'localhost'},
             url: '/tiles/gadm4/6/31/24.png?' + sql,
             method: 'GET'
         },{
@@ -424,7 +496,7 @@ suite('server', function() {
         // NOTE: may fail if grainstore < 0.3.0 is used by Windshaft
         var sql = querystring.stringify({sql: "SELECT * FROM test_table_private_1", map_key: 1234})
         assert.response(server, {
-            headers: {host: 'vizzuality.localhost.lan'},
+            headers: {host: 'localhost'},
             // NOTE: we encode a public table in the URL !
             url: '/tiles/gadm4/6/31/24.png?' + sql,
             method: 'GET'
@@ -434,13 +506,27 @@ suite('server', function() {
         }, function() { done(); });
     });
 
+    // See https://github.com/Vizzuality/Windshaft-cartodb/issues/38
+    test("get'ing a tile with data from private table should succeed when authenticated with api_key", function(done){
+        // NOTE: may fail if grainstore < 0.3.0 is used by Windshaft
+        var sql = querystring.stringify({sql: "SELECT * FROM test_table_private_1", api_key: 1234})
+        assert.response(server, {
+            headers: {host: 'localhost'},
+            url: '/tiles/gadm4/6/31/24.png?' + sql,
+            method: 'GET'
+        },{
+            status: 200,
+            headers: { 'Content-Type': 'image/png' }
+        }, function() { done(); });
+    });
+
     test("get'ing a tile with data from private table should fail when unauthenticated", function(done){
         var sql = querystring.stringify({
           sql: "SELECT * FROM test_table_private_1",
           cache_buster:2 // this is to avoid getting the cached response
         });
         assert.response(server, {
-            headers: {host: 'vizzuality.localhost.lan'},
+            headers: {host: 'localhost'},
             url: '/tiles/gadm4/6/31/24.png?' + sql,
             method: 'GET'
         },{
@@ -455,12 +541,12 @@ suite('server', function() {
         var sql = querystring.stringify({
           sql: "SELECT * FROM test_table_private_1",
           cache_buster:3,
-          // 1235 is written in rails:users:vizzuality:map_key SET
+          // 1235 is written in rails:users:localhost:map_key SET
           // See https://github.com/Vizzuality/Windshaft-cartodb/issues/39
           map_key: 1235
         });
         assert.response(server, {
-            headers: {host: 'vizzuality.localhost.lan'},
+            headers: {host: 'localhost'},
             url: '/tiles/gadm4/6/31/24.png?' + sql,
             method: 'GET'
         },{
@@ -471,5 +557,133 @@ suite('server', function() {
         });
     });
 
+    /////////////////////////////////////////////////////////////////////////////////
+    //
+    // DELETE CACHE 
+    //
+    /////////////////////////////////////////////////////////////////////////////////
+
+    test("forbids flushing cache without specifying table name", function(done) {
+        assert.response(server, {
+            headers: {host: 'localhost'},
+            url: '/tiles/flush_cache',
+            method: 'DELETE'
+        },{}, function(res) {
+          assert.equal(res.statusCode, 404, res.statusCode + ': ' + res.body);
+          done();
+        });
+    });
+
+    test("allows flushing table cache by unauthenticated user", function(done) {
+        assert.response(server, {
+            headers: {host: 'localhost'},
+            url: '/tiles/gadm4/flush_cache',
+            method: 'DELETE'
+        },{}, function(res) {
+          assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+          // TODO: also check that varnish is signalled (using VarnishEmu)
+          // NOTE: requires enable_cache=1 in test.js
+          done();
+        });
+    });
+
+    /////////////////////////////////////////////////////////////////////////////////
+    //
+    // GET METADATA
+    //
+    /////////////////////////////////////////////////////////////////////////////////
+
+    test("does not provide metadata of private table to unauthenticated requests", function(done){
+        assert.response(server, {
+            headers: {host: 'localhost'},
+            url: '/tiles/test_table_private_1/map_metadata',
+            method: 'GET'
+        },{}, function(res) {
+          // FIXME: should be 401 instead
+          assert.equal(res.statusCode, 500, res.statusCode + ': ' + res.body);
+          done();
+        });
+    });
+
+    test("does provide metadata of private table to authenticated requests", function(done){
+        assert.response(server, {
+            headers: {host: 'localhost'},
+            url: '/tiles/test_table_private_1/map_metadata?map_key=1234',
+            method: 'GET'
+        },{}, function(res) {
+          assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+          done();
+        });
+    });
+
+    test("does provide metadata of public table to unauthenticated requests", function(done){
+        assert.response(server, {
+            headers: {host: 'localhost'},
+            url: '/tiles/gadm4/map_metadata',
+            method: 'GET'
+        },{}, function(res) {
+          assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+          // TODO: show metadata ?
+          done();
+        });
+    });
+
+    test("does provide metadata of public table to authenticated requests", function(done){
+        assert.response(server, {
+            headers: {host: 'localhost'},
+            url: '/tiles/gadm4/map_metadata?map_key=1234',
+            method: 'GET'
+        },{}, function(res) {
+          assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+          // TODO: show metadata ?
+          done();
+        });
+    });
+
+    /////////////////////////////////////////////////////////////////////////////////
+    //
+    // DELETE CACHE 
+    //
+    /////////////////////////////////////////////////////////////////////////////////
+
+    test("forbids flushing cache without specifying table name", function(done) {
+        assert.response(server, {
+            headers: {host: 'localhost'},
+            url: '/tiles/flush_cache',
+            method: 'DELETE'
+        },{}, function(res) {
+          assert.equal(res.statusCode, 404, res.statusCode + ': ' + res.body);
+          done();
+        });
+    });
+
+    test("allows flushing table cache by unauthenticated user", function(done) {
+        assert.response(server, {
+            headers: {host: 'localhost'},
+            url: '/tiles/gadm4/flush_cache',
+            method: 'DELETE'
+        },{}, function(res) {
+          assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+          // TODO: also check that varnish is signalled (using VarnishEmu)
+          // NOTE: requires enable_cache=1 in test.js
+          done();
+        });
+    });
+
+    /////////////////////////////////////////////////////////////////////////////////
+    //
+    // Tear down
+    //
+    /////////////////////////////////////////////////////////////////////////////////
+
+    suiteTeardown(function(done) {
+        // This test will add map_style records, like
+        // 'map_style|null|publicuser|my_table',
+        redis_client.keys("map_style|*", function(err, matches) {
+            _.each(matches, function(k) { redis_client.del(k); });
+            done();
+        });
+    });
+    
 });
 

test/support/VarnishEmu.js

@@ -0,0 +1,25 @@
+var net = require('net');
+
+module.exports = function(on_cmd_recieved, test_callback) {
+    var self = this;
+    var welcome_msg = 'hi, im a varnish emu, right?';
+
+    self.commands_recieved = [];
+
+    var server = net.createServer(function (socket) {
+      var command = '';
+      socket.write("200 " + welcome_msg.length + "\n");
+      socket.write(welcome_msg);
+      socket.on('data', function(data) {
+        self.commands_recieved.push(data);
+        on_cmd_recieved && on_cmd_recieved(self.commands_recieved);
+        socket.write('200 0\n');
+      });
+    });
+    server.listen(1337, "127.0.0.1");
+
+    server.on('listening', function(){
+      test_callback();
+    });
+};
+

test/support/prepare_db.sh

@@ -26,10 +26,10 @@ psql "${TEST_DB}" < ./sql/windshaft.test.sql
 psql "${TEST_DB}" < ./sql/gadm4.sql
 
 echo "preparing redis..."
-echo "HSET rails:users:vizzuality id 1" | redis-cli -p ${REDIS_PORT} -n 5
-echo 'HSET rails:users:vizzuality database_name "'"${TEST_DB}"'"' | redis-cli -p ${REDIS_PORT} -n 5
-echo "HSET rails:users:vizzuality map_key 1234" | redis-cli -p ${REDIS_PORT} -n 5
-echo "SADD rails:users:vizzuality:map_key 1235" | redis-cli -p ${REDIS_PORT} -n 5
+echo "HSET rails:users:localhost id 1" | redis-cli -p ${REDIS_PORT} -n 5
+echo 'HSET rails:users:localhost database_name "'"${TEST_DB}"'"' | redis-cli -p ${REDIS_PORT} -n 5
+echo "HSET rails:users:localhost map_key 1234" | redis-cli -p ${REDIS_PORT} -n 5
+echo "SADD rails:users:localhost:map_key 1235" | redis-cli -p ${REDIS_PORT} -n 5
 echo 'HSET rails:'"${TEST_DB}"':my_table infowindow "this, that, the other"' | redis-cli -p ${REDIS_PORT} -n 0
 echo 'HSET rails:'"${TEST_DB}"':test_table_private_1 privacy "0"' | redis-cli -p ${REDIS_PORT} -n 0
 

test/unit/cartodb/req2params.test.js

@@ -14,27 +14,27 @@ suite('req2params', function() {
     });
 
     test('cleans up request', function(done){
-      opts.req2params({headers: { host:'h1' }, query: {dbuser:'hacker',dbname:'secret'}}, function(err, req) {
+      opts.req2params({headers: { host:'localhost' }, query: {dbuser:'hacker',dbname:'secret'}}, function(err, req) {
           if ( err ) { console.log(err); throw new Error(err); }
           assert.ok(_.isObject(req.query), 'request has query');
           assert.ok(!req.query.hasOwnProperty('dbuser'), 'dbuser was removed from query');
           assert.ok(req.hasOwnProperty('params'), 'request has params');
           assert.ok(req.params.hasOwnProperty('interactivity'), 'request params have interactivity');
-          assert.ok(_.isNull(req.params.dbname), 'could forge dbname');
+          assert.equal(req.params.dbname, 'cartodb_test_user_1_db', 'could forge dbname: '+ req.params.dbname);
           assert.ok(!req.hasOwnProperty('dbuser'), 'could inject dbuser ('+req.params.dbuser+')');
           done();
       });
     });
 
     test('sets dbname from redis metadata', function(done){
-      opts.req2params({headers: { host:'vizzuality' }, query: {} }, function(err, req) {
+      opts.req2params({headers: { host:'localhost' }, query: {} }, function(err, req) {
           if ( err ) { console.log(err); throw new Error(err); }
           //console.dir(req);
           assert.ok(_.isObject(req.query), 'request has query');
           assert.ok(!req.query.hasOwnProperty('dbuser'), 'dbuser was removed from query');
           assert.ok(req.hasOwnProperty('params'), 'request has params');
           assert.ok(req.params.hasOwnProperty('interactivity'), 'request params have interactivity');
-          // database_name for user "vizzuality" (see test/support/prepare_db.sh)
+          // database_name for user "localhost" (see test/support/prepare_db.sh)
           assert.equal(req.params.dbname, 'cartodb_test_user_1_db');
           // unauthenticated request gets no dbuser
           assert.ok(!req.params.hasOwnProperty('dbuser'), 'could inject dbuser ('+req.params.dbuser+')');
@@ -43,19 +43,19 @@ suite('req2params', function() {
     });
 
     test('sets also dbuser for authenticated requests', function(done){
-      opts.req2params({headers: { host:'vizzuality' }, query: {map_key: '1234'} }, function(err, req) {
+      opts.req2params({headers: { host:'localhost' }, query: {map_key: '1234'} }, function(err, req) {
           if ( err ) { console.log(err); throw new Error(err); }
           //console.dir(req);
           assert.ok(_.isObject(req.query), 'request has query');
           assert.ok(!req.query.hasOwnProperty('dbuser'), 'dbuser was removed from query');
           assert.ok(req.hasOwnProperty('params'), 'request has params');
           assert.ok(req.params.hasOwnProperty('interactivity'), 'request params have interactivity');
-          // database_name for user "vizzuality" (see test/support/prepare_db.sh)
+          // database_name for user "localhost" (see test/support/prepare_db.sh)
           assert.equal(req.params.dbname, 'cartodb_test_user_1_db');
-          // id for user "vizzuality" (see test/support/prepare_db.sh)
+          // id for user "localhost" (see test/support/prepare_db.sh)
           assert.equal(req.dbuser, 'test_cartodb_user_1');
  
-          opts.req2params({headers: { host:'vizzuality' }, query: {map_key: '1235'} }, function(err, req) {
+          opts.req2params({headers: { host:'localhost' }, query: {map_key: '1235'} }, function(err, req) {
               // wrong key resets params to no user
               assert.ok(!req.hasOwnProperty('dbuser'), 'could inject dbuser ('+req.params.dbuser+')');
               done();

tools/reset_styles

@@ -1,35 +1,85 @@
 #!/usr/bin/env node
 
-// Reset redis-stored XML styles so that they are regenerated
-// from CartoCSS on next tile request
+var path = require('path');
 
-var redis = require('redis')
+// Reset all styles in the store
+var grainstore = require('../node_modules/grainstore/lib/grainstore');
+var mapnik = require('mapnik');
+var redis = require('redis');
 
-var REDIS_PORT = 6379; // TODO: make a parameter
+function usage(me, exitcode) {
+  console.log("Usage: " + me + " [--convert] [<target_mapnik_version>]");
+  process.exit(exitcode);
+}
+
+var doConvert = false;
+var MAPNIK_VERSION;
+
+var node_path = process.argv.shift(); 
+var script_path = process.argv.shift(); 
+var me = path.basename(script_path);
+var arg;
+while ( arg = process.argv.shift() ) {
+  if ( arg == '--convert' ) {
+    doConvert = true;
+  } else if ( ! MAPNIK_VERSION ) {
+    MAPNIK_VERSION = arg;
+  }
+  else {
+    usage(me, 1);
+  }
+}
+
+if ( ! MAPNIK_VERSION ) {
+  MAPNIK_VERSION = mapnik.versions.mapnik;
+}
+
+console.log( (doConvert ? "Converting" : "Resetting" ) + ' all styles to target ' + MAPNIK_VERSION);
+
+var REDIS_PORT = 6379; // TODO: make a command line parameter
 
 var dbnum = 0;
 
+var mml_store = new grainstore.MMLStore({port:REDIS_PORT}, {mapnik_version:MAPNIK_VERSION});
+
+var failures = [];
+
 var client = redis.createClient(REDIS_PORT, 'localhost');
 client.on('connect', function() {
   client.select(dbnum);
   client.keys('map_style|*', function(err, matches) {
 
     processNext = function() {
-      if ( ! matches.length ) process.exit(0);
+      if ( ! matches.length ) process.exit(failures.length);
       var k = matches.shift();
-      console.log("Resetting XML in key: " + k);
-      client.get(k, function(err, val) {
-        if ( err ) throw err;
-        val = JSON.parse(val);
-        delete val.xml;
-        client.set(k, JSON.stringify(val), function() {
-          console.log("done with style " + k);
-          processNext();
-        });
+
+      if ( /map_style\|.*\|.*\|/.test(k) ) {
+        //console.warn("Key " + k + " is EXTENDED, skipping");
+        processNext();
+      } 
+
+      var params = RegExp(/map_style\|(.*)\|(.*)/).exec(k);
+      var db = params[1];
+      var tab = params[2];
+      var out = 'map_style|' + db + '|' + tab + ': ';
+
+      var mml_builder = mml_store.mml_builder({dbname:db, table:tab},
+                                              function(err, payload) {
+
+          if ( err ) { console.warn(out + err.message); failures.push(k); processNext(); }
+          else {
+            mml_builder.resetStyle(function(err, data) {
+              if ( err ) { console.warn(out + err.message); failures.push(k); }
+              else console.log(out + 'OK' + ( doConvert ? ' (converted)' : '' ));
+              processNext();
+            }, doConvert);
+          }
       });
-    }
+
+    };
 
     processNext();
 
   });
+
 });