updated NEW for v1.1.0

Ensures all errors are logged

NEWS.md

@@ -1,11 +1,20 @@
-1.1.0 (DD/MM/YY)
+1.1.0 (30/10/12)
 -----
+* Add /version entry point
+* CartoCSS versioning
+ * Include version in GET /style response
+ * Support version and convert parameters in POST /style request
+ * Autodetect target mapnik version and let config override it
+ * Add tools/reset_styles script to batch-reset (and optionally convert) styles
 * Configurable logging format (#4)
 * Detailed error on missing user metadata 
 * Properly handle unauthenticated requests for metadata
 * Accept "api_key" in addition to "map_key",
   both in query_string and POST body (#38)
 * Add ./configure script
+* Allow listening on host IP
+* Replaced environment configs by .example ones
+* Fixed some issues with cluster2
 
 1.0.0 (03/10/12)
 -----

app.js

@@ -33,5 +33,7 @@ var Windshaft = require('windshaft');
 var serverOptions = require('./lib/cartodb/server_options');
 
 ws = CartodbWindshaft(serverOptions);
-ws.listen(global.environment.port);
-console.log("Windshaft tileserver started on port " + global.environment.port);
+ws.listen(global.environment.port, global.environment.host);
+ws.on('listening', function() {
+  console.log("Windshaft tileserver started on " + global.environment.host + ':' + global.environment.port);
+});

cluster.js

@@ -41,12 +41,15 @@ var ws = CartodbWindshaft(serverOptions);
 //.use(cluster.pidfiles('pids'))
 var cluster = new Cluster({
   port: global.environment.port,
+  host: global.environment.host,
   monPort: global.environment.port+1,
-  noWorkers: 1 // .set('workers', 1)
+  monHost: global.environment.host,
+  noWorkers: 1 
 });
 
 cluster.listen(function(cb) {
   cb(ws);
+}, function() {
+  console.log("Windshaft tileserver started on port " + global.environment.port);
 });
 
-console.log("Windshaft tileserver started on port " + global.environment.port);

config/environments/development.js.example

@@ -19,6 +19,7 @@ var config = {
         */
         simplify: true
     }
+    ,mapnik_version: undefined
     ,millstone: {
         cache_basedir: '/tmp/cdb-tiler-dev/millstone-dev'
     }

config/environments/production.js.example

@@ -13,6 +13,7 @@ var config = {
         extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188",
         simplify: true
     }
+    ,mapnik_version: undefined
     ,millstone: {
         cache_basedir: '/home/ubuntu/tile_assets/'
     }

config/environments/staging.js.example

@@ -13,6 +13,7 @@ var config = {
         extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188",
         simplify: true
     }
+    ,mapnik_version: undefined
     ,millstone: {
         cache_basedir: '/home/ubuntu/tile_assets/'
     }

config/environments/test.js.example

@@ -14,6 +14,7 @@ var config = {
         extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188",
         simplify: true
     }
+    ,mapnik_version: '2.0.2'
     ,millstone: {
         cache_basedir: '/tmp/cdb-tiler-test/millstone'
     }

lib/cartodb/cartodb_windshaft.js

@@ -37,7 +37,8 @@ var CartodbWindshaft = function(serverOptions) {
             },
             function(err, data){
                 if (err){
-                    res.send({error: err.message}, 500);
+                    ws.sendError(res, {error: err.message}, 500, 'GET INFOWINDOW');
+                    //res.send({error: err.message}, 500);
                 } else {
                     res.send({infowindow: data}, 200);
                 }
@@ -57,7 +58,8 @@ var CartodbWindshaft = function(serverOptions) {
             },
             function(err, data){
                 if (err){
-                    res.send(err.message, 500);
+                    ws.sendError(res, {error: err.message}, 500, 'GET MAP_METADATA');
+                    //res.send(err.message, 500);
                 } else {
                     res.send({map_metadata: data}, 200);
                 }
@@ -77,7 +79,8 @@ var CartodbWindshaft = function(serverOptions) {
             },
             function(err, data){
                 if (err){
-                    res.send(500);
+                    ws.sendError(res, {error: err.message}, 500, 'DELETE CACHE');
+                    //res.send(500);
                 } else {
                     res.send({status: 'ok'}, 200);
                 }

lib/cartodb/server_options.js

@@ -1,14 +1,17 @@
 var   _          = require('underscore')
     , Step       = require('step')
     , cartoData  = require('./carto_data')
-    , Cache = require('./cache_validator');
+    , Cache = require('./cache_validator')
+    , mapnik     = require('mapnik')
+  ;
 
 module.exports = function(){
     var me = {
         base_url: '/tiles/:table',
         grainstore: {
           datasource: global.environment.postgres,
-          cachedir: global.environment.millstone.cache_basedir
+          cachedir: global.environment.millstone.cache_basedir,
+          mapnik_version: global.environment.mapnik_version || mapnik.versions.mapnik
         },
         redis: global.environment.redis,
         enable_cors: global.environment.enable_cors,
@@ -18,6 +21,14 @@ module.exports = function(){
         log_format: global.environment.log_format
     };
 
+    // Be nice and warn if configured mapnik version
+    // is != instaled mapnik version
+    if ( mapnik.versions.mapnik != me.grainstore.mapnik_version ) {
+      console.warn("WARNING: detected mapnik version ("
+        + mapnik.versions.mapnik + ") != configured mapnik version ("
+        + me.grainstore.mapnik_version + ")");
+    }
+
     // Set the cache chanel info to invalidate the cache on the frontend server
     //
     // @param req The request object.

npm-shrinkwrap.json

@@ -1,9 +1,10 @@
 {
   "name": "windshaft-cartodb",
-  "version": "0.2.0-dev",
+  "version": "1.1.0",
   "dependencies": {
     "cluster2": {
-      "version": "0.3.5",
+      "version": "0.3.5-cdb02",
+      "from": "git://github.com/CartoDB/cluster2.git#cdb_production",
       "dependencies": {
         "express": {
           "version": "2.5.11",
@@ -176,20 +177,16 @@
       "version": "0.1.1"
     },
     "underscore": {
-      "version": "1.1.7"
+      "version": "1.3.3"
     },
     "grainstore": {
-      "version": "0.6.4",
+      "version": "0.9.1",
       "dependencies": {
         "carto": {
-          "version": "0.8.2-cdb-dev-3",
-          "from": "git://github.com/CartoDB/carto.git#cdb-0.8",
+          "version": "0.9.2",
           "dependencies": {
-            "underscore": {
-              "version": "1.3.3"
-            },
             "mapnik-reference": {
-              "version": "4.0.5"
+              "version": "5.0.0"
             },
             "xml2js": {
               "version": "0.1.14",
@@ -202,12 +199,8 @@
           }
         },
         "millstone": {
-          "version": "0.5.10-cdb-01",
-          "from": "git://github.com/CartoDB/millstone.git#cdb-node04-devel",
+          "version": "0.5.10",
           "dependencies": {
-            "underscore": {
-              "version": "1.3.3"
-            },
             "request": {
               "version": "2.11.4",
               "dependencies": {
@@ -252,11 +245,8 @@
       }
     },
     "windshaft": {
-      "version": "0.5.8",
+      "version": "0.6.2",
       "dependencies": {
-        "underscore": {
-          "version": "1.3.3"
-        },
         "express": {
           "version": "2.5.11",
           "dependencies": {
@@ -283,7 +273,7 @@
           "version": "4.3.1",
           "dependencies": {
             "optimist": {
-              "version": "0.3.4",
+              "version": "0.3.5",
               "dependencies": {
                 "wordwrap": {
                   "version": "0.0.2"
@@ -299,9 +289,6 @@
           "version": "0.3.3-dev",
           "from": "git://github.com/Vizzuality/tilelive-mapnik.git#7df70554",
           "dependencies": {
-            "mapnik": {
-              "version": "0.7.14"
-            },
             "eio": {
               "version": "0.1.0"
             },
@@ -327,6 +314,9 @@
     "request": {
       "version": "2.9.202"
     },
+    "mapnik": {
+      "version": "0.7.14"
+    },
     "mocha": {
       "version": "1.2.1",
       "dependencies": {

package.json

@@ -1,7 +1,7 @@
 {
     "private": true,
     "name": "windshaft-cartodb",
-    "version": "1.0.1",
+    "version": "1.1.0",
     "description": "A map tile server for CartoDB",
     "url": "https://github.com/Vizzuality/Windshaft-cartodb",
     "licenses": [{
@@ -18,16 +18,17 @@
       "email": "simon@vizzuality.com"
     },
     "dependencies": {
-        "cluster2": "git://github.com/CartoDB/cluster2.git#28cde11",
+        "cluster2": "git://github.com/CartoDB/cluster2.git#cdb_production",
         "node-varnish": "0.1.1",
-        "underscore"  : "1.1.x",
-        "grainstore"  : "~0.6.2",
-        "windshaft"  : "~0.5.8",
+        "underscore"  : "~1.3.3",
+        "grainstore"  : "~0.9.1",
+        "windshaft"  : "~0.6.2",
         "step": "0.0.x",
         "generic-pool": "1.0.x",
         "redis": "0.7.2",
         "hiredis": "~0.1.14",
-        "request": "2.9.202"
+        "request": "2.9.202",
+        "mapnik": "~0.7.14"
     },
     "devDependencies": {
         "mocha": "1.2.1"

test/acceptance/server.js

@@ -47,8 +47,12 @@ suite('server', function() {
         },{
             status: 200,
             headers: { 'X-Cache-Channel': 'cartodb_test_user_1_db:my_table' },
-            body: '{"style":"#my_table {marker-fill: #FF6600;marker-opacity: 1;marker-width: 8;marker-line-color: white;marker-line-width: 3;marker-line-opacity: 0.9;marker-placement: point;marker-type: ellipse;marker-allow-overlap: true;}"}'
-        }, function() { done(); });
+        }, function(res) {
+            var parsed = JSON.parse(res.body);
+            assert.equal(parsed.style, "#my_table {marker-fill: #FF6600;marker-opacity: 1;marker-width: 8;marker-line-color: white;marker-line-width: 3;marker-line-opacity: 0.9;marker-placement: point;marker-type: ellipse;marker-allow-overlap: true;}");
+            assert.equal(parsed.version, '2.0.0');
+            done();
+        });
     });
 
     // See https://github.com/Vizzuality/Windshaft-cartodb/issues/43
@@ -77,7 +81,9 @@ suite('server', function() {
         },{
         }, function(res) {
           assert.equal(res.statusCode, 200, res.body);
-          assert.deepEqual(res.body, '{"style":"#test_table_private_1 {marker-fill: #FF6600;marker-opacity: 1;marker-width: 8;marker-line-color: white;marker-line-width: 3;marker-line-opacity: 0.9;marker-placement: point;marker-type: ellipse;marker-allow-overlap: true;}"}');
+          var parsed = JSON.parse(res.body);
+          assert.equal(parsed.style, "#test_table_private_1 {marker-fill: #FF6600;marker-opacity: 1;marker-width: 8;marker-line-color: white;marker-line-width: 3;marker-line-opacity: 0.9;marker-placement: point;marker-type: ellipse;marker-allow-overlap: true;}");
+          assert.equal(parsed.version, '2.0.0');
           done();
         });
     });
@@ -192,8 +198,12 @@ suite('server', function() {
                 method: 'GET'
             },{
                 status: 200,
-                body: JSON.stringify({style: 'Map {background-color:#fff;}'})
-            }, function() { done(); });
+            }, function(res) {
+              var parsed = JSON.parse(res.body);
+              assert.equal(parsed.style, 'Map {background-color:#fff;}');
+              //assert.equal(parsed.version, '2.0.0');
+              done();
+            });
 
           });
         });
@@ -217,8 +227,12 @@ suite('server', function() {
                 method: 'GET'
             },{
                 status: 200,
-                body: JSON.stringify({style: style})
-            }, function() { done(); });
+            }, function(res) {
+              var parsed = JSON.parse(res.body);
+              assert.equal(parsed.style, style);
+              //assert.equal(parsed.version, '2.0.0');
+              done();
+            });
 
         });
     
@@ -248,8 +262,12 @@ suite('server', function() {
               method: 'GET'
           },{
               status: 200,
-              body: JSON.stringify({style: 'Map {background-color:#fff;}'})
-          }, function() { done(); });
+          }, function(res) {
+              var parsed = JSON.parse(res.body);
+              assert.equal(parsed.style, 'Map {background-color:#fff;}');
+              //assert.equal(parsed.version, '2.0.0');
+              done();
+          });
         });
     });
 

tools/reset_styles

@@ -1,35 +1,85 @@
 #!/usr/bin/env node
 
-// Reset redis-stored XML styles so that they are regenerated
-// from CartoCSS on next tile request
+var path = require('path');
 
-var redis = require('redis')
+// Reset all styles in the store
+var grainstore = require('../node_modules/grainstore/lib/grainstore');
+var mapnik = require('mapnik');
+var redis = require('redis');
 
-var REDIS_PORT = 6379; // TODO: make a parameter
+function usage(me, exitcode) {
+  console.log("Usage: " + me + " [--convert] [<target_mapnik_version>]");
+  process.exit(exitcode);
+}
+
+var doConvert = false;
+var MAPNIK_VERSION;
+
+var node_path = process.argv.shift(); 
+var script_path = process.argv.shift(); 
+var me = path.basename(script_path);
+var arg;
+while ( arg = process.argv.shift() ) {
+  if ( arg == '--convert' ) {
+    doConvert = true;
+  } else if ( ! MAPNIK_VERSION ) {
+    MAPNIK_VERSION = arg;
+  }
+  else {
+    usage(me, 1);
+  }
+}
+
+if ( ! MAPNIK_VERSION ) {
+  MAPNIK_VERSION = mapnik.versions.mapnik;
+}
+
+console.log( (doConvert ? "Converting" : "Resetting" ) + ' all styles to target ' + MAPNIK_VERSION);
+
+var REDIS_PORT = 6379; // TODO: make a command line parameter
 
 var dbnum = 0;
 
+var mml_store = new grainstore.MMLStore({port:REDIS_PORT}, {mapnik_version:MAPNIK_VERSION});
+
+var failures = [];
+
 var client = redis.createClient(REDIS_PORT, 'localhost');
 client.on('connect', function() {
   client.select(dbnum);
   client.keys('map_style|*', function(err, matches) {
 
     processNext = function() {
-      if ( ! matches.length ) process.exit(0);
+      if ( ! matches.length ) process.exit(failures.length);
       var k = matches.shift();
-      console.log("Resetting XML in key: " + k);
-      client.get(k, function(err, val) {
-        if ( err ) throw err;
-        val = JSON.parse(val);
-        delete val.xml;
-        client.set(k, JSON.stringify(val), function() {
-          console.log("done with style " + k);
-          processNext();
-        });
+
+      if ( /map_style\|.*\|.*\|/.test(k) ) {
+        //console.warn("Key " + k + " is EXTENDED, skipping");
+        processNext();
+      } 
+
+      var params = RegExp(/map_style\|(.*)\|(.*)/).exec(k);
+      var db = params[1];
+      var tab = params[2];
+      var out = 'map_style|' + db + '|' + tab + ': ';
+
+      var mml_builder = mml_store.mml_builder({dbname:db, table:tab},
+                                              function(err, payload) {
+
+          if ( err ) { console.warn(out + err.message); failures.push(k); processNext(); }
+          else {
+            mml_builder.resetStyle(function(err, data) {
+              if ( err ) { console.warn(out + err.message); failures.push(k); }
+              else console.log(out + 'OK' + ( doConvert ? ' (converted)' : '' ));
+              processNext();
+            }, doConvert);
+          }
       });
-    }
+
+    };
 
     processNext();
 
   });
+
 });