From 90f6e464d27b220840f972dcf4f1d191931cbd87 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Mon, 8 Oct 2012 16:41:16 +0200 Subject: [PATCH 01/28] Make test tolerant to additional fields in responses to POST style --- test/acceptance/server.js | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/test/acceptance/server.js b/test/acceptance/server.js index 0f30db81..ad88d7f0 100644 --- a/test/acceptance/server.js +++ b/test/acceptance/server.js @@ -192,8 +192,12 @@ suite('server', function() { method: 'GET' },{ status: 200, - body: JSON.stringify({style: 'Map {background-color:#fff;}'}) - }, function() { done(); }); + }, function(res) { + var parsed = JSON.parse(res.body); + assert.equal(parsed.style, 'Map {background-color:#fff;}'); + //assert.equal(parsed.version, '2.0.0'); + done(); + }); }); }); @@ -217,8 +221,12 @@ suite('server', function() { method: 'GET' },{ status: 200, - body: JSON.stringify({style: style}) - }, function() { done(); }); + }, function(res) { + var parsed = JSON.parse(res.body); + assert.equal(parsed.style, style); + //assert.equal(parsed.version, '2.0.0'); + done(); + }); }); @@ -248,8 +256,12 @@ suite('server', function() { method: 'GET' },{ status: 200, - body: JSON.stringify({style: 'Map {background-color:#fff;}'}) - }, function() { done(); }); + }, function(res) { + var parsed = JSON.parse(res.body); + assert.equal(parsed.style, 'Map {background-color:#fff;}'); + //assert.equal(parsed.version, '2.0.0'); + done(); + }); }); }); From 961269fa1f1cf1478a831866373133ef6f75a52d Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Mon, 24 Sep 2012 17:57:39 +0200 Subject: [PATCH 02/28] Autodetect target mapnik version and let config override it Closes #40 --- config/environments/development.js.example | 1 + config/environments/production.js.example | 1 + config/environments/staging.js.example | 1 + config/environments/test.js.example | 1 + lib/cartodb/server_options.js | 7 +++++-- npm-shrinkwrap.json | 11 ++++++----- package.json | 3 ++- 7 files changed, 17 insertions(+), 8 deletions(-) diff --git a/config/environments/development.js.example b/config/environments/development.js.example index 8e6c1f77..d44a2eca 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -19,6 +19,7 @@ var config = { */ simplify: true } + ,mapnik_version: '2.0.2' ,millstone: { cache_basedir: '/tmp/cdb-tiler-dev/millstone-dev' } diff --git a/config/environments/production.js.example b/config/environments/production.js.example index 1198afd5..da3e8c25 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -13,6 +13,7 @@ var config = { extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188", simplify: true } + ,mapnik_version: '2.0.2' ,millstone: { cache_basedir: '/home/ubuntu/tile_assets/' } diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 707b2a70..e2522451 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -13,6 +13,7 @@ var config = { extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188", simplify: true } + ,mapnik_version: '2.0.2' ,millstone: { cache_basedir: '/home/ubuntu/tile_assets/' } diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 2d6d4891..a021f25b 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -14,6 +14,7 @@ var config = { extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188", simplify: true } + ,mapnik_version: '2.0.2' ,millstone: { cache_basedir: '/tmp/cdb-tiler-test/millstone' } diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index 5bdecdf5..2a2a11aa 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -1,14 +1,17 @@ var _ = require('underscore') , Step = require('step') , cartoData = require('./carto_data') - , Cache = require('./cache_validator'); + , Cache = require('./cache_validator') + , mapnik = require('mapnik') + ; module.exports = function(){ var me = { base_url: '/tiles/:table', grainstore: { datasource: global.environment.postgres, - cachedir: global.environment.millstone.cache_basedir + cachedir: global.environment.millstone.cache_basedir, + mapnik_version: global.environment.mapnik_version || mapnik.versions.mapnik }, redis: global.environment.redis, enable_cors: global.environment.enable_cors, diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 975a914f..b36c59cb 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -1,9 +1,10 @@ { "name": "windshaft-cartodb", - "version": "0.2.0-dev", + "version": "1.0.0", "dependencies": { "cluster2": { - "version": "0.3.5", + "version": "0.3.5-cdb01", + "from": "git://github.com/CartoDB/cluster2.git#28cde11", "dependencies": { "express": { "version": "2.5.11", @@ -299,9 +300,6 @@ "version": "0.3.3-dev", "from": "git://github.com/Vizzuality/tilelive-mapnik.git#7df70554", "dependencies": { - "mapnik": { - "version": "0.7.14" - }, "eio": { "version": "0.1.0" }, @@ -327,6 +325,9 @@ "request": { "version": "2.9.202" }, + "mapnik": { + "version": "0.7.14" + }, "mocha": { "version": "1.2.1", "dependencies": { diff --git a/package.json b/package.json index c014b39c..698b94b3 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,8 @@ "generic-pool": "1.0.x", "redis": "0.7.2", "hiredis": "~0.1.14", - "request": "2.9.202" + "request": "2.9.202", + "mapnik": "~0.7.14" }, "devDependencies": { "mocha": "1.2.1" From f8e88153f4e0eb4d2255764c634169b2df5a35eb Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Mon, 8 Oct 2012 17:45:52 +0200 Subject: [PATCH 03/28] Updated --- NEWS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.md b/NEWS.md index 9f6cb4af..7dc4577f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,6 @@ 1.1.0 (DD/MM/YY) ----- +* Autodetected (and configurable) target mapnik version * Configurable logging format (#4) * Detailed error on missing user metadata * Properly handle unauthenticated requests for metadata From 89ca4e1a5a7b4a282f31d4224c18c79e90ac3eea Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Mon, 8 Oct 2012 18:02:28 +0200 Subject: [PATCH 04/28] Update windshaft to 0.6 exposing CartoCSS versioning support --- NEWS.md | 6 +++++- npm-shrinkwrap.json | 25 +++++++------------------ package.json | 8 ++++---- 3 files changed, 16 insertions(+), 23 deletions(-) diff --git a/NEWS.md b/NEWS.md index 7dc4577f..29626c1b 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,10 @@ 1.1.0 (DD/MM/YY) ----- -* Autodetected (and configurable) target mapnik version +* Add /version entry point +* CartoCSS versioning + * Include version in GET /style response + * Support version and convert parameters in POST /style request + * Autodetect target mapnik version and let config override it * Configurable logging format (#4) * Detailed error on missing user metadata * Properly handle unauthenticated requests for metadata diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index b36c59cb..305558bf 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -1,6 +1,6 @@ { "name": "windshaft-cartodb", - "version": "1.0.0", + "version": "1.1.0", "dependencies": { "cluster2": { "version": "0.3.5-cdb01", @@ -177,20 +177,16 @@ "version": "0.1.1" }, "underscore": { - "version": "1.1.7" + "version": "1.3.3" }, "grainstore": { - "version": "0.6.4", + "version": "0.8.1", "dependencies": { "carto": { - "version": "0.8.2-cdb-dev-3", - "from": "git://github.com/CartoDB/carto.git#cdb-0.8", + "version": "0.9.2", "dependencies": { - "underscore": { - "version": "1.3.3" - }, "mapnik-reference": { - "version": "4.0.5" + "version": "5.0.0" }, "xml2js": { "version": "0.1.14", @@ -203,12 +199,8 @@ } }, "millstone": { - "version": "0.5.10-cdb-01", - "from": "git://github.com/CartoDB/millstone.git#cdb-node04-devel", + "version": "0.5.10", "dependencies": { - "underscore": { - "version": "1.3.3" - }, "request": { "version": "2.11.4", "dependencies": { @@ -253,11 +245,8 @@ } }, "windshaft": { - "version": "0.5.8", + "version": "0.6.0", "dependencies": { - "underscore": { - "version": "1.3.3" - }, "express": { "version": "2.5.11", "dependencies": { diff --git a/package.json b/package.json index 698b94b3..1fdde1bf 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "1.0.1", + "version": "1.1.0", "description": "A map tile server for CartoDB", "url": "https://github.com/Vizzuality/Windshaft-cartodb", "licenses": [{ @@ -20,9 +20,9 @@ "dependencies": { "cluster2": "git://github.com/CartoDB/cluster2.git#28cde11", "node-varnish": "0.1.1", - "underscore" : "1.1.x", - "grainstore" : "~0.6.2", - "windshaft" : "~0.5.8", + "underscore" : "~1.3.3", + "grainstore" : "~0.8.0", + "windshaft" : "~0.6.0", "step": "0.0.x", "generic-pool": "1.0.x", "redis": "0.7.2", From 703c63d5d655b4c8f9600e9e5a49ddd36a113a68 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Mon, 8 Oct 2012 18:06:12 +0200 Subject: [PATCH 05/28] Batch-convert and mapnik version detection support in reset_styles --- NEWS.md | 1 + tools/reset_styles | 80 +++++++++++++++++++++++++++++++++++++--------- 2 files changed, 66 insertions(+), 15 deletions(-) diff --git a/NEWS.md b/NEWS.md index 29626c1b..286c78f1 100644 --- a/NEWS.md +++ b/NEWS.md @@ -5,6 +5,7 @@ * Include version in GET /style response * Support version and convert parameters in POST /style request * Autodetect target mapnik version and let config override it + * Add tools/reset_styles script to batch-reset (and optionally convert) styles * Configurable logging format (#4) * Detailed error on missing user metadata * Properly handle unauthenticated requests for metadata diff --git a/tools/reset_styles b/tools/reset_styles index 53b736dc..863312ef 100755 --- a/tools/reset_styles +++ b/tools/reset_styles @@ -1,35 +1,85 @@ #!/usr/bin/env node -// Reset redis-stored XML styles so that they are regenerated -// from CartoCSS on next tile request +var path = require('path'); -var redis = require('redis') +// Reset all styles in the store +var grainstore = require('../node_modules/grainstore/lib/grainstore'); +var mapnik = require('mapnik'); +var redis = require('redis'); -var REDIS_PORT = 6379; // TODO: make a parameter +function usage(me, exitcode) { + console.log("Usage: " + me + " [--convert] []"); + process.exit(exitcode); +} + +var doConvert = false; +var MAPNIK_VERSION; + +var node_path = process.argv.shift(); +var script_path = process.argv.shift(); +var me = path.basename(script_path); +var arg; +while ( arg = process.argv.shift() ) { + if ( arg == '--convert' ) { + doConvert = true; + } else if ( ! MAPNIK_VERSION ) { + MAPNIK_VERSION = arg; + } + else { + usage(me, 1); + } +} + +if ( ! MAPNIK_VERSION ) { + MAPNIK_VERSION = mapnik.versions.mapnik; +} + +console.log( (doConvert ? "Converting" : "Resetting" ) + ' all styles to target ' + MAPNIK_VERSION); + +var REDIS_PORT = 6379; // TODO: make a command line parameter var dbnum = 0; +var mml_store = new grainstore.MMLStore({port:REDIS_PORT}, {mapnik_version:MAPNIK_VERSION}); + +var failures = []; + var client = redis.createClient(REDIS_PORT, 'localhost'); client.on('connect', function() { client.select(dbnum); client.keys('map_style|*', function(err, matches) { processNext = function() { - if ( ! matches.length ) process.exit(0); + if ( ! matches.length ) process.exit(failures.length); var k = matches.shift(); - console.log("Resetting XML in key: " + k); - client.get(k, function(err, val) { - if ( err ) throw err; - val = JSON.parse(val); - delete val.xml; - client.set(k, JSON.stringify(val), function() { - console.log("done with style " + k); - processNext(); - }); + + if ( /map_style\|.*\|.*\|/.test(k) ) { + //console.warn("Key " + k + " is EXTENDED, skipping"); + processNext(); + } + + var params = RegExp(/map_style\|(.*)\|(.*)/).exec(k); + var db = params[1]; + var tab = params[2]; + var out = 'map_style|' + db + '|' + tab + ': '; + + var mml_builder = mml_store.mml_builder({dbname:db, table:tab}, + function(err, payload) { + + if ( err ) { console.warn(out + err.message); failures.push(k); processNext(); } + else { + mml_builder.resetStyle(function(err, data) { + if ( err ) { console.warn(out + err.message); failures.push(k); } + else console.log(out + 'OK' + ( doConvert ? ' (converted)' : '' )); + processNext(); + }, doConvert); + } }); - } + + }; processNext(); }); + }); From 6a1933bed952a0000592c08822422d13cfc51afd Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Tue, 9 Oct 2012 11:45:57 +0200 Subject: [PATCH 06/28] Print a warning when configured mapnik version doesn't match installed --- lib/cartodb/server_options.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index 2a2a11aa..b0ce3ba8 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -21,6 +21,14 @@ module.exports = function(){ log_format: global.environment.log_format }; + // Be nice and warn if configured mapnik version + // is != instaled mapnik version + if ( mapnik.versions.mapnik != me.grainstore.mapnik_version ) { + console.warn("WARNING: detected mapnik version (" + + mapnik.versions.mapnik + ") != configured mapnik version (" + + me.grainstore.mapnik_version + ")"); + } + // Set the cache chanel info to invalidate the cache on the frontend server // // @param req The request object. From 0697873a644ca59a3359904019865049020d93d2 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Tue, 9 Oct 2012 11:53:23 +0200 Subject: [PATCH 07/28] Upgrade windshaft/grainstore to fix /version route --- npm-shrinkwrap.json | 4 ++-- package.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 305558bf..f6b6a926 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -180,7 +180,7 @@ "version": "1.3.3" }, "grainstore": { - "version": "0.8.1", + "version": "0.9.0", "dependencies": { "carto": { "version": "0.9.2", @@ -245,7 +245,7 @@ } }, "windshaft": { - "version": "0.6.0", + "version": "0.6.1", "dependencies": { "express": { "version": "2.5.11", diff --git a/package.json b/package.json index 1fdde1bf..4bc47881 100644 --- a/package.json +++ b/package.json @@ -21,8 +21,8 @@ "cluster2": "git://github.com/CartoDB/cluster2.git#28cde11", "node-varnish": "0.1.1", "underscore" : "~1.3.3", - "grainstore" : "~0.8.0", - "windshaft" : "~0.6.0", + "grainstore" : "~0.9.0", + "windshaft" : "~0.6.1", "step": "0.0.x", "generic-pool": "1.0.x", "redis": "0.7.2", From 9cdd2800fa44f13ecdad627ef6aeca5f22c82f38 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Tue, 9 Oct 2012 12:12:33 +0200 Subject: [PATCH 08/28] Use "undefined" mapnik_version in the example configs Using "undefined" for mapnik_version triggers autodetection, which is more appropriate. --- config/environments/development.js.example | 2 +- config/environments/production.js.example | 2 +- config/environments/staging.js.example | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config/environments/development.js.example b/config/environments/development.js.example index d44a2eca..caed4f92 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -19,7 +19,7 @@ var config = { */ simplify: true } - ,mapnik_version: '2.0.2' + ,mapnik_version: undefined ,millstone: { cache_basedir: '/tmp/cdb-tiler-dev/millstone-dev' } diff --git a/config/environments/production.js.example b/config/environments/production.js.example index da3e8c25..8d910b7b 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -13,7 +13,7 @@ var config = { extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188", simplify: true } - ,mapnik_version: '2.0.2' + ,mapnik_version: undefined ,millstone: { cache_basedir: '/home/ubuntu/tile_assets/' } diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index e2522451..9c10dad7 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -13,7 +13,7 @@ var config = { extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188", simplify: true } - ,mapnik_version: '2.0.2' + ,mapnik_version: undefined ,millstone: { cache_basedir: '/home/ubuntu/tile_assets/' } From bc51b1448537b80956df62a6f21db741b2f75957 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Tue, 9 Oct 2012 14:39:38 +0200 Subject: [PATCH 09/28] Require grainstore 0.9.1 for automatic styles reset on mapnik upgrade --- npm-shrinkwrap.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index f6b6a926..dd2a909b 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -180,7 +180,7 @@ "version": "1.3.3" }, "grainstore": { - "version": "0.9.0", + "version": "0.9.1", "dependencies": { "carto": { "version": "0.9.2", diff --git a/package.json b/package.json index 4bc47881..2f37affd 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "cluster2": "git://github.com/CartoDB/cluster2.git#28cde11", "node-varnish": "0.1.1", "underscore" : "~1.3.3", - "grainstore" : "~0.9.0", + "grainstore" : "~0.9.1", "windshaft" : "~0.6.1", "step": "0.0.x", "generic-pool": "1.0.x", From bb00bf1c05dc31fd5150e81ffd63c83e03573303 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Tue, 9 Oct 2012 17:09:40 +0200 Subject: [PATCH 10/28] Use 'host' configuration for HTTP listening (both app and cluster) --- app.js | 6 ++++-- cluster.js | 7 +++++-- npm-shrinkwrap.json | 2 +- package.json | 2 +- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/app.js b/app.js index 4bd8a6b4..f432a617 100755 --- a/app.js +++ b/app.js @@ -33,5 +33,7 @@ var Windshaft = require('windshaft'); var serverOptions = require('./lib/cartodb/server_options'); ws = CartodbWindshaft(serverOptions); -ws.listen(global.environment.port); -console.log("Windshaft tileserver started on port " + global.environment.port); +ws.listen(global.environment.port, global.environment.host); +ws.on('listening', function() { + console.log("Windshaft tileserver started on " + global.environment.host + ':' + global.environment.port); +}); diff --git a/cluster.js b/cluster.js index d72600d1..fc342a34 100755 --- a/cluster.js +++ b/cluster.js @@ -41,12 +41,15 @@ var ws = CartodbWindshaft(serverOptions); //.use(cluster.pidfiles('pids')) var cluster = new Cluster({ port: global.environment.port, + host: global.environment.host, monPort: global.environment.port+1, - noWorkers: 1 // .set('workers', 1) + monHost: global.environment.host, + noWorkers: 1 }); cluster.listen(function(cb) { cb(ws); +}, function() { + console.log("Windshaft tileserver started on port " + global.environment.port); }); -console.log("Windshaft tileserver started on port " + global.environment.port); diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index dd2a909b..5862215f 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -4,7 +4,7 @@ "dependencies": { "cluster2": { "version": "0.3.5-cdb01", - "from": "git://github.com/CartoDB/cluster2.git#28cde11", + "from": "git://github.com/CartoDB/cluster2.git#cdb_production", "dependencies": { "express": { "version": "2.5.11", diff --git a/package.json b/package.json index 2f37affd..ebceadeb 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "email": "simon@vizzuality.com" }, "dependencies": { - "cluster2": "git://github.com/CartoDB/cluster2.git#28cde11", + "cluster2": "git://github.com/CartoDB/cluster2.git#cdb_production", "node-varnish": "0.1.1", "underscore" : "~1.3.3", "grainstore" : "~0.9.1", From 926b47b0091c3d8e0992741a9527c1af531d5551 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Tue, 9 Oct 2012 18:35:27 +0200 Subject: [PATCH 11/28] Fix test expectance after windshaft/grainstore upgrade Now GET /style response includes CartoCSS version... --- test/acceptance/server.js | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/test/acceptance/server.js b/test/acceptance/server.js index ad88d7f0..dcf590ff 100644 --- a/test/acceptance/server.js +++ b/test/acceptance/server.js @@ -47,8 +47,12 @@ suite('server', function() { },{ status: 200, headers: { 'X-Cache-Channel': 'cartodb_test_user_1_db:my_table' }, - body: '{"style":"#my_table {marker-fill: #FF6600;marker-opacity: 1;marker-width: 8;marker-line-color: white;marker-line-width: 3;marker-line-opacity: 0.9;marker-placement: point;marker-type: ellipse;marker-allow-overlap: true;}"}' - }, function() { done(); }); + }, function(res) { + var parsed = JSON.parse(res.body); + assert.equal(parsed.style, "#my_table {marker-fill: #FF6600;marker-opacity: 1;marker-width: 8;marker-line-color: white;marker-line-width: 3;marker-line-opacity: 0.9;marker-placement: point;marker-type: ellipse;marker-allow-overlap: true;}"); + assert.equal(parsed.version, '2.0.0'); + done(); + }); }); // See https://github.com/Vizzuality/Windshaft-cartodb/issues/43 @@ -77,7 +81,9 @@ suite('server', function() { },{ }, function(res) { assert.equal(res.statusCode, 200, res.body); - assert.deepEqual(res.body, '{"style":"#test_table_private_1 {marker-fill: #FF6600;marker-opacity: 1;marker-width: 8;marker-line-color: white;marker-line-width: 3;marker-line-opacity: 0.9;marker-placement: point;marker-type: ellipse;marker-allow-overlap: true;}"}'); + var parsed = JSON.parse(res.body); + assert.equal(parsed.style, "#test_table_private_1 {marker-fill: #FF6600;marker-opacity: 1;marker-width: 8;marker-line-color: white;marker-line-width: 3;marker-line-opacity: 0.9;marker-placement: point;marker-type: ellipse;marker-allow-overlap: true;}"); + assert.equal(parsed.version, '2.0.0'); done(); }); }); From 20dca2e8f88d593935aa57b37a5de842d44e2bcd Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Thu, 11 Oct 2012 16:48:41 +0200 Subject: [PATCH 12/28] Use windshaft-0.6.2 sendError function to send non-200 responses Ensures all errors are logged --- lib/cartodb/cartodb_windshaft.js | 9 ++++++--- npm-shrinkwrap.json | 6 +++--- package.json | 2 +- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/cartodb/cartodb_windshaft.js b/lib/cartodb/cartodb_windshaft.js index e4536a82..86854861 100644 --- a/lib/cartodb/cartodb_windshaft.js +++ b/lib/cartodb/cartodb_windshaft.js @@ -37,7 +37,8 @@ var CartodbWindshaft = function(serverOptions) { }, function(err, data){ if (err){ - res.send({error: err.message}, 500); + ws.sendError(res, {error: err.message}, 500, 'GET INFOWINDOW'); + //res.send({error: err.message}, 500); } else { res.send({infowindow: data}, 200); } @@ -57,7 +58,8 @@ var CartodbWindshaft = function(serverOptions) { }, function(err, data){ if (err){ - res.send(err.message, 500); + ws.sendError(res, {error: err.message}, 500, 'GET MAP_METADATA'); + //res.send(err.message, 500); } else { res.send({map_metadata: data}, 200); } @@ -77,7 +79,8 @@ var CartodbWindshaft = function(serverOptions) { }, function(err, data){ if (err){ - res.send(500); + ws.sendError(res, {error: err.message}, 500, 'DELETE CACHE'); + //res.send(500); } else { res.send({status: 'ok'}, 200); } diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 5862215f..d49f6031 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -3,7 +3,7 @@ "version": "1.1.0", "dependencies": { "cluster2": { - "version": "0.3.5-cdb01", + "version": "0.3.5-cdb02", "from": "git://github.com/CartoDB/cluster2.git#cdb_production", "dependencies": { "express": { @@ -245,7 +245,7 @@ } }, "windshaft": { - "version": "0.6.1", + "version": "0.6.2", "dependencies": { "express": { "version": "2.5.11", @@ -273,7 +273,7 @@ "version": "4.3.1", "dependencies": { "optimist": { - "version": "0.3.4", + "version": "0.3.5", "dependencies": { "wordwrap": { "version": "0.0.2" diff --git a/package.json b/package.json index ebceadeb..c9cd7025 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "node-varnish": "0.1.1", "underscore" : "~1.3.3", "grainstore" : "~0.9.1", - "windshaft" : "~0.6.1", + "windshaft" : "~0.6.2", "step": "0.0.x", "generic-pool": "1.0.x", "redis": "0.7.2", From ab8cb5bbb32eb2bfb268ea1b2b07de2b0857177d Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Mon, 8 Oct 2012 17:26:12 +0200 Subject: [PATCH 13/28] Add Windshaft-cartodb version to the /version route --- lib/cartodb/cartodb_windshaft.js | 8 ++++++++ test/acceptance/server.js | 29 +++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/lib/cartodb/cartodb_windshaft.js b/lib/cartodb/cartodb_windshaft.js index 86854861..ded52079 100644 --- a/lib/cartodb/cartodb_windshaft.js +++ b/lib/cartodb/cartodb_windshaft.js @@ -26,6 +26,14 @@ var CartodbWindshaft = function(serverOptions) { // boot var ws = new Windshaft.Server(serverOptions); + // Override getVersion to include cartodb-specific versions + var wsversion = ws.getVersion; + ws.getVersion = function() { + var version = wsversion(); + version.windshaft_cartodb = require('../../package.json').version; + return version; + } + /** * Helper to allow access to the layer to be used in the maps infowindow popup. */ diff --git a/test/acceptance/server.js b/test/acceptance/server.js index dcf590ff..a81662ce 100644 --- a/test/acceptance/server.js +++ b/test/acceptance/server.js @@ -33,6 +33,35 @@ suite('server', function() { }, function() { done(); }); }); + ///////////////////////////////////////////////////////////////////////////////// + // + // GET VERSION + // + ///////////////////////////////////////////////////////////////////////////////// + + test("get call to server returns 200", function(done){ + assert.response(server, { + url: '/version', + method: 'GET' + },{ + status: 200 + }, function(res) { + var parsed = JSON.parse(res.body); + assert.ok(parsed.hasOwnProperty('windshaft_cartodb'), "No 'windshaft_cartodb' version in " + parsed); + console.log("Windshaft-cartodb: " + parsed.windshaft_cartodb); + assert.ok(parsed.hasOwnProperty('windshaft'), "No 'windshaft' version in " + parsed); + console.log("Windshaft: " + parsed.windshaft); + assert.ok(parsed.hasOwnProperty('grainstore'), "No 'grainstore' version in " + parsed); + console.log("Grainstore: " + parsed.grainstore); + assert.ok(parsed.hasOwnProperty('node_mapnik'), "No 'node_mapnik' version in " + parsed); + console.log("Node-mapnik: " + parsed.node_mapnik); + assert.ok(parsed.hasOwnProperty('mapnik'), "No 'mapnik' version in " + parsed); + console.log("Mapnik: " + parsed.mapnik); + // TODO: check actual versions ? + done(); + }); + }); + ///////////////////////////////////////////////////////////////////////////////// // // GET STYLE From 1a60c55feab0bc49bfd9eddc5291b7397d15329e Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Fri, 19 Oct 2012 12:52:39 +0200 Subject: [PATCH 14/28] Rename carto version parameters and add its support to GET tile Also upgrades "carto" to 0.9.3 and "millstone" to 0.5.11 --- NEWS.md | 5 +++-- npm-shrinkwrap.json | 10 +++++----- package.json | 4 ++-- test/acceptance/server.js | 4 ++-- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/NEWS.md b/NEWS.md index 286c78f1..2821ad66 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,8 +2,9 @@ ----- * Add /version entry point * CartoCSS versioning - * Include version in GET /style response - * Support version and convert parameters in POST /style request + * Include style_version in GET /style response + * Support style_version and style_convert parameters in POST /style request + * Support style_version in GET /:z/:x/:y request * Autodetect target mapnik version and let config override it * Add tools/reset_styles script to batch-reset (and optionally convert) styles * Configurable logging format (#4) diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index d49f6031..da09b3ac 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -180,10 +180,10 @@ "version": "1.3.3" }, "grainstore": { - "version": "0.9.1", + "version": "0.9.2", "dependencies": { "carto": { - "version": "0.9.2", + "version": "0.9.3", "dependencies": { "mapnik-reference": { "version": "5.0.0" @@ -199,7 +199,7 @@ } }, "millstone": { - "version": "0.5.10", + "version": "0.5.11", "dependencies": { "request": { "version": "2.11.4", @@ -226,7 +226,7 @@ } }, "srs": { - "version": "0.2.16" + "version": "0.2.17" }, "zipfile": { "version": "0.3.2" @@ -245,7 +245,7 @@ } }, "windshaft": { - "version": "0.6.2", + "version": "0.7.0", "dependencies": { "express": { "version": "2.5.11", diff --git a/package.json b/package.json index c9cd7025..44fa89fd 100644 --- a/package.json +++ b/package.json @@ -21,8 +21,8 @@ "cluster2": "git://github.com/CartoDB/cluster2.git#cdb_production", "node-varnish": "0.1.1", "underscore" : "~1.3.3", - "grainstore" : "~0.9.1", - "windshaft" : "~0.6.2", + "grainstore" : "~0.9.2", + "windshaft" : "~0.7.0", "step": "0.0.x", "generic-pool": "1.0.x", "redis": "0.7.2", diff --git a/test/acceptance/server.js b/test/acceptance/server.js index a81662ce..7dad73d6 100644 --- a/test/acceptance/server.js +++ b/test/acceptance/server.js @@ -79,7 +79,7 @@ suite('server', function() { }, function(res) { var parsed = JSON.parse(res.body); assert.equal(parsed.style, "#my_table {marker-fill: #FF6600;marker-opacity: 1;marker-width: 8;marker-line-color: white;marker-line-width: 3;marker-line-opacity: 0.9;marker-placement: point;marker-type: ellipse;marker-allow-overlap: true;}"); - assert.equal(parsed.version, '2.0.0'); + assert.equal(parsed.style_version, '2.0.0'); done(); }); }); @@ -112,7 +112,7 @@ suite('server', function() { assert.equal(res.statusCode, 200, res.body); var parsed = JSON.parse(res.body); assert.equal(parsed.style, "#test_table_private_1 {marker-fill: #FF6600;marker-opacity: 1;marker-width: 8;marker-line-color: white;marker-line-width: 3;marker-line-opacity: 0.9;marker-placement: point;marker-type: ellipse;marker-allow-overlap: true;}"); - assert.equal(parsed.version, '2.0.0'); + assert.equal(parsed.style_version, '2.0.0'); done(); }); }); From d9e6aeb2543c94e01ef2dd82c0921f35ab36fbf5 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Mon, 22 Oct 2012 15:30:16 +0200 Subject: [PATCH 15/28] Fix crash on unknown user. Closes #55. --- NEWS.md | 1 + lib/cartodb/carto_data.js | 2 +- test/acceptance/server.js | 70 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 2821ad66..7525f350 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,6 @@ 1.1.0 (DD/MM/YY) ----- +* Fix crash on unknown user (#55) * Add /version entry point * CartoCSS versioning * Include style_version in GET /style response diff --git a/lib/cartodb/carto_data.js b/lib/cartodb/carto_data.js index 87a8bf8c..5ad4723c 100644 --- a/lib/cartodb/carto_data.js +++ b/lib/cartodb/carto_data.js @@ -126,7 +126,7 @@ module.exports = function() { that.retrieve(that.table_metadata_db, redisKey, 'privacy', this); }, function(err, data){ - if (err) throw err; + //if (err) throw err; callback(err, data); } ); diff --git a/test/acceptance/server.js b/test/acceptance/server.js index 7dad73d6..eb5dd025 100644 --- a/test/acceptance/server.js +++ b/test/acceptance/server.js @@ -101,6 +101,23 @@ suite('server', function() { }); }); + // See http://github.com/Vizzuality/Windshaft-cartodb/issues/55 + test("get'ing style of private table should fail on unknown username", + function(done) { + assert.response(server, { + headers: {host: 'unknown_user'}, + url: '/tiles/test_table_private_1/style', + method: 'GET' + },{ + }, function(res) { + // FIXME: should be 401 Unauthorized + assert.equal(res.statusCode, 500, res.body); + assert.deepEqual(JSON.parse(res.body), + {error:"missing unknown_user's dbname in redis (try CARTODB/script/restore_redis)"}); + done(); + }); + }); + test("get'ing style of private table should succeed when authenticated", function(done) { assert.response(server, { @@ -401,6 +418,23 @@ suite('server', function() { }); }); + // See http://github.com/Vizzuality/Windshaft-cartodb/issues/55 + test("get'ing infowindow of private table should fail on unknown username", + function(done) { + assert.response(server, { + headers: {host: 'unknown_user'}, + url: '/tiles/test_table_private_1/infowindow', + method: 'GET' + },{ + }, function(res) { + // FIXME: should be 401 Unauthorized + assert.equal(res.statusCode, 500, res.body); + assert.deepEqual(JSON.parse(res.body), + {error:"missing unknown_user's dbname in redis (try CARTODB/script/restore_redis)"}); + done(); + }); + }); + test("get'ing infowindow of private table should succeed when authenticated", function(done) { assert.response(server, { @@ -467,6 +501,23 @@ suite('server', function() { }); }); + // See http://github.com/Vizzuality/Windshaft-cartodb/issues/55 + test("get'ing grid of private table should fail on unknown username", + function(done) { + assert.response(server, { + headers: {host: 'unknown_user'}, + url: '/tiles/test_table_private_1/6/31/24.grid.json', + method: 'GET' + },{ + }, function(res) { + // FIXME: should be 401 Unauthorized + assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body); + assert.deepEqual(JSON.parse(res.body), + {error:"missing unknown_user's dbname in redis (try CARTODB/script/restore_redis)"}); + done(); + }); + }); + test("get'ing the grid of a private table should succeed when authenticated", function(done) { assert.response(server, { @@ -566,6 +617,25 @@ suite('server', function() { }); }); + test("get'ing a tile with data from private table should fail on unknown username", function(done){ + var sql = querystring.stringify({ + sql: "SELECT * FROM test_table_private_1", + cache_buster:2 // this is to avoid getting the cached response + }); + assert.response(server, { + headers: {host: 'unknown_user'}, + url: '/tiles/gadm4/6/31/24.png?' + sql, + method: 'GET' + },{ + }, function(res) { + // FIXME: should be 401 Unauthorized + assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body); + assert.deepEqual(JSON.parse(res.body), + {error:"missing unknown_user's dbname in redis (try CARTODB/script/restore_redis)"}); + done(); + }); + }); + test("get'ing a tile with data from private table should fail when unauthenticated (uses old redis key)", function(done){ var sql = querystring.stringify({ sql: "SELECT * FROM test_table_private_1", From 2bc09a61cf7dd45c440c0b0a5725e9a8b0881730 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Wed, 24 Oct 2012 09:40:05 +0200 Subject: [PATCH 16/28] Add support for cache_policy=persistent When cache_policy=persistent is given the response will contain a Cache-Control header requesting for 1 year lifetime caching --- README.md | 12 ++++++++++-- lib/cartodb/server_options.js | 11 ++++++++--- test/acceptance/server.js | 35 ++++++++++++++++++++++++++++++++++- 3 files changed, 52 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 47a48866..e1f66eeb 100644 --- a/README.md +++ b/README.md @@ -80,8 +80,16 @@ Args: * sql - plain SQL arguments * interactivity - specify the column to use in UTFGrid -* cache_buster - if needed you can add a cachebuster to make sure you're - rendering new +* cache_buster - Specify an identifier for the internal tile cache. + Requesting tiles with the same cache_buster value may + result in being served a cached version of the tile + (even when requesting a tile for the first time, as tiles + can be prepared in advance) +* cache_policy - Set to "persist" to have the server send an Cache-Control + header requesting caching devices to keep the response + cached as much as possible. This is best used with a + timestamp value in cache_buster for manual control of + updates. * geom_type - override the cartodb default * style - override the default map style with Carto diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index b0ce3ba8..e1ab444b 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -45,8 +45,13 @@ module.exports = function(){ var ttl = global.environment.varnish.ttl || 86400; Cache.generateCacheChannel(req, function(channel){ res.header('X-Cache-Channel', channel); - res.header('Last-Modified', new Date().toUTCString()); - res.header('Cache-Control', 'no-cache,max-age='+ttl+',must-revalidate, public'); + var cache_policy = req.query.cache_policy; + if ( cache_policy == 'persist' ) { + res.header('Cache-Control', 'public,max-age=31536000'); // 1 year + } else { + res.header('Last-Modified', new Date().toUTCString()); + res.header('Cache-Control', 'no-cache,max-age='+ttl+',must-revalidate, public'); + } cb(null, channel); // add last-modified too ? }); } @@ -60,7 +65,7 @@ module.exports = function(){ me.req2params = function(req, callback){ // Whitelist query parameters and attach format - var good_query = ['sql', 'geom_type', 'cache_buster','callback', 'interactivity', 'map_key', 'api_key', 'style']; + var good_query = ['sql', 'geom_type', 'cache_buster', 'cache_policy', 'callback', 'interactivity', 'map_key', 'api_key', 'style']; var bad_query = _.difference(_.keys(req.query), good_query); _.each(bad_query, function(key){ delete req.query[key]; }); diff --git a/test/acceptance/server.js b/test/acceptance/server.js index eb5dd025..fb564ff7 100644 --- a/test/acceptance/server.js +++ b/test/acceptance/server.js @@ -535,7 +535,40 @@ suite('server', function() { // GET TILE // ///////////////////////////////////////////////////////////////////////////////// - + + test("should send Cache-Control header with short expiration by default", function(done){ + assert.response(server, { + headers: {host: 'localhost'}, + url: '/tiles/gadm4/6/31/24.png', + method: 'GET' + },{ + status: 200, + }, function(res) { + var cc = res.headers['cache-control']; + assert.ok(cc); + //assert.equal(cc, 'public,max-age=31536000'); // 1 year + assert.ok(cc.match('no-cache'), cc); + assert.ok(cc.match('must-revalidate'), cc); + assert.ok(cc.match('public'), cc); + done(); + }); + }); + + test("should send Cache-Control header with long expiration when requested", function(done){ + assert.response(server, { + headers: {host: 'localhost'}, + url: '/tiles/gadm4/6/31/24.png?cache_policy=persist', + method: 'GET' + },{ + status: 200, + }, function(res) { + var cc = res.headers['cache-control']; + assert.ok(cc); + assert.equal(cc, 'public,max-age=31536000'); // 1 year + done(); + }); + }); + test("get'ing a tile with default style should return an image", function(done){ assert.response(server, { headers: {host: 'localhost'}, From 1da7484c2ac0bec906c2d9e801ee62166be72dbb Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Tue, 30 Oct 2012 18:00:16 +0100 Subject: [PATCH 17/28] Shrinkwrap newest windshaft and grainstore --- npm-shrinkwrap.json | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index da09b3ac..13ed6f0d 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -32,13 +32,13 @@ "version": "0.8.3" }, "npm": { - "version": "1.1.62", + "version": "1.1.65", "dependencies": { "semver": { - "version": "1.0.14" + "version": "1.1.0" }, "ini": { - "version": "1.0.4" + "version": "1.0.5" }, "slide": { "version": "1.1.3" @@ -50,7 +50,7 @@ "version": "1.1.14" }, "minimatch": { - "version": "0.2.6" + "version": "0.2.8" }, "nopt": { "version": "2.0.0" @@ -93,10 +93,10 @@ "version": "2.0.4" }, "node-gyp": { - "version": "0.6.11" + "version": "0.7.1" }, "fstream-npm": { - "version": "0.1.2", + "version": "0.1.3", "dependencies": { "fstream-ignore": { "version": "0.0.5" @@ -119,19 +119,24 @@ "version": "0.1.2" }, "npm-registry-client": { - "version": "0.2.7" + "version": "0.2.10", + "dependencies": { + "couch-login": { + "version": "0.1.15" + } + } }, "read-package-json": { - "version": "0.1.5" + "version": "0.1.8" }, "read-installed": { - "version": "0.0.2" + "version": "0.0.3" }, "glob": { - "version": "3.1.12" + "version": "3.1.14" }, "init-package-json": { - "version": "0.0.5", + "version": "0.0.6", "dependencies": { "promzard": { "version": "0.2.0" @@ -147,9 +152,6 @@ "retry": { "version": "0.6.0" }, - "couch-login": { - "version": "0.1.12" - }, "once": { "version": "1.1.1" }, @@ -180,7 +182,7 @@ "version": "1.3.3" }, "grainstore": { - "version": "0.9.2", + "version": "0.9.3", "dependencies": { "carto": { "version": "0.9.3", @@ -245,7 +247,7 @@ } }, "windshaft": { - "version": "0.7.0", + "version": "0.7.1", "dependencies": { "express": { "version": "2.5.11", @@ -315,7 +317,7 @@ "version": "2.9.202" }, "mapnik": { - "version": "0.7.14" + "version": "0.7.16" }, "mocha": { "version": "1.2.1", From f9e0b8708cc83d53a827cb88dece564f4119cf5c Mon Sep 17 00:00:00 2001 From: Luis Bosque Date: Tue, 30 Oct 2012 18:53:49 +0100 Subject: [PATCH 18/28] target version 1.1.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 44fa89fd..caee3996 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "1.1.0", + "version": "1.1.1", "description": "A map tile server for CartoDB", "url": "https://github.com/Vizzuality/Windshaft-cartodb", "licenses": [{ From 8120e6579aa1b85215ef5191efb5a1e3a12ed854 Mon Sep 17 00:00:00 2001 From: Luis Bosque Date: Fri, 2 Nov 2012 15:46:54 +0100 Subject: [PATCH 19/28] updated NEWS --- NEWS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.md b/NEWS.md index 80b213e9..6ad4f13d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,6 @@ 1.1.1 (DD//MM//YY) ----- +* Add support for persistent client cache headers * Fix crash on unknown user (#55) * Add /version entry point * CartoCSS versioning From d1d321194daec3ad87066f5d290edd35cdb0c87e Mon Sep 17 00:00:00 2001 From: Luis Bosque Date: Fri, 2 Nov 2012 15:47:54 +0100 Subject: [PATCH 20/28] target 1.1.2 version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 6ad4f13d..643ff5c1 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,7 @@ +1.1.2 (DD//MM//YY) +----- + + 1.1.1 (DD//MM//YY) ----- * Add support for persistent client cache headers diff --git a/package.json b/package.json index caee3996..455b143e 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "1.1.1", + "version": "1.1.2", "description": "A map tile server for CartoDB", "url": "https://github.com/Vizzuality/Windshaft-cartodb", "licenses": [{ From b5db3a8138d1e8c49b74be3a5a437ef32b7bcfd6 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Fri, 2 Nov 2012 17:50:45 +0100 Subject: [PATCH 21/28] Require grainstore 0.9.4, for better 2.0 -> 2.1 css transforms --- npm-shrinkwrap.json | 8 ++++++-- package.json | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 13ed6f0d..7f73c49c 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -182,10 +182,14 @@ "version": "1.3.3" }, "grainstore": { - "version": "0.9.3", + "version": "0.9.4", "dependencies": { + "semver": { + "version": "1.1.0" + }, "carto": { - "version": "0.9.3", + "version": "0.9.3-cdb1", + "from": "git://github.com/CartoDB/carto.git#cdb-0.9", "dependencies": { "mapnik-reference": { "version": "5.0.0" diff --git a/package.json b/package.json index 455b143e..39d4e92c 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "cluster2": "git://github.com/CartoDB/cluster2.git#cdb_production", "node-varnish": "0.1.1", "underscore" : "~1.3.3", - "grainstore" : "~0.9.2", + "grainstore" : "~0.9.4", "windshaft" : "~0.7.0", "step": "0.0.x", "generic-pool": "1.0.x", From 8ef952f1385ca292097dddc8ee159a16188f8668 Mon Sep 17 00:00:00 2001 From: Luis Bosque Date: Fri, 2 Nov 2012 23:24:45 +0100 Subject: [PATCH 22/28] set 600 seconds timeout in cluster.js --- cluster.js | 1 + 1 file changed, 1 insertion(+) diff --git a/cluster.js b/cluster.js index fc342a34..e40adeeb 100755 --- a/cluster.js +++ b/cluster.js @@ -44,6 +44,7 @@ var cluster = new Cluster({ host: global.environment.host, monPort: global.environment.port+1, monHost: global.environment.host, + timeout: 600, noWorkers: 1 }); From 37cba2836c1f91eb97df4cbe64f2722f803c74d5 Mon Sep 17 00:00:00 2001 From: Luis Bosque Date: Fri, 2 Nov 2012 23:48:08 +0100 Subject: [PATCH 23/28] timeout is in milseconds, not in seconds --- cluster.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster.js b/cluster.js index e40adeeb..3d426735 100755 --- a/cluster.js +++ b/cluster.js @@ -44,7 +44,7 @@ var cluster = new Cluster({ host: global.environment.host, monPort: global.environment.port+1, monHost: global.environment.host, - timeout: 600, + timeout: 600000, noWorkers: 1 }); From 52303e7821d3f1c697b4f9d62c8f9d03dbe8909d Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Tue, 6 Nov 2012 12:31:17 +0100 Subject: [PATCH 24/28] Fix use of "style_version" with GET (inline styles) It took a lot of time to produce a testcase for this as the test config was setting srid to 4326 but not changing geom column name thus all tiles fetched by tests returned blank (ouch!) --- NEWS.md | 1 + config/environments/test.js.example | 1 - lib/cartodb/server_options.js | 2 +- test/acceptance/server.js | 47 +++++++++++++++++++++++++++-- test/support/assert.js | 28 ++++++++++------- 5 files changed, 63 insertions(+), 16 deletions(-) diff --git a/NEWS.md b/NEWS.md index 643ff5c1..f8bb3b17 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,6 @@ 1.1.2 (DD//MM//YY) ----- +* Fix use of "style_version" with GET (inline styles) 1.1.1 (DD//MM//YY) diff --git a/config/environments/test.js.example b/config/environments/test.js.example index a021f25b..908d39a6 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -10,7 +10,6 @@ var config = { user: "publicuser", host: '127.0.0.1', port: 5432, - srid: 4326, extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188", simplify: true } diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index e1ab444b..7550d871 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -65,7 +65,7 @@ module.exports = function(){ me.req2params = function(req, callback){ // Whitelist query parameters and attach format - var good_query = ['sql', 'geom_type', 'cache_buster', 'cache_policy', 'callback', 'interactivity', 'map_key', 'api_key', 'style']; + var good_query = ['sql', 'geom_type', 'cache_buster', 'cache_policy', 'callback', 'interactivity', 'map_key', 'api_key', 'style', 'style_version']; var bad_query = _.difference(_.keys(req.query), good_query); _.each(bad_query, function(key){ delete req.query[key]; }); diff --git a/test/acceptance/server.js b/test/acceptance/server.js index fb564ff7..a31eaa46 100644 --- a/test/acceptance/server.js +++ b/test/acceptance/server.js @@ -247,7 +247,7 @@ suite('server', function() { }, function(res) { var parsed = JSON.parse(res.body); assert.equal(parsed.style, 'Map {background-color:#fff;}'); - //assert.equal(parsed.version, '2.0.0'); + assert.equal(parsed.style_version, '2.0.0'); done(); }); @@ -261,7 +261,7 @@ suite('server', function() { url: '/tiles/my_table5/style?map_key=1234', method: 'POST', headers: {host: 'localhost', 'Content-Type': 'application/x-www-form-urlencoded' }, - data: querystring.stringify({style: style}) + data: querystring.stringify({style: style, style_version: '2.0.2'}) },{ }, function(res) { @@ -276,7 +276,7 @@ suite('server', function() { }, function(res) { var parsed = JSON.parse(res.body); assert.equal(parsed.style, style); - //assert.equal(parsed.version, '2.0.0'); + assert.equal(parsed.style_version, '2.0.2'); done(); }); @@ -689,6 +689,47 @@ suite('server', function() { }); }); + var test_style_black_200 = "#test_table{marker-fill:black;marker-line-color:red;marker-width:10}"; + var test_style_black_210 = "#test_table{marker-fill:black;marker-line-color:red;marker-width:20}"; + + test("get'ing a tile with url specified 2.0.0 style should return an expected tile", function(done){ + var style = querystring.stringify({style: test_style_black_200, style_version: '2.0.0'}); + assert.response(server, { + headers: {host: 'localhost'}, + url: '/tiles/test_table/15/16046/12354.png?cache_buster=4&' + style, // madrid + method: 'GET', + encoding: 'binary' + },{}, function(res){ + assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); + var ct = res.headers['content-type']; + assert.equal(ct, 'image/png'); + assert.imageEqualsFile(res.body, './test/fixtures/test_table_15_16046_12354_styled_black.png', 2, + function(err, similarity) { + if (err) throw err; + done(); + }); + }); + }); + + test("get'ing a tile with url specified 2.1.0 style should return an expected tile", function(done){ + var style = querystring.stringify({style: test_style_black_210, style_version: '2.1.0'}); + assert.response(server, { + headers: {host: 'localhost'}, + url: '/tiles/test_table/15/16046/12354.png?cache_buster=4&' + style, // madrid + method: 'GET', + encoding: 'binary' + },{}, function(res){ + assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); + var ct = res.headers['content-type']; + assert.equal(ct, 'image/png'); + assert.imageEqualsFile(res.body, './test/fixtures/test_table_15_16046_12354_styled_black.png', 2, + function(err, similarity) { + if (err) throw err; + done(); + }); + }); + }); + ///////////////////////////////////////////////////////////////////////////////// // // DELETE CACHE diff --git a/test/support/assert.js b/test/support/assert.js index bd1c45ce..82fcf99d 100644 --- a/test/support/assert.js +++ b/test/support/assert.js @@ -8,29 +8,35 @@ var exec = require('child_process').exec; var assert = module.exports = exports = require('assert'); -assert.imageEqualsFile = function(buffer, file_b, callback) { +// +// @param tol tolerated color distance as a percent over max channel value +// by default this is zero. For meaningful values, see +// http://www.imagemagick.org/script/command-line-options.php#metric +// +assert.imageEqualsFile = function(buffer, file_b, tol, callback) { if (!callback) callback = function(err) { if (err) throw err; }; file_b = path.resolve(file_b); - var file_a = '/tmp/' + (Math.random() * 1e16); + var file_a = '/tmp/windshaft-test-image-test.png'; // + (Math.random() * 1e16); // TODO: make predictable var err = fs.writeFileSync(file_a, buffer, 'binary'); if (err) throw err; - exec('compare -metric PSNR "' + file_a + '" "' + + var fuzz = tol + '%'; + exec('compare -fuzz ' + fuzz + ' -metric AE "' + file_a + '" "' + file_b + '" /dev/null', function(err, stdout, stderr) { if (err) { fs.unlinkSync(file_a); callback(err); } else { stderr = stderr.trim(); - if (stderr === 'inf') { - fs.unlinkSync(file_a); - callback(null); + var similarity = parseFloat(stderr); + if ( similarity > 0 ) { + var err = new Error('Images not equal(' + similarity + '): ' + + file_a + ' ' + file_b); + err.similarity = similarity; + callback(err); } else { - var similarity = parseFloat(stderr); - var err = new Error('Images not equal(' + similarity + '): ' + - file_a + ' ' + file_b); - err.similarity = similarity; - callback(err); + fs.unlinkSync(file_a); + callback(null); } } }); From 43bb96cc6047c5b9a3b5ace0fb346a8f24573660 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Tue, 6 Nov 2012 18:02:46 +0100 Subject: [PATCH 25/28] Require grainstore 0.9.5, fixing one type of style transform --- npm-shrinkwrap.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 7f73c49c..0647a1b0 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -182,7 +182,7 @@ "version": "1.3.3" }, "grainstore": { - "version": "0.9.4", + "version": "0.9.5", "dependencies": { "semver": { "version": "1.1.0" diff --git a/package.json b/package.json index 39d4e92c..120b37dc 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "cluster2": "git://github.com/CartoDB/cluster2.git#cdb_production", "node-varnish": "0.1.1", "underscore" : "~1.3.3", - "grainstore" : "~0.9.4", + "grainstore" : "~0.9.5", "windshaft" : "~0.7.0", "step": "0.0.x", "generic-pool": "1.0.x", From cd6002879e1bc00a90ddc4f4dcdb8c09e1325eba Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Wed, 7 Nov 2012 10:59:39 +0100 Subject: [PATCH 26/28] Add tool to show formatted style stored in redis from user/table --- tools/show_style | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100755 tools/show_style diff --git a/tools/show_style b/tools/show_style new file mode 100755 index 00000000..d98e7889 --- /dev/null +++ b/tools/show_style @@ -0,0 +1,17 @@ +#!/bin/sh + +# TODO: port to node, if you really need it + +REDIS_PORT=6379 # default port + + +if -z "$2"; then + echo "Usage: $0 " >&2 + exit 1 +fi + +username="$1" +tabname="$2" + +dbname=`redis-cli -p ${REDIS_PORT} -n 5 hget "rails:users:${username}" "database_name"` +redis-cli get "map_style|${dbname}|${tabname}" | sed -e 's/\\n/\n/g' -e 's/\\//g' From 4b8c1652618310a3aee087b0d2ef619f6900e50a Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Thu, 8 Nov 2012 18:57:59 +0100 Subject: [PATCH 27/28] Fix usage string for "show_style" tool --- tools/show_style | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/show_style b/tools/show_style index d98e7889..d3b9163b 100755 --- a/tools/show_style +++ b/tools/show_style @@ -5,7 +5,7 @@ REDIS_PORT=6379 # default port -if -z "$2"; then +if test -z "$2"; then echo "Usage: $0 " >&2 exit 1 fi From b5658a9f438448fb1ccac16b3c9e7e37b773108e Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Fri, 9 Nov 2012 17:28:22 +0100 Subject: [PATCH 28/28] Fix point markers shift due to polygon clipping (2.0 -> 2.1) Done by requiring grainstore 0.9.6 --- NEWS.md | 6 +++++- npm-shrinkwrap.json | 2 +- package.json | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/NEWS.md b/NEWS.md index f8bb3b17..06f90363 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,10 @@ 1.1.2 (DD//MM//YY) ----- -* Fix use of "style_version" with GET (inline styles) +* CartoCSS versioning + * Fix use of "style_version" with GET (inline styles) + * Enhance 2.0 -> 2.1 transforms: + * styles with no semicolon + * markers shift due to geometry clipping 1.1.1 (DD//MM//YY) diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 0647a1b0..130d34e2 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -182,7 +182,7 @@ "version": "1.3.3" }, "grainstore": { - "version": "0.9.5", + "version": "0.9.6", "dependencies": { "semver": { "version": "1.1.0" diff --git a/package.json b/package.json index 120b37dc..a6c1d7ae 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "cluster2": "git://github.com/CartoDB/cluster2.git#cdb_production", "node-varnish": "0.1.1", "underscore" : "~1.3.3", - "grainstore" : "~0.9.5", + "grainstore" : "~0.9.6", "windshaft" : "~0.7.0", "step": "0.0.x", "generic-pool": "1.0.x",