From adf1124e701696a26049cea7b76d9f529d4de998 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Tue, 18 Sep 2012 17:36:20 +0200 Subject: [PATCH 1/9] Add script to drop the XML element from all redis stored map styles WARNING: as of grainstore-0.6.1 a map style with no XML will break the application, so do _not_ run this script unless you're sure about what you're doing. --- tools/reset_styles | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100755 tools/reset_styles diff --git a/tools/reset_styles b/tools/reset_styles new file mode 100755 index 00000000..13cace58 --- /dev/null +++ b/tools/reset_styles @@ -0,0 +1,31 @@ +#!/usr/bin/env node + +// Reset redis-stored XML styles so that they are regenerated +// from CartoCSS on next tile request + +var redis = require('redis') + +var REDIS_PORT = 6379; // TODO: make a parameter + +var dbnum = 0; + +var client = redis.createClient(REDIS_PORT, 'localhost'); +client.on('connect', function() { + client.select(dbnum); + client.keys('map_style|*', function(err, matches) { + for (var i=0; i Date: Wed, 19 Sep 2012 18:13:28 +0200 Subject: [PATCH 2/9] Fix test to use expected hostname --- test/acceptance/server.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/acceptance/server.js b/test/acceptance/server.js index fd294a3e..90733899 100644 --- a/test/acceptance/server.js +++ b/test/acceptance/server.js @@ -216,7 +216,7 @@ suite('server', function() { // Retrive style with authenticated request assert.response(server, { - headers: {host: 'localhost'}, + headers: {host: 'vizzuality'}, url: '/tiles/my_table5/style?map_key=1234', method: 'GET' },{}, function(res) { @@ -225,7 +225,7 @@ suite('server', function() { // Now retrive style with unauthenticated request assert.response(server, { - headers: {host: 'localhost'}, + headers: {host: 'vizzuality'}, url: '/tiles/my_table5/style', method: 'GET' }, {}, function(res) { From dd34fc507a052c25a948d751beffa32f6bbf71c0 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Wed, 19 Sep 2012 18:52:13 +0200 Subject: [PATCH 3/9] Automated localization of external resources referenced in carto --- config/environments/development.js | 3 +++ config/environments/production.js | 3 +++ config/environments/staging.js | 3 +++ config/environments/test.js | 3 +++ lib/cartodb/server_options.js | 5 ++++- package.json | 4 ++-- 6 files changed, 18 insertions(+), 3 deletions(-) diff --git a/config/environments/development.js b/config/environments/development.js index e141bfda..9ed0731f 100644 --- a/config/environments/development.js +++ b/config/environments/development.js @@ -18,6 +18,9 @@ var config = { */ simplify: true } + ,millstone: { + cache_basedir: '/tmp/cdb-tiler-dev/millstone-dev' + } ,redis: { host: '127.0.0.1', port: 6379, diff --git a/config/environments/production.js b/config/environments/production.js index 28365971..15a3c9bc 100644 --- a/config/environments/production.js +++ b/config/environments/production.js @@ -12,6 +12,9 @@ var config = { extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188", simplify: true } + ,millstone: { + cache_basedir: '/home/ubuntu/tile_assets/' + } ,redis: { host: '127.0.0.1', port: 6379 diff --git a/config/environments/staging.js b/config/environments/staging.js index 87c0e1c6..c96bb78a 100644 --- a/config/environments/staging.js +++ b/config/environments/staging.js @@ -12,6 +12,9 @@ var config = { extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188", simplify: true } + ,millstone: { + cache_basedir: '/home/ubuntu/tile_assets/' + } ,redis: { host: '127.0.0.1', port: 6379 diff --git a/config/environments/test.js b/config/environments/test.js index d7ee5739..3b8f27f3 100644 --- a/config/environments/test.js +++ b/config/environments/test.js @@ -13,6 +13,9 @@ var config = { extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188", simplify: true } + ,millstone: { + cache_basedir: '/tmp/cdb-tiler-test/millstone' + } ,redis: { host: '127.0.0.1', port: 6333, diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index 81433005..9a56d360 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -5,7 +5,10 @@ var _ = require('underscore') module.exports = function(){ var me = { base_url: '/tiles/:table', - grainstore: {datasource: global.environment.postgres}, + grainstore: { + datasource: global.environment.postgres, + cachedir: global.environment.millstone.cache_basedir + }, redis: global.environment.redis, enable_cors: global.environment.enable_cors, varnish_host: global.environment.varnish.host, diff --git a/package.json b/package.json index 97b35645..0cd4f08e 100644 --- a/package.json +++ b/package.json @@ -22,8 +22,8 @@ "cluster": "0.6.4", "node-varnish": "0.1.1", "underscore" : "1.1.x", - "grainstore" : "~0.5.0", - "windshaft" : "~0.4.13", + "grainstore" : "~0.6.2", + "windshaft" : "~0.4.14", "step": "0.0.x", "generic-pool": "1.0.x", "redis": "0.6.7", From 0654f2eae7812e30ab2611b6014de2be7a4cee74 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Wed, 19 Sep 2012 19:06:47 +0200 Subject: [PATCH 4/9] Tweak redis and hiredis deps to match those of Windshaft --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 0cd4f08e..cd389792 100644 --- a/package.json +++ b/package.json @@ -26,8 +26,8 @@ "windshaft" : "~0.4.14", "step": "0.0.x", "generic-pool": "1.0.x", - "redis": "0.6.7", - "hiredis": "0.1.12", + "redis": "0.7.2", + "hiredis": "~0.1.12", "request": "2.9.202" }, "devDependencies": { From 5c067b39393e66f35e212071979987cab93a47e1 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Wed, 19 Sep 2012 19:09:42 +0200 Subject: [PATCH 5/9] Make test for unauthenticated style setting request predictable Closes #50 --- test/acceptance/server.js | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/test/acceptance/server.js b/test/acceptance/server.js index 90733899..b61c4eb6 100644 --- a/test/acceptance/server.js +++ b/test/acceptance/server.js @@ -129,24 +129,33 @@ suite('server', function() { test("post'ing good style with no authentication returns an error", function(done){ assert.response(server, { - url: '/tiles/my_table5/style', + url: '/tiles/my_table5/style?map_key=1234', method: 'POST', headers: {host: 'vizzuality.localhost.lan', 'Content-Type': 'application/x-www-form-urlencoded' }, - data: querystring.stringify({style: 'Map {background-color:#aaa;}'}) - },{}, function(res) { - // FIXME: should be 401 Unauthorized - assert.equal(res.statusCode, 500, res.body); - assert.ok(res.body.indexOf('map state cannot be changed by unauthenticated request') != -1, res.body); - + data: querystring.stringify({style: 'Map {background-color:#fff;}'}) + },{ + }, function(res) { + assert.equal(res.statusCode, 200, res.body); assert.response(server, { - headers: {host: 'vizzuality.localhost.lan'}, url: '/tiles/my_table5/style', - method: 'GET' - },{ - status: 200, - body: JSON.stringify({style: 'Map {background-color:#fff;}'}) - }, function() { done(); }); + method: 'POST', + headers: {host: 'vizzuality.localhost.lan', 'Content-Type': 'application/x-www-form-urlencoded' }, + data: querystring.stringify({style: 'Map {background-color:#aaa;}'}) + },{}, function(res) { + // FIXME: should be 401 Unauthorized + assert.equal(res.statusCode, 500, res.body); + assert.ok(res.body.indexOf('map state cannot be changed by unauthenticated request') != -1, res.body); + assert.response(server, { + headers: {host: 'vizzuality.localhost.lan'}, + url: '/tiles/my_table5/style', + method: 'GET' + },{ + status: 200, + body: JSON.stringify({style: 'Map {background-color:#fff;}'}) + }, function() { done(); }); + + }); }); }); From 446627e3055972fe883236f4cf4ea31c32c50d89 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Fri, 21 Sep 2012 12:58:34 +0200 Subject: [PATCH 6/9] Fix iteration on redis keys --- tools/reset_styles | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tools/reset_styles b/tools/reset_styles index 13cace58..53b736dc 100755 --- a/tools/reset_styles +++ b/tools/reset_styles @@ -13,19 +13,23 @@ var client = redis.createClient(REDIS_PORT, 'localhost'); client.on('connect', function() { client.select(dbnum); client.keys('map_style|*', function(err, matches) { - for (var i=0; i Date: Mon, 24 Sep 2012 18:57:48 +0200 Subject: [PATCH 7/9] Add an X-Cache-Channel header to all GET requests. Closes #53. --- lib/cartodb/cartodb_windshaft.js | 11 ----------- lib/cartodb/server_options.js | 33 ++++++++++++++++++++++++++++++-- test/acceptance/server.js | 7 +++++-- 3 files changed, 36 insertions(+), 15 deletions(-) diff --git a/lib/cartodb/cartodb_windshaft.js b/lib/cartodb/cartodb_windshaft.js index 52655dc6..69687eff 100644 --- a/lib/cartodb/cartodb_windshaft.js +++ b/lib/cartodb/cartodb_windshaft.js @@ -6,17 +6,6 @@ var _ = require('underscore') var CartodbWindshaft = function(serverOptions) { - // set the cache chanel info to invalidate the cache on the frontend server - serverOptions.afterTileRender = function(req, res, tile, headers, callback) { - var ttl = global.environment.varnish.ttl || 86400; - Cache.generateCacheChannel(req, function(channel){ - res.header('X-Cache-Channel', channel); - res.header('Last-Modified', new Date().toUTCString()); - res.header('Cache-Control', 'no-cache,max-age='+ttl+',must-revalidate, public'); - callback(null, tile, headers); - }); - }; - if(serverOptions.cache_enabled) { console.log("cache invalidation enabled, varnish on ", serverOptions.varnish_host, ' ', serverOptions.varnish_port); Cache.init(serverOptions.varnish_host, serverOptions.varnish_port); diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index 9a56d360..183d7cae 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -1,6 +1,7 @@ var _ = require('underscore') , Step = require('step') - , cartoData = require('./carto_data'); + , cartoData = require('./carto_data') + , Cache = require('./cache_validator'); module.exports = function(){ var me = { @@ -17,6 +18,28 @@ module.exports = function(){ log_format: '[:date] :req[X-Real-IP] \033[90m:method\033[0m \033[36m:req[Host]:url\033[0m \033[90m:status :response-time ms -> :res[Content-Type]\033[0m' }; + // Set the cache chanel info to invalidate the cache on the frontend server + // + // @param req The request object. + // The function will have no effect unless req.res exists. + // It is expected that req.params contains 'table' and 'dbname' + // + // @param cb function(err, channel) will be called when ready. + // the channel parameter will be null if nothing was added + // + me.addCacheChannel = function(req, cb) { + // skip non-GET requests, or requests for which there's no response + if ( req.method != 'GET' || ! req.res ) { cb(null, null); return; } + var res = req.res; + var ttl = global.environment.varnish.ttl || 86400; + Cache.generateCacheChannel(req, function(channel){ + res.header('X-Cache-Channel', channel); + res.header('Last-Modified', new Date().toUTCString()); + res.header('Cache-Control', 'no-cache,max-age='+ttl+',must-revalidate, public'); + cb(null, channel); // add last-modified too ? + }); + } + /** * Whitelist input and get database name & default geometry type from * subdomain/user metadata held in CartoDB Redis @@ -45,6 +68,8 @@ module.exports = function(){ callback(null, xml); } + var that = this; + Step( function getPrivacy(){ cartoData.authorize(req, this); @@ -66,10 +91,14 @@ module.exports = function(){ cartoData.getGeometryType(req, this); }, function finishSetup(err, data){ + if ( err ) { callback(err, req); return; } + if (!_.isNull(data)) _.extend(req.params, {geom_type: data}); - callback(err, req); + that.addCacheChannel(req, function(err, chan) { + callback(err, req); + }); } ); }; diff --git a/test/acceptance/server.js b/test/acceptance/server.js index b61c4eb6..b6806cf4 100644 --- a/test/acceptance/server.js +++ b/test/acceptance/server.js @@ -40,6 +40,7 @@ suite('server', function() { method: 'GET' },{ status: 200, + headers: { 'X-Cache-Channel': 'cartodb_test_user_1_db:my_table' }, body: '{"style":"#my_table {marker-fill: #FF6600;marker-opacity: 1;marker-width: 8;marker-line-color: white;marker-line-width: 3;marker-line-opacity: 0.9;marker-placement: point;marker-type: ellipse;marker-allow-overlap: true;}"}' }, function() { done(); }); }); @@ -261,6 +262,7 @@ suite('server', function() { method: 'GET' },{ status: 200, + headers: { 'X-Cache-Channel': 'cartodb_test_user_1_db:my_tablez' }, body: '{"infowindow":null}' }, function() { done(); }); }); @@ -326,7 +328,8 @@ suite('server', function() { method: 'GET' },{ status: 200, - headers: { 'Content-Type': 'text/javascript; charset=utf-8; charset=utf-8' } + headers: { 'Content-Type': 'text/javascript; charset=utf-8; charset=utf-8', + 'X-Cache-Channel': 'cartodb_test_user_1_db:gadm4' } }, function() { done(); }); }); @@ -391,7 +394,7 @@ suite('server', function() { method: 'GET' },{ status: 200, - headers: { 'Content-Type': 'image/png' } + headers: { 'Content-Type': 'image/png', 'X-Cache-Channel': 'cartodb_test_user_1_db:gadm4' } }, function() { done(); }); }); From 4d524d88d20c8f6278b6340c6e6152ece54fd602 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Tue, 25 Sep 2012 10:18:47 +0200 Subject: [PATCH 8/9] Reduce GET style error verbosity --- package.json | 2 +- test/acceptance/server.js | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index cd389792..8e8d183c 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ "node-varnish": "0.1.1", "underscore" : "1.1.x", "grainstore" : "~0.6.2", - "windshaft" : "~0.4.14", + "windshaft" : "~0.4.16", "step": "0.0.x", "generic-pool": "1.0.x", "redis": "0.7.2", diff --git a/test/acceptance/server.js b/test/acceptance/server.js index b6806cf4..96d819d6 100644 --- a/test/acceptance/server.js +++ b/test/acceptance/server.js @@ -56,6 +56,8 @@ suite('server', function() { }, function(res) { // FIXME: should be 401 Unauthorized assert.equal(res.statusCode, 500, res.body); + assert.deepEqual(JSON.parse(res.body), + {error: 'Sorry, you are unauthorized (permission denied)'}); done(); }); }); From d7839799cee20ca69438972df74e34196334a31b Mon Sep 17 00:00:00 2001 From: Luis Bosque Date: Tue, 25 Sep 2012 13:46:41 +0200 Subject: [PATCH 9/9] added NEWS.md for 0.9.0 version --- NEWS.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 NEWS.md diff --git a/NEWS.md b/NEWS.md new file mode 100644 index 00000000..49a49b51 --- /dev/null +++ b/NEWS.md @@ -0,0 +1,5 @@ +0.9.0 (25/09/12) +----- +* External resources in CartoCSS +* Added X-Cache-Channel header in all the tiler GET requests +* Small fixes