diff --git a/NEWS.md b/NEWS.md new file mode 100644 index 00000000..49a49b51 --- /dev/null +++ b/NEWS.md @@ -0,0 +1,5 @@ +0.9.0 (25/09/12) +----- +* External resources in CartoCSS +* Added X-Cache-Channel header in all the tiler GET requests +* Small fixes diff --git a/config/environments/development.js b/config/environments/development.js index e141bfda..9ed0731f 100644 --- a/config/environments/development.js +++ b/config/environments/development.js @@ -18,6 +18,9 @@ var config = { */ simplify: true } + ,millstone: { + cache_basedir: '/tmp/cdb-tiler-dev/millstone-dev' + } ,redis: { host: '127.0.0.1', port: 6379, diff --git a/config/environments/production.js b/config/environments/production.js index 28365971..15a3c9bc 100644 --- a/config/environments/production.js +++ b/config/environments/production.js @@ -12,6 +12,9 @@ var config = { extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188", simplify: true } + ,millstone: { + cache_basedir: '/home/ubuntu/tile_assets/' + } ,redis: { host: '127.0.0.1', port: 6379 diff --git a/config/environments/staging.js b/config/environments/staging.js index 87c0e1c6..c96bb78a 100644 --- a/config/environments/staging.js +++ b/config/environments/staging.js @@ -12,6 +12,9 @@ var config = { extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188", simplify: true } + ,millstone: { + cache_basedir: '/home/ubuntu/tile_assets/' + } ,redis: { host: '127.0.0.1', port: 6379 diff --git a/config/environments/test.js b/config/environments/test.js index d7ee5739..3b8f27f3 100644 --- a/config/environments/test.js +++ b/config/environments/test.js @@ -13,6 +13,9 @@ var config = { extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188", simplify: true } + ,millstone: { + cache_basedir: '/tmp/cdb-tiler-test/millstone' + } ,redis: { host: '127.0.0.1', port: 6333, diff --git a/lib/cartodb/cartodb_windshaft.js b/lib/cartodb/cartodb_windshaft.js index 52655dc6..69687eff 100644 --- a/lib/cartodb/cartodb_windshaft.js +++ b/lib/cartodb/cartodb_windshaft.js @@ -6,17 +6,6 @@ var _ = require('underscore') var CartodbWindshaft = function(serverOptions) { - // set the cache chanel info to invalidate the cache on the frontend server - serverOptions.afterTileRender = function(req, res, tile, headers, callback) { - var ttl = global.environment.varnish.ttl || 86400; - Cache.generateCacheChannel(req, function(channel){ - res.header('X-Cache-Channel', channel); - res.header('Last-Modified', new Date().toUTCString()); - res.header('Cache-Control', 'no-cache,max-age='+ttl+',must-revalidate, public'); - callback(null, tile, headers); - }); - }; - if(serverOptions.cache_enabled) { console.log("cache invalidation enabled, varnish on ", serverOptions.varnish_host, ' ', serverOptions.varnish_port); Cache.init(serverOptions.varnish_host, serverOptions.varnish_port); diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index 81433005..183d7cae 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -1,11 +1,15 @@ var _ = require('underscore') , Step = require('step') - , cartoData = require('./carto_data'); + , cartoData = require('./carto_data') + , Cache = require('./cache_validator'); module.exports = function(){ var me = { base_url: '/tiles/:table', - grainstore: {datasource: global.environment.postgres}, + grainstore: { + datasource: global.environment.postgres, + cachedir: global.environment.millstone.cache_basedir + }, redis: global.environment.redis, enable_cors: global.environment.enable_cors, varnish_host: global.environment.varnish.host, @@ -14,6 +18,28 @@ module.exports = function(){ log_format: '[:date] :req[X-Real-IP] \033[90m:method\033[0m \033[36m:req[Host]:url\033[0m \033[90m:status :response-time ms -> :res[Content-Type]\033[0m' }; + // Set the cache chanel info to invalidate the cache on the frontend server + // + // @param req The request object. + // The function will have no effect unless req.res exists. + // It is expected that req.params contains 'table' and 'dbname' + // + // @param cb function(err, channel) will be called when ready. + // the channel parameter will be null if nothing was added + // + me.addCacheChannel = function(req, cb) { + // skip non-GET requests, or requests for which there's no response + if ( req.method != 'GET' || ! req.res ) { cb(null, null); return; } + var res = req.res; + var ttl = global.environment.varnish.ttl || 86400; + Cache.generateCacheChannel(req, function(channel){ + res.header('X-Cache-Channel', channel); + res.header('Last-Modified', new Date().toUTCString()); + res.header('Cache-Control', 'no-cache,max-age='+ttl+',must-revalidate, public'); + cb(null, channel); // add last-modified too ? + }); + } + /** * Whitelist input and get database name & default geometry type from * subdomain/user metadata held in CartoDB Redis @@ -42,6 +68,8 @@ module.exports = function(){ callback(null, xml); } + var that = this; + Step( function getPrivacy(){ cartoData.authorize(req, this); @@ -63,10 +91,14 @@ module.exports = function(){ cartoData.getGeometryType(req, this); }, function finishSetup(err, data){ + if ( err ) { callback(err, req); return; } + if (!_.isNull(data)) _.extend(req.params, {geom_type: data}); - callback(err, req); + that.addCacheChannel(req, function(err, chan) { + callback(err, req); + }); } ); }; diff --git a/package.json b/package.json index 97b35645..8e8d183c 100644 --- a/package.json +++ b/package.json @@ -22,12 +22,12 @@ "cluster": "0.6.4", "node-varnish": "0.1.1", "underscore" : "1.1.x", - "grainstore" : "~0.5.0", - "windshaft" : "~0.4.13", + "grainstore" : "~0.6.2", + "windshaft" : "~0.4.16", "step": "0.0.x", "generic-pool": "1.0.x", - "redis": "0.6.7", - "hiredis": "0.1.12", + "redis": "0.7.2", + "hiredis": "~0.1.12", "request": "2.9.202" }, "devDependencies": { diff --git a/test/acceptance/server.js b/test/acceptance/server.js index fd294a3e..96d819d6 100644 --- a/test/acceptance/server.js +++ b/test/acceptance/server.js @@ -40,6 +40,7 @@ suite('server', function() { method: 'GET' },{ status: 200, + headers: { 'X-Cache-Channel': 'cartodb_test_user_1_db:my_table' }, body: '{"style":"#my_table {marker-fill: #FF6600;marker-opacity: 1;marker-width: 8;marker-line-color: white;marker-line-width: 3;marker-line-opacity: 0.9;marker-placement: point;marker-type: ellipse;marker-allow-overlap: true;}"}' }, function() { done(); }); }); @@ -55,6 +56,8 @@ suite('server', function() { }, function(res) { // FIXME: should be 401 Unauthorized assert.equal(res.statusCode, 500, res.body); + assert.deepEqual(JSON.parse(res.body), + {error: 'Sorry, you are unauthorized (permission denied)'}); done(); }); }); @@ -129,24 +132,33 @@ suite('server', function() { test("post'ing good style with no authentication returns an error", function(done){ assert.response(server, { - url: '/tiles/my_table5/style', + url: '/tiles/my_table5/style?map_key=1234', method: 'POST', headers: {host: 'vizzuality.localhost.lan', 'Content-Type': 'application/x-www-form-urlencoded' }, - data: querystring.stringify({style: 'Map {background-color:#aaa;}'}) - },{}, function(res) { - // FIXME: should be 401 Unauthorized - assert.equal(res.statusCode, 500, res.body); - assert.ok(res.body.indexOf('map state cannot be changed by unauthenticated request') != -1, res.body); - + data: querystring.stringify({style: 'Map {background-color:#fff;}'}) + },{ + }, function(res) { + assert.equal(res.statusCode, 200, res.body); assert.response(server, { - headers: {host: 'vizzuality.localhost.lan'}, url: '/tiles/my_table5/style', - method: 'GET' - },{ - status: 200, - body: JSON.stringify({style: 'Map {background-color:#fff;}'}) - }, function() { done(); }); + method: 'POST', + headers: {host: 'vizzuality.localhost.lan', 'Content-Type': 'application/x-www-form-urlencoded' }, + data: querystring.stringify({style: 'Map {background-color:#aaa;}'}) + },{}, function(res) { + // FIXME: should be 401 Unauthorized + assert.equal(res.statusCode, 500, res.body); + assert.ok(res.body.indexOf('map state cannot be changed by unauthenticated request') != -1, res.body); + assert.response(server, { + headers: {host: 'vizzuality.localhost.lan'}, + url: '/tiles/my_table5/style', + method: 'GET' + },{ + status: 200, + body: JSON.stringify({style: 'Map {background-color:#fff;}'}) + }, function() { done(); }); + + }); }); }); @@ -216,7 +228,7 @@ suite('server', function() { // Retrive style with authenticated request assert.response(server, { - headers: {host: 'localhost'}, + headers: {host: 'vizzuality'}, url: '/tiles/my_table5/style?map_key=1234', method: 'GET' },{}, function(res) { @@ -225,7 +237,7 @@ suite('server', function() { // Now retrive style with unauthenticated request assert.response(server, { - headers: {host: 'localhost'}, + headers: {host: 'vizzuality'}, url: '/tiles/my_table5/style', method: 'GET' }, {}, function(res) { @@ -252,6 +264,7 @@ suite('server', function() { method: 'GET' },{ status: 200, + headers: { 'X-Cache-Channel': 'cartodb_test_user_1_db:my_tablez' }, body: '{"infowindow":null}' }, function() { done(); }); }); @@ -317,7 +330,8 @@ suite('server', function() { method: 'GET' },{ status: 200, - headers: { 'Content-Type': 'text/javascript; charset=utf-8; charset=utf-8' } + headers: { 'Content-Type': 'text/javascript; charset=utf-8; charset=utf-8', + 'X-Cache-Channel': 'cartodb_test_user_1_db:gadm4' } }, function() { done(); }); }); @@ -382,7 +396,7 @@ suite('server', function() { method: 'GET' },{ status: 200, - headers: { 'Content-Type': 'image/png' } + headers: { 'Content-Type': 'image/png', 'X-Cache-Channel': 'cartodb_test_user_1_db:gadm4' } }, function() { done(); }); }); diff --git a/tools/reset_styles b/tools/reset_styles new file mode 100755 index 00000000..53b736dc --- /dev/null +++ b/tools/reset_styles @@ -0,0 +1,35 @@ +#!/usr/bin/env node + +// Reset redis-stored XML styles so that they are regenerated +// from CartoCSS on next tile request + +var redis = require('redis') + +var REDIS_PORT = 6379; // TODO: make a parameter + +var dbnum = 0; + +var client = redis.createClient(REDIS_PORT, 'localhost'); +client.on('connect', function() { + client.select(dbnum); + client.keys('map_style|*', function(err, matches) { + + processNext = function() { + if ( ! matches.length ) process.exit(0); + var k = matches.shift(); + console.log("Resetting XML in key: " + k); + client.get(k, function(err, val) { + if ( err ) throw err; + val = JSON.parse(val); + delete val.xml; + client.set(k, JSON.stringify(val), function() { + console.log("done with style " + k); + processNext(); + }); + }); + } + + processNext(); + + }); +});