diff --git a/NEWS.md b/NEWS.md index 54f8b731..06f90363 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,24 @@ -1.1.0 (30/10/12) +1.1.2 (DD//MM//YY) ----- +* CartoCSS versioning + * Fix use of "style_version" with GET (inline styles) + * Enhance 2.0 -> 2.1 transforms: + * styles with no semicolon + * markers shift due to geometry clipping + + +1.1.1 (DD//MM//YY) +----- +* Add support for persistent client cache headers +* Fix crash on unknown user (#55) +* Add /version entry point +* CartoCSS versioning + * Include style_version in GET /style response + * Support style_version and style_convert parameters in POST /style request + * Support style_version in GET /:z/:x/:y request + +1.1.0 (30/10/12) +======= * Add /version entry point * CartoCSS versioning * Include version in GET /style response diff --git a/README.md b/README.md index 47a48866..e1f66eeb 100644 --- a/README.md +++ b/README.md @@ -80,8 +80,16 @@ Args: * sql - plain SQL arguments * interactivity - specify the column to use in UTFGrid -* cache_buster - if needed you can add a cachebuster to make sure you're - rendering new +* cache_buster - Specify an identifier for the internal tile cache. + Requesting tiles with the same cache_buster value may + result in being served a cached version of the tile + (even when requesting a tile for the first time, as tiles + can be prepared in advance) +* cache_policy - Set to "persist" to have the server send an Cache-Control + header requesting caching devices to keep the response + cached as much as possible. This is best used with a + timestamp value in cache_buster for manual control of + updates. * geom_type - override the cartodb default * style - override the default map style with Carto diff --git a/cluster.js b/cluster.js index fc342a34..3d426735 100755 --- a/cluster.js +++ b/cluster.js @@ -44,6 +44,7 @@ var cluster = new Cluster({ host: global.environment.host, monPort: global.environment.port+1, monHost: global.environment.host, + timeout: 600000, noWorkers: 1 }); diff --git a/config/environments/test.js.example b/config/environments/test.js.example index a021f25b..908d39a6 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -10,7 +10,6 @@ var config = { user: "publicuser", host: '127.0.0.1', port: 5432, - srid: 4326, extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188", simplify: true } diff --git a/lib/cartodb/carto_data.js b/lib/cartodb/carto_data.js index 87a8bf8c..5ad4723c 100644 --- a/lib/cartodb/carto_data.js +++ b/lib/cartodb/carto_data.js @@ -126,7 +126,7 @@ module.exports = function() { that.retrieve(that.table_metadata_db, redisKey, 'privacy', this); }, function(err, data){ - if (err) throw err; + //if (err) throw err; callback(err, data); } ); diff --git a/lib/cartodb/cartodb_windshaft.js b/lib/cartodb/cartodb_windshaft.js index 86854861..ded52079 100644 --- a/lib/cartodb/cartodb_windshaft.js +++ b/lib/cartodb/cartodb_windshaft.js @@ -26,6 +26,14 @@ var CartodbWindshaft = function(serverOptions) { // boot var ws = new Windshaft.Server(serverOptions); + // Override getVersion to include cartodb-specific versions + var wsversion = ws.getVersion; + ws.getVersion = function() { + var version = wsversion(); + version.windshaft_cartodb = require('../../package.json').version; + return version; + } + /** * Helper to allow access to the layer to be used in the maps infowindow popup. */ diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index b0ce3ba8..7550d871 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -45,8 +45,13 @@ module.exports = function(){ var ttl = global.environment.varnish.ttl || 86400; Cache.generateCacheChannel(req, function(channel){ res.header('X-Cache-Channel', channel); - res.header('Last-Modified', new Date().toUTCString()); - res.header('Cache-Control', 'no-cache,max-age='+ttl+',must-revalidate, public'); + var cache_policy = req.query.cache_policy; + if ( cache_policy == 'persist' ) { + res.header('Cache-Control', 'public,max-age=31536000'); // 1 year + } else { + res.header('Last-Modified', new Date().toUTCString()); + res.header('Cache-Control', 'no-cache,max-age='+ttl+',must-revalidate, public'); + } cb(null, channel); // add last-modified too ? }); } @@ -60,7 +65,7 @@ module.exports = function(){ me.req2params = function(req, callback){ // Whitelist query parameters and attach format - var good_query = ['sql', 'geom_type', 'cache_buster','callback', 'interactivity', 'map_key', 'api_key', 'style']; + var good_query = ['sql', 'geom_type', 'cache_buster', 'cache_policy', 'callback', 'interactivity', 'map_key', 'api_key', 'style', 'style_version']; var bad_query = _.difference(_.keys(req.query), good_query); _.each(bad_query, function(key){ delete req.query[key]; }); diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index d49f6031..130d34e2 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -32,13 +32,13 @@ "version": "0.8.3" }, "npm": { - "version": "1.1.62", + "version": "1.1.65", "dependencies": { "semver": { - "version": "1.0.14" + "version": "1.1.0" }, "ini": { - "version": "1.0.4" + "version": "1.0.5" }, "slide": { "version": "1.1.3" @@ -50,7 +50,7 @@ "version": "1.1.14" }, "minimatch": { - "version": "0.2.6" + "version": "0.2.8" }, "nopt": { "version": "2.0.0" @@ -93,10 +93,10 @@ "version": "2.0.4" }, "node-gyp": { - "version": "0.6.11" + "version": "0.7.1" }, "fstream-npm": { - "version": "0.1.2", + "version": "0.1.3", "dependencies": { "fstream-ignore": { "version": "0.0.5" @@ -119,19 +119,24 @@ "version": "0.1.2" }, "npm-registry-client": { - "version": "0.2.7" + "version": "0.2.10", + "dependencies": { + "couch-login": { + "version": "0.1.15" + } + } }, "read-package-json": { - "version": "0.1.5" + "version": "0.1.8" }, "read-installed": { - "version": "0.0.2" + "version": "0.0.3" }, "glob": { - "version": "3.1.12" + "version": "3.1.14" }, "init-package-json": { - "version": "0.0.5", + "version": "0.0.6", "dependencies": { "promzard": { "version": "0.2.0" @@ -147,9 +152,6 @@ "retry": { "version": "0.6.0" }, - "couch-login": { - "version": "0.1.12" - }, "once": { "version": "1.1.1" }, @@ -180,10 +182,14 @@ "version": "1.3.3" }, "grainstore": { - "version": "0.9.1", + "version": "0.9.6", "dependencies": { + "semver": { + "version": "1.1.0" + }, "carto": { - "version": "0.9.2", + "version": "0.9.3-cdb1", + "from": "git://github.com/CartoDB/carto.git#cdb-0.9", "dependencies": { "mapnik-reference": { "version": "5.0.0" @@ -199,7 +205,7 @@ } }, "millstone": { - "version": "0.5.10", + "version": "0.5.11", "dependencies": { "request": { "version": "2.11.4", @@ -226,7 +232,7 @@ } }, "srs": { - "version": "0.2.16" + "version": "0.2.17" }, "zipfile": { "version": "0.3.2" @@ -245,7 +251,7 @@ } }, "windshaft": { - "version": "0.6.2", + "version": "0.7.1", "dependencies": { "express": { "version": "2.5.11", @@ -315,7 +321,7 @@ "version": "2.9.202" }, "mapnik": { - "version": "0.7.14" + "version": "0.7.16" }, "mocha": { "version": "1.2.1", diff --git a/package.json b/package.json index c9cd7025..a6c1d7ae 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "1.1.0", + "version": "1.1.2", "description": "A map tile server for CartoDB", "url": "https://github.com/Vizzuality/Windshaft-cartodb", "licenses": [{ @@ -21,8 +21,8 @@ "cluster2": "git://github.com/CartoDB/cluster2.git#cdb_production", "node-varnish": "0.1.1", "underscore" : "~1.3.3", - "grainstore" : "~0.9.1", - "windshaft" : "~0.6.2", + "grainstore" : "~0.9.6", + "windshaft" : "~0.7.0", "step": "0.0.x", "generic-pool": "1.0.x", "redis": "0.7.2", diff --git a/test/acceptance/server.js b/test/acceptance/server.js index dcf590ff..a31eaa46 100644 --- a/test/acceptance/server.js +++ b/test/acceptance/server.js @@ -33,6 +33,35 @@ suite('server', function() { }, function() { done(); }); }); + ///////////////////////////////////////////////////////////////////////////////// + // + // GET VERSION + // + ///////////////////////////////////////////////////////////////////////////////// + + test("get call to server returns 200", function(done){ + assert.response(server, { + url: '/version', + method: 'GET' + },{ + status: 200 + }, function(res) { + var parsed = JSON.parse(res.body); + assert.ok(parsed.hasOwnProperty('windshaft_cartodb'), "No 'windshaft_cartodb' version in " + parsed); + console.log("Windshaft-cartodb: " + parsed.windshaft_cartodb); + assert.ok(parsed.hasOwnProperty('windshaft'), "No 'windshaft' version in " + parsed); + console.log("Windshaft: " + parsed.windshaft); + assert.ok(parsed.hasOwnProperty('grainstore'), "No 'grainstore' version in " + parsed); + console.log("Grainstore: " + parsed.grainstore); + assert.ok(parsed.hasOwnProperty('node_mapnik'), "No 'node_mapnik' version in " + parsed); + console.log("Node-mapnik: " + parsed.node_mapnik); + assert.ok(parsed.hasOwnProperty('mapnik'), "No 'mapnik' version in " + parsed); + console.log("Mapnik: " + parsed.mapnik); + // TODO: check actual versions ? + done(); + }); + }); + ///////////////////////////////////////////////////////////////////////////////// // // GET STYLE @@ -50,7 +79,7 @@ suite('server', function() { }, function(res) { var parsed = JSON.parse(res.body); assert.equal(parsed.style, "#my_table {marker-fill: #FF6600;marker-opacity: 1;marker-width: 8;marker-line-color: white;marker-line-width: 3;marker-line-opacity: 0.9;marker-placement: point;marker-type: ellipse;marker-allow-overlap: true;}"); - assert.equal(parsed.version, '2.0.0'); + assert.equal(parsed.style_version, '2.0.0'); done(); }); }); @@ -72,6 +101,23 @@ suite('server', function() { }); }); + // See http://github.com/Vizzuality/Windshaft-cartodb/issues/55 + test("get'ing style of private table should fail on unknown username", + function(done) { + assert.response(server, { + headers: {host: 'unknown_user'}, + url: '/tiles/test_table_private_1/style', + method: 'GET' + },{ + }, function(res) { + // FIXME: should be 401 Unauthorized + assert.equal(res.statusCode, 500, res.body); + assert.deepEqual(JSON.parse(res.body), + {error:"missing unknown_user's dbname in redis (try CARTODB/script/restore_redis)"}); + done(); + }); + }); + test("get'ing style of private table should succeed when authenticated", function(done) { assert.response(server, { @@ -83,7 +129,7 @@ suite('server', function() { assert.equal(res.statusCode, 200, res.body); var parsed = JSON.parse(res.body); assert.equal(parsed.style, "#test_table_private_1 {marker-fill: #FF6600;marker-opacity: 1;marker-width: 8;marker-line-color: white;marker-line-width: 3;marker-line-opacity: 0.9;marker-placement: point;marker-type: ellipse;marker-allow-overlap: true;}"); - assert.equal(parsed.version, '2.0.0'); + assert.equal(parsed.style_version, '2.0.0'); done(); }); }); @@ -201,7 +247,7 @@ suite('server', function() { }, function(res) { var parsed = JSON.parse(res.body); assert.equal(parsed.style, 'Map {background-color:#fff;}'); - //assert.equal(parsed.version, '2.0.0'); + assert.equal(parsed.style_version, '2.0.0'); done(); }); @@ -215,7 +261,7 @@ suite('server', function() { url: '/tiles/my_table5/style?map_key=1234', method: 'POST', headers: {host: 'localhost', 'Content-Type': 'application/x-www-form-urlencoded' }, - data: querystring.stringify({style: style}) + data: querystring.stringify({style: style, style_version: '2.0.2'}) },{ }, function(res) { @@ -230,7 +276,7 @@ suite('server', function() { }, function(res) { var parsed = JSON.parse(res.body); assert.equal(parsed.style, style); - //assert.equal(parsed.version, '2.0.0'); + assert.equal(parsed.style_version, '2.0.2'); done(); }); @@ -372,6 +418,23 @@ suite('server', function() { }); }); + // See http://github.com/Vizzuality/Windshaft-cartodb/issues/55 + test("get'ing infowindow of private table should fail on unknown username", + function(done) { + assert.response(server, { + headers: {host: 'unknown_user'}, + url: '/tiles/test_table_private_1/infowindow', + method: 'GET' + },{ + }, function(res) { + // FIXME: should be 401 Unauthorized + assert.equal(res.statusCode, 500, res.body); + assert.deepEqual(JSON.parse(res.body), + {error:"missing unknown_user's dbname in redis (try CARTODB/script/restore_redis)"}); + done(); + }); + }); + test("get'ing infowindow of private table should succeed when authenticated", function(done) { assert.response(server, { @@ -438,6 +501,23 @@ suite('server', function() { }); }); + // See http://github.com/Vizzuality/Windshaft-cartodb/issues/55 + test("get'ing grid of private table should fail on unknown username", + function(done) { + assert.response(server, { + headers: {host: 'unknown_user'}, + url: '/tiles/test_table_private_1/6/31/24.grid.json', + method: 'GET' + },{ + }, function(res) { + // FIXME: should be 401 Unauthorized + assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body); + assert.deepEqual(JSON.parse(res.body), + {error:"missing unknown_user's dbname in redis (try CARTODB/script/restore_redis)"}); + done(); + }); + }); + test("get'ing the grid of a private table should succeed when authenticated", function(done) { assert.response(server, { @@ -455,7 +535,40 @@ suite('server', function() { // GET TILE // ///////////////////////////////////////////////////////////////////////////////// - + + test("should send Cache-Control header with short expiration by default", function(done){ + assert.response(server, { + headers: {host: 'localhost'}, + url: '/tiles/gadm4/6/31/24.png', + method: 'GET' + },{ + status: 200, + }, function(res) { + var cc = res.headers['cache-control']; + assert.ok(cc); + //assert.equal(cc, 'public,max-age=31536000'); // 1 year + assert.ok(cc.match('no-cache'), cc); + assert.ok(cc.match('must-revalidate'), cc); + assert.ok(cc.match('public'), cc); + done(); + }); + }); + + test("should send Cache-Control header with long expiration when requested", function(done){ + assert.response(server, { + headers: {host: 'localhost'}, + url: '/tiles/gadm4/6/31/24.png?cache_policy=persist', + method: 'GET' + },{ + status: 200, + }, function(res) { + var cc = res.headers['cache-control']; + assert.ok(cc); + assert.equal(cc, 'public,max-age=31536000'); // 1 year + done(); + }); + }); + test("get'ing a tile with default style should return an image", function(done){ assert.response(server, { headers: {host: 'localhost'}, @@ -537,6 +650,25 @@ suite('server', function() { }); }); + test("get'ing a tile with data from private table should fail on unknown username", function(done){ + var sql = querystring.stringify({ + sql: "SELECT * FROM test_table_private_1", + cache_buster:2 // this is to avoid getting the cached response + }); + assert.response(server, { + headers: {host: 'unknown_user'}, + url: '/tiles/gadm4/6/31/24.png?' + sql, + method: 'GET' + },{ + }, function(res) { + // FIXME: should be 401 Unauthorized + assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body); + assert.deepEqual(JSON.parse(res.body), + {error:"missing unknown_user's dbname in redis (try CARTODB/script/restore_redis)"}); + done(); + }); + }); + test("get'ing a tile with data from private table should fail when unauthenticated (uses old redis key)", function(done){ var sql = querystring.stringify({ sql: "SELECT * FROM test_table_private_1", @@ -557,6 +689,47 @@ suite('server', function() { }); }); + var test_style_black_200 = "#test_table{marker-fill:black;marker-line-color:red;marker-width:10}"; + var test_style_black_210 = "#test_table{marker-fill:black;marker-line-color:red;marker-width:20}"; + + test("get'ing a tile with url specified 2.0.0 style should return an expected tile", function(done){ + var style = querystring.stringify({style: test_style_black_200, style_version: '2.0.0'}); + assert.response(server, { + headers: {host: 'localhost'}, + url: '/tiles/test_table/15/16046/12354.png?cache_buster=4&' + style, // madrid + method: 'GET', + encoding: 'binary' + },{}, function(res){ + assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); + var ct = res.headers['content-type']; + assert.equal(ct, 'image/png'); + assert.imageEqualsFile(res.body, './test/fixtures/test_table_15_16046_12354_styled_black.png', 2, + function(err, similarity) { + if (err) throw err; + done(); + }); + }); + }); + + test("get'ing a tile with url specified 2.1.0 style should return an expected tile", function(done){ + var style = querystring.stringify({style: test_style_black_210, style_version: '2.1.0'}); + assert.response(server, { + headers: {host: 'localhost'}, + url: '/tiles/test_table/15/16046/12354.png?cache_buster=4&' + style, // madrid + method: 'GET', + encoding: 'binary' + },{}, function(res){ + assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); + var ct = res.headers['content-type']; + assert.equal(ct, 'image/png'); + assert.imageEqualsFile(res.body, './test/fixtures/test_table_15_16046_12354_styled_black.png', 2, + function(err, similarity) { + if (err) throw err; + done(); + }); + }); + }); + ///////////////////////////////////////////////////////////////////////////////// // // DELETE CACHE diff --git a/test/support/assert.js b/test/support/assert.js index bd1c45ce..82fcf99d 100644 --- a/test/support/assert.js +++ b/test/support/assert.js @@ -8,29 +8,35 @@ var exec = require('child_process').exec; var assert = module.exports = exports = require('assert'); -assert.imageEqualsFile = function(buffer, file_b, callback) { +// +// @param tol tolerated color distance as a percent over max channel value +// by default this is zero. For meaningful values, see +// http://www.imagemagick.org/script/command-line-options.php#metric +// +assert.imageEqualsFile = function(buffer, file_b, tol, callback) { if (!callback) callback = function(err) { if (err) throw err; }; file_b = path.resolve(file_b); - var file_a = '/tmp/' + (Math.random() * 1e16); + var file_a = '/tmp/windshaft-test-image-test.png'; // + (Math.random() * 1e16); // TODO: make predictable var err = fs.writeFileSync(file_a, buffer, 'binary'); if (err) throw err; - exec('compare -metric PSNR "' + file_a + '" "' + + var fuzz = tol + '%'; + exec('compare -fuzz ' + fuzz + ' -metric AE "' + file_a + '" "' + file_b + '" /dev/null', function(err, stdout, stderr) { if (err) { fs.unlinkSync(file_a); callback(err); } else { stderr = stderr.trim(); - if (stderr === 'inf') { - fs.unlinkSync(file_a); - callback(null); + var similarity = parseFloat(stderr); + if ( similarity > 0 ) { + var err = new Error('Images not equal(' + similarity + '): ' + + file_a + ' ' + file_b); + err.similarity = similarity; + callback(err); } else { - var similarity = parseFloat(stderr); - var err = new Error('Images not equal(' + similarity + '): ' + - file_a + ' ' + file_b); - err.similarity = similarity; - callback(err); + fs.unlinkSync(file_a); + callback(null); } } }); diff --git a/tools/show_style b/tools/show_style new file mode 100755 index 00000000..d3b9163b --- /dev/null +++ b/tools/show_style @@ -0,0 +1,17 @@ +#!/bin/sh + +# TODO: port to node, if you really need it + +REDIS_PORT=6379 # default port + + +if test -z "$2"; then + echo "Usage: $0 " >&2 + exit 1 +fi + +username="$1" +tabname="$2" + +dbname=`redis-cli -p ${REDIS_PORT} -n 5 hget "rails:users:${username}" "database_name"` +redis-cli get "map_style|${dbname}|${tabname}" | sed -e 's/\\n/\n/g' -e 's/\\//g'