diff --git a/NEWS.md b/NEWS.md index ed0aff1e..8af5b674 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,13 +1,23 @@ -1.1.10 +1.2.1 ------ +1.2.0 +------ +* Multilayer API changes + * Layers passed by index in grid fetching url + * Interactivity only specified in layergroup config + * Embed cache_buster within token + * Use ISO format for last_modified timestamp +* Expected LZMA encoding changed to base64 + +1.1.10 +------ * Fix regression with default interactivity parameter (#74) * More verbose logging for SQL api connection errors * Write stats for multilayer map token request 1.1.9 ----- - * Handle SQL API errors by requesting no Varnish cache * Fix X-Cache-Channel for multilayer (by token) responses * Add last_modified field to layergroup creation response (#72) diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index cc4c1f7e..c67b134a 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -105,7 +105,14 @@ module.exports = function(){ me.affectedTables = function (username, api_key, sql, callback) { - var sql = 'SELECT CDB_QueryTables($windshaft$' + sql + '$windshaft$)'; + // Replace mapnik tokens + sql = sql.replace(RegExp('!bbox!', 'g'), 'ST_MakeEnvelope(0,0,0,0)') + .replace(RegExp('!pixel_width!', 'g'), '1') + .replace(RegExp('!pixel_height!', 'g'), '1') + ; + + // Pass to CDB_QueryTables + sql = 'SELECT CDB_QueryTables($windshaft$' + sql + '$windshaft$)'; // call sql api me.sqlQuery(username, api_key, sql, function(err, rows){ @@ -263,7 +270,8 @@ module.exports = function(){ // find last updated me.findLastUpdated(usr, key, tableNames, function(err, lastUpdated) { if ( err ) { done(err); return; } - response.last_updated = lastUpdated; + response.layergroupid = response.layergroupid + ':' + lastUpdated; // use epoch + response.last_updated = new Date(lastUpdated).toISOString(); // TODO: use ISO format done(null); }); }); @@ -283,26 +291,17 @@ module.exports = function(){ // TODO: check ? //console.log("type of req.query.lzma is " + typeof(req.query.lzma)); - //console.log("req.query.lzma is " + req.query.lzma); - // Decode - var lzma = []; - for (var i=0; i 127 ) num = 127-num; - //console.log(i + " hex: " + hex + " decodes as " + num); - lzma.push( num ); - } + // Decode (from base64) + var lzma = (new Buffer(req.query.lzma, 'base64').toString('binary')).split('').map(function(c) { return c.charCodeAt(0) - 128 }) // Decompress - //console.log("LZMA decompression starts with " + lzma); LZMA.decompress( lzma, function(result) { - //console.log("LZMA decompression completed, payload: "); console.dir(result); try { - req.query = JSON.parse(result); + delete req.query.lzma + _.extend(req.query, JSON.parse(result)) me.req2params(req, callback); } catch (err) { callback(new Error('Error parsing lzma as JSON: ' + err)); @@ -316,12 +315,19 @@ module.exports = function(){ } // Whitelist query parameters and attach format - var good_query = ['sql', 'geom_type', 'cache_buster', 'cache_policy', 'callback', 'interactivity', 'map_key', 'api_key', 'style', 'style_version', 'style_convert' ]; + var good_query = ['sql', 'geom_type', 'cache_buster', 'cache_policy', 'callback', 'interactivity', 'map_key', 'api_key', 'style', 'style_version', 'style_convert', 'config' ]; var bad_query = _.difference(_.keys(req.query), good_query); _.each(bad_query, function(key){ delete req.query[key]; }); req.params = _.extend({}, req.params); // shuffle things as request is a strange array/object + if ( req.params.token ) { + //console.log("Request parameters include token " + req.params.token); + var tksplit = req.params.token.split(':'); + req.params.token = tksplit[0]; + if ( tksplit.length > 1 ) req.params.cache_buster= tksplit[1]; + } + // bring all query values onto req.params object _.extend(req.params, req.query); diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index b6bca7a0..b9dec20d 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -193,10 +193,10 @@ "version": "1.3.3" }, "windshaft": { - "version": "0.11.2", + "version": "0.12.0", "dependencies": { "grainstore": { - "version": "0.12.0", + "version": "0.13.0", "dependencies": { "carto": { "version": "0.9.3-cdb3", @@ -288,7 +288,7 @@ } }, "tilelive": { - "version": "4.3.3", + "version": "4.4.2", "dependencies": { "optimist": { "version": "0.3.7", @@ -307,9 +307,6 @@ "version": "0.5.0", "from": "git://github.com/Vizzuality/tilelive-mapnik.git#6a360ee50", "dependencies": { - "generic-pool": { - "version": "2.0.3" - }, "eio": { "version": "0.1.0" }, @@ -327,7 +324,7 @@ "version": "0.0.5" }, "generic-pool": { - "version": "1.0.12" + "version": "2.0.3" }, "redis": { "version": "0.7.2" diff --git a/package.json b/package.json index 931ce277..7a3c888b 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "1.1.10", + "version": "1.2.1", "description": "A map tile server for CartoDB", "url": "https://github.com/Vizzuality/Windshaft-cartodb", "licenses": [{ @@ -21,9 +21,9 @@ "cluster2": "git://github.com/CartoDB/cluster2.git#cdb_production", "node-varnish": "0.1.1", "underscore" : "~1.3.3", - "windshaft" : "~0.11.2", + "windshaft" : "~0.12.0", "step": "0.0.x", - "generic-pool": "~1.0.12", + "generic-pool": "~2.0.3", "redis": "0.7.2", "hiredis": "~0.1.14", "request": "2.9.202", diff --git a/run_tests.sh b/run_tests.sh index 1b93c8fd..23be79db 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -76,7 +76,7 @@ fi PATH=node_modules/.bin/:$PATH echo "Running tests" -mocha -u tdd ${MOCHA_OPTS} ${TESTS} +mocha -t 10000 -u tdd ${MOCHA_OPTS} ${TESTS} ret=$? cleanup diff --git a/test/acceptance/multilayer.js b/test/acceptance/multilayer.js index 35e0027e..00a01290 100644 --- a/test/acceptance/multilayer.js +++ b/test/acceptance/multilayer.js @@ -21,6 +21,8 @@ suite('multilayer', function() { var redis_client = redis.createClient(global.environment.redis.port); var sqlapi_server; + var expected_last_updated_epoch = 1234567890123; // this is hard-coded into SQLAPIEmu + var expected_last_updated = new Date(expected_last_updated_epoch).toISOString(); suiteSetup(function(done){ sqlapi_server = new SQLAPIEmu(global.environment.sqlapi.port, done); @@ -34,17 +36,19 @@ suite('multilayer', function() { { options: { sql: 'select cartodb_id, ST_Translate(the_geom_webmercator, 5e6, 0) as the_geom_webmercator from test_table limit 2', cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', - cartocss_version: '2.0.1' + cartocss_version: '2.0.1', + interactivity: 'cartodb_id' } }, { options: { sql: 'select cartodb_id, ST_Translate(the_geom_webmercator, -5e6, 0) as the_geom_webmercator from test_table limit 2 offset 2', cartocss: '#layer { marker-fill:blue; marker-allow-overlap:true; }', - cartocss_version: '2.0.2' + cartocss_version: '2.0.2', + interactivity: 'cartodb_id' } } ] }; - var expected_token = "d55208dccb30b5ff972562f563db0d22"; + var expected_token = "e34dd7e235138a062f8ba7ad051aa3a7"; Step( function do_post() { @@ -65,15 +69,9 @@ suite('multilayer', function() { + layergroup.layers[1].options.sql + '$windshaft$)' }); - expectedBody.last_updated = JSON.stringify({ - 'q': 'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max ' - + 'FROM CDB_TableMetadata m WHERE m.tabname::name = any (\'{' - + qTables + '}\')' - }); + assert.equal(parsedBody.last_updated, expected_last_updated); if ( expected_token ) { - //assert.equal(parsedBody.layergroupid, expectedBody.layergroupid); - //assert.equal(parsedBody.last_updated, expectedBody.last_updated); - assert.deepEqual(parsedBody, expectedBody); + assert.equal(parsedBody.layergroupid, expected_token + ':' + expected_last_updated_epoch); } else expected_token = parsedBody.layergroupid; next(null, res); @@ -84,7 +82,7 @@ suite('multilayer', function() { if ( err ) throw err; var next = this; assert.response(server, { - url: '/tiles/layergroup/' + expected_token + '/0/0/0.png', + url: '/tiles/layergroup/' + expected_token + ':cb0/0/0/0.png', method: 'GET', headers: {host: 'localhost' }, encoding: 'binary' @@ -116,7 +114,7 @@ suite('multilayer', function() { var next = this; assert.response(server, { url: '/tiles/layergroup/' + expected_token - + '/layer0/0/0/0.grid.json?interactivity=cartodb_id', + + '/0/0/0/0.grid.json', headers: {host: 'localhost' }, method: 'GET' }, {}, function(res) { @@ -134,7 +132,7 @@ suite('multilayer', function() { var next = this; assert.response(server, { url: '/tiles/layergroup/' + expected_token - + '/layer1/0/0/0.grid.json?interactivity=cartodb_id', + + '/1/0/0/0.grid.json', headers: {host: 'localhost' }, method: 'GET' }, {}, function(res) { @@ -175,12 +173,13 @@ suite('multilayer', function() { sql: 'select 1 as cartodb_id, ' + 'ST_Buffer(!bbox!, -32*greatest(!pixel_width!,!pixel_height!)) as the_geom_webmercator', cartocss: '#layer { polygon-fill:red; }', - cartocss_version: '2.0.1' + cartocss_version: '2.0.1', + interactivity: 'cartodb_id' } } ] }; - var expected_token = "20f5710c00e3a1b0b4950de65ef0d875"; + var expected_token = "6d8e4ad5458e2d25cf0eef38e38717a6"; Step( function do_post() { @@ -200,15 +199,9 @@ suite('multilayer', function() { + layergroup.layers[0].options.sql + '$windshaft$)' }); - expectedBody.last_updated = JSON.stringify({ - 'q': 'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max ' - + 'FROM CDB_TableMetadata m WHERE m.tabname::name = any (\'{' - + qTables + '}\')' - }); + assert.equal(parsedBody.last_updated, expected_last_updated); if ( expected_token ) { - //assert.equal(parsedBody.layergroupid, expectedBody.layergroupid); - //assert.equal(parsedBody.last_updated, expectedBody.last_updated); - assert.deepEqual(parsedBody, expectedBody); + assert.equal(parsedBody.layergroupid, expected_token + ':' + expected_last_updated_epoch); } else expected_token = parsedBody.layergroupid; next(null, res); @@ -219,7 +212,7 @@ suite('multilayer', function() { if ( err ) throw err; var next = this; assert.response(server, { - url: '/tiles/layergroup/' + expected_token + '/1/0/0.png', + url: '/tiles/layergroup/' + expected_token + ':cb10/1/0/0.png', method: 'GET', headers: {host: 'localhost' }, encoding: 'binary' @@ -236,6 +229,9 @@ suite('multilayer', function() { var sentquery = JSON.parse(jsonquery); assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$' + layergroup.layers[0].options.sql + .replace(RegExp('!bbox!', 'g'), 'ST_MakeEnvelope(0,0,0,0)') + .replace(RegExp('!pixel_width!', 'g'), '1') + .replace(RegExp('!pixel_height!', 'g'), '1') + '$windshaft$)'); assert.imageEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.png', 2, @@ -249,7 +245,7 @@ suite('multilayer', function() { if ( err ) throw err; var next = this; assert.response(server, { - url: '/tiles/layergroup/' + expected_token + '/4/0/0.png', + url: '/tiles/layergroup/' + expected_token + ':cb11/4/0/0.png', method: 'GET', headers: {host: 'localhost' }, encoding: 'binary' @@ -266,6 +262,9 @@ suite('multilayer', function() { var sentquery = JSON.parse(jsonquery); assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$' + layergroup.layers[0].options.sql + .replace('!bbox!', 'ST_MakeEnvelope(0,0,0,0)') + .replace('!pixel_width!', '1') + .replace('!pixel_height!', '1') + '$windshaft$)'); assert.imageEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.png', 2, @@ -280,7 +279,7 @@ suite('multilayer', function() { var next = this; assert.response(server, { url: '/tiles/layergroup/' + expected_token - + '/layer0/1/0/0.grid.json?interactivity=cartodb_id', + + '/0/1/0/0.grid.json', headers: {host: 'localhost' }, method: 'GET' }, {}, function(res) { @@ -298,7 +297,7 @@ suite('multilayer', function() { var next = this; assert.response(server, { url: '/tiles/layergroup/' + expected_token - + '/layer0/4/0/0.grid.json?interactivity=cartodb_id', + + '/0/4/0/0.grid.json', headers: {host: 'localhost' }, method: 'GET' }, {}, function(res) { @@ -334,7 +333,7 @@ suite('multilayer', function() { version: '1.0.0', layers: [ { options: { - sql: 'select 1 as cartodb_id, ' + sql: 'select 1 as cartodb_id, !pixel_height! as h' + 'ST_Buffer(!bbox!, -32*greatest(!pixel_width!,!pixel_height!)) as the_geom_webmercator', cartocss: '#layer { polygon-fill:red; }', cartocss_version: '2.0.1' @@ -397,6 +396,8 @@ suite('multilayer', function() { errors.push(err.message); console.log("Error: " + err); } + // trip epoch + expected_token = expected_token.split(':')[0]; redis_client.keys("map_style|cartodb_test_user_1_db|~" + expected_token, function(err, matches) { if ( err ) errors.push(err.message); assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches); diff --git a/test/acceptance/server.js b/test/acceptance/server.js index cddc1cd4..7a383a4c 100644 --- a/test/acceptance/server.js +++ b/test/acceptance/server.js @@ -7,37 +7,15 @@ var semver = require('semver'); var mapnik = require('mapnik'); var Step = require('step'); var http = require('http'); -var LZMA = require('lzma/lzma_worker.js').LZMA; var SQLAPIEmu = require(__dirname + '/../support/SQLAPIEmu.js'); -require(__dirname + '/../support/test_helper'); +var helper = require(__dirname + '/../support/test_helper'); var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft'); var serverOptions = require(__dirname + '/../../lib/cartodb/server_options'); var server = new CartodbWindshaft(serverOptions); server.setMaxListeners(0); -// Utility function to compress & encode LZMA -function lzma_compress_to_hex(payload, mode, callback) { - var HEX = [ '0','1','2','3','4','5','6','7', - '8','9','a','b','c','d','e','f' ]; - LZMA.compress(payload, mode, - function(ints) { - for (var i=0; i> 4; - var lo = ints[i] & 0x0f; - ints[i] = HEX[hi] + HEX[lo]; - }; - var hex = ints.join(''); - callback(null, hex); - }, - function(percent) { - //console.log("Compressing: " + percent + "%"); - } - ); -} - suite('server', function() { var redis_client = redis.createClient(global.environment.redis.port); @@ -814,27 +792,32 @@ suite('server', function() { function compressQuery () { //console.log("Compressing starts"); var next = this; - lzma_compress_to_hex(JSON.stringify(qo), 1, this); - //cosole.log("compress returned " + x ); + helper.lzma_compress_to_base64(JSON.stringify(qo), 1, this); }, function sendRequest(err, lzma) { + if ( err ) throw err; + var next = this; //console.log("Compressing ends: " + typeof(lzma) + " - " + lzma); assert.response(server, { headers: {host: 'localhost'}, - url: '/tiles/test_table/15/16046/12354.png?lzma=' + lzma, + url: '/tiles/test_table/15/16046/12354.png?lzma=' + encodeURIComponent(lzma), method: 'GET', encoding: 'binary' - },{}, this); + },{}, function(res) { next(null, res); }); }, - function checkResponse(res) { + function checkResponse(err, res) { + if ( err ) throw err; + var next = this; assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); var ct = res.headers['content-type']; assert.equal(ct, 'image/png'); assert.imageEqualsFile(res.body, './test/fixtures/test_table_15_16046_12354_styled_black.png', 2, function(err, similarity) { - if (err) throw err; - done(); + next(err); }); + }, + function finish(err) { + done(err); } ); }); diff --git a/test/support/SQLAPIEmu.js b/test/support/SQLAPIEmu.js index 68ab26db..cbb61b7c 100644 --- a/test/support/SQLAPIEmu.js +++ b/test/support/SQLAPIEmu.js @@ -3,11 +3,20 @@ var url = require('url'); var o = function(port, cb) { + this.queries = []; + var that = this; this.sqlapi_server = http.createServer(function(req,res) { var query = url.parse(req.url, true).query; + that.queries.push(query); if ( query.q.match('SQLAPIERROR') ) { res.statusCode = 400; res.write(JSON.stringify({'error':'Some error occurred'})); + } else if ( query.q.match('EPOCH.* as max') ) { + // This is the structure of the known query sent by tiler + var row = { + 'max': 1234567890123 + }; + res.write(JSON.stringify({rows: [ row ]})); } else { var qs = JSON.stringify(query); var row = { diff --git a/test/support/test_helper.js b/test/support/test_helper.js index 3c59cac0..7b465611 100644 --- a/test/support/test_helper.js +++ b/test/support/test_helper.js @@ -6,6 +6,7 @@ */ var _ = require('underscore'); +var LZMA = require('lzma/lzma_worker.js').LZMA; // set environment specific variables global.settings = require(__dirname + '/../../config/settings'); @@ -13,4 +14,21 @@ global.environment = require(__dirname + '/../../config/environments/test'); _.extend(global.settings, global.environment); +// Utility function to compress & encode LZMA +function lzma_compress_to_base64(payload, mode, callback) { + LZMA.compress(payload, mode, + function(ints) { + ints = ints.map(function(c) { return String.fromCharCode(c + 128) }).join('') + var base64 = new Buffer(ints, 'binary').toString('base64'); + callback(null, base64); + }, + function(percent) { + //console.log("Compressing: " + percent + "%"); + } + ); +} + +module.exports = { + lzma_compress_to_base64: lzma_compress_to_base64 +} diff --git a/test/unit/cartodb/req2params.test.js b/test/unit/cartodb/req2params.test.js index 2562b076..3b7bb845 100644 --- a/test/unit/cartodb/req2params.test.js +++ b/test/unit/cartodb/req2params.test.js @@ -62,5 +62,24 @@ suite('req2params', function() { }); }); }); + + test('it should extend params with decoded lzma', function(done) { + var qo = { + style: 'test', + style_version: '2.1.0', + cache_buster: 5 + }; + test_helper.lzma_compress_to_base64(JSON.stringify(qo), 1, function(err, data) { + opts.req2params({ query: { non_included: 'toberemoved', api_key: 'test', style: 'override', lzma: data }}, function(err, req) { + var query = req.params + assert.equal(qo.style, query.style) + assert.equal(qo.style_version, query.style_version) + assert.equal(qo.cache_buster, query.cache_buster) + assert.equal('test', query.api_key) + assert.equal(undefined, query.non_included) + done(); + }); + }); + }); }); diff --git a/tools/reset_styles b/tools/reset_styles index b45c1a6b..9f770f0c 100755 --- a/tools/reset_styles +++ b/tools/reset_styles @@ -16,7 +16,7 @@ this script. var path = require('path'); // Reset all styles in the store -var grainstore = require('../node_modules/grainstore/lib/grainstore'); +var grainstore = require('../node_modules/windshaft/node_modules/grainstore/lib/grainstore'); var mapnik = require('mapnik'); var redis = require('redis');