diff --git a/.gitignore b/.gitignore index 96b65b68..55d81916 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ -node_modules +node_modules* config/environments/*.js .idea tools/munin/windshaft.conf +logs/ +pids/ diff --git a/Makefile b/Makefile index c6eee208..6c479c25 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,8 @@ check-local: config/environments/test.js test/unit/cartodb/redis_pool.test.js \ test/unit/cartodb/req2params.test.js \ test/acceptance/cache_validator.js \ - test/acceptance/server.js + test/acceptance/server.js \ + test/acceptance/multilayer.js check-submodules: for sub in windshaft grainstore mapnik; do \ diff --git a/NEWS.md b/NEWS.md index 57db4db3..78c528c5 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,14 @@ +1.1.9 +----- + +* Handle SQL API errors by requesting no Varnish cache +* Fix X-Cache-Channel for multilayer (by token) responses +* Add last_modified field to layergroup creation response (#72) +* Deprecate signal handler for USR1, add handler for USR2 (#71) +* Fix support for ampersend characters in CartoCSS +* Add support for LZMA compressed GET parameters +* Add support for creating layergroups via GET + 1.1.8 ----- * Require Windshaft-0.9.1, to reduce harmfulness of cache_buster param diff --git a/app.js b/app.js index 845a3cd6..ab75651a 100755 --- a/app.js +++ b/app.js @@ -46,6 +46,12 @@ ws.on('listening', function() { console.log("Windshaft tileserver started on " + global.environment.host + ':' + global.environment.port); }); +// DEPRECATED, use SIGUSR2 process.on('SIGUSR1', function() { + console.log('WARNING: handling of SIGUSR1 by Windshaft-CartoDB is deprecated, please send SIGUSR2 instead'); + ws.dumpCacheStats(); +}); + +process.on('SIGUSR2', function() { ws.dumpCacheStats(); }); diff --git a/config/environments/test.js.example b/config/environments/test.js.example index b97b9a6d..ff636581 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -48,8 +48,10 @@ var config = { } ,sqlapi: { protocol: 'http', - host: 'localhost.lan', - port: 8080, + host: '', + // This port will be used by "make check" for testing purposes + // It must be available + port: 1080, version: 'v1' } ,varnish: { diff --git a/lib/cartodb/cache_validator.js b/lib/cartodb/cache_validator.js index 479568f6..8449861e 100644 --- a/lib/cartodb/cache_validator.js +++ b/lib/cartodb/cache_validator.js @@ -1,8 +1,5 @@ var _ = require('underscore'), Varnish = require('node-varnish'), - request = require('request'), - crypto = require('crypto'), - channelCache = {}, varnish_queue = null; function init(host, port) { @@ -18,72 +15,7 @@ function invalidate_db(dbname, table) { } } -function generateCacheChannel(req, callback){ - var cacheChannel = ""; - - // use key to call sql api with sql request if present, else just return dbname and table name - // base key - var tableNames = req.params.table; - var dbName = req.params.dbname; - var username = req.headers.host.split('.')[0]; - - // replace tableNames with the results of the explain if present - if (_.isString(req.params.sql) && req.params.sql != ''){ - // initialise MD5 key of sql for cache lookups - var sql_md5 = generateMD5(req.params.sql); - var api = global.environment.sqlapi; - var qs = {}; - - // use cache if present - if (!_.isNull(channelCache[sql_md5]) && !_.isUndefined(channelCache[sql_md5])) { - callback(channelCache[sql_md5]); - } else{ - // strip out windshaft/mapnik inserted sql if present - var sql = req.params.sql.match(/^\((.*)\)\sas\scdbq$/); - sql = (sql != null) ? sql[1] : req.params.sql; - - // build up api string - var sqlapi = api.protocol + '://' + username + '.' + api.host + ':' + api.port + '/api/' + api.version + '/sql' - - // add query to querystring - qs.q = 'SELECT CDB_QueryTables($windshaft$' + sql + '$windshaft$)'; - - // add api_key if present in tile request (means table is private) - if (_.isString(req.params.map_key) && req.params.map_key != ''){ - qs.api_key = req.params.map_key; - } - - // call sql api - request.get({url:sqlapi, qs:qs, json:true}, function(err, response, body){ - if (!err && response.statusCode == 200) { - tableNames = body.rows[0].cdb_querytables.split(/^\{(.*)\}$/)[1]; - } else { - //oops, no SQL API. Just cache using fallback 'table' key - tableNames = 'table'; - } - cacheChannel = buildCacheChannel(dbName,tableNames); - channelCache[sql_md5] = cacheChannel; // store for caching - callback(cacheChannel); - }); - } - } else { - cacheChannel = buildCacheChannel(dbName,tableNames); - callback(cacheChannel); - } -} - -function buildCacheChannel(dbName, tableNames){ - return dbName + ':' + tableNames; -} - -function generateMD5(data){ - var hash = crypto.createHash('md5'); - hash.update(data); - return hash.digest('hex'); -} - module.exports = { init: init, - invalidate_db: invalidate_db, - generateCacheChannel: generateCacheChannel + invalidate_db: invalidate_db } diff --git a/lib/cartodb/cartodb_windshaft.js b/lib/cartodb/cartodb_windshaft.js index ded52079..b20212e4 100644 --- a/lib/cartodb/cartodb_windshaft.js +++ b/lib/cartodb/cartodb_windshaft.js @@ -82,10 +82,10 @@ var CartodbWindshaft = function(serverOptions) { ws.del(serverOptions.base_url + '/flush_cache', function(req, res){ ws.doCORS(res); Step( - function(){ + function flushCache(){ serverOptions.flushCache(req, serverOptions.cache_enabled ? Cache : null, this); }, - function(err, data){ + function sendResponse(err, data){ if (err){ ws.sendError(res, {error: err.message}, 500, 'DELETE CACHE'); //res.send(500); diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index ac9c458f..b4e47c43 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -3,6 +3,9 @@ var _ = require('underscore') , cartoData = require('./carto_data') , Cache = require('./cache_validator') , mapnik = require('mapnik') + , crypto = require('crypto') + , request = require('request') + , LZMA = require('lzma/lzma_worker.js').LZMA ; module.exports = function(){ @@ -44,6 +47,141 @@ module.exports = function(){ + me.grainstore.mapnik_version + ")"); } +/* This whole block is about generating X-Cache-Channel { */ + + // TODO: review lifetime of elements of this cache + // NOTE: by-token indices should only be dropped when + // the corresponding layegroup is dropped, because + // we have no SQL after layer creation. + me.channelCache = {}; + + // Run a query through the SQL api + me.sqlQuery = function (username, api_key, sql, callback) { + var api = global.environment.sqlapi; + + // build up api string + var sqlapi = api.protocol + '://' + username + '.' + api.host + ':' + api.port + '/api/' + api.version + '/sql' + + var qs = { q: sql } + + // add api_key if given + if (_.isString(api_key) && api_key != '') { qs.api_key = api_key; } + + // call sql api + request.get({url:sqlapi, qs:qs, json:true}, function(err, res, body){ + if (err){ + callback(err); + return; + } + if (res.statusCode != 200) { + var msg = res.body.error ? res.body.error : res.body; + callback(new Error('unexpected response status (' + res.statusCode + ') for sql query: ' + sql)); + return; + } + callback(null, body.rows); + }); + }; + + me.findLastUpdated = function (username, api_key, tableNames, callback) { + var sql = 'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max FROM CDB_TableMetadata m WHERE m.tabname::name = any (\'{' + + tableNames.join(',') + '}\')'; + + // call sql api + me.sqlQuery(username, api_key, sql, function(err, rows){ + if (err){ + var msg = err.message ? err.message : err; + callback(new Error('could not find last updated timestamp: ' + msg)); + return; + } + // when the table has not updated_at means it hasn't been changed so a default last_updated is set + var last_updated = 0; + if(rows.length !== 0) { + last_updated = rows[0].max || 0; + } + callback(null, last_updated); + }); + }; + + me.affectedTables = function (username, api_key, sql, callback) { + + var sql = 'SELECT CDB_QueryTables($windshaft$' + sql + '$windshaft$)'; + + // call sql api + me.sqlQuery(username, api_key, sql, function(err, rows){ + if (err){ + var msg = err.message ? err.message : err; + callback(new Error('could not fetch source tables: ' + msg)); + return; + } + var qtables = rows[0].cdb_querytables; + var tableNames = qtables.split(/^\{(.*)\}$/)[1]; + tableNames = tableNames.split(','); + callback(null, tableNames); + }); + }; + + me.buildCacheChannel = function (dbName, tableNames){ + return dbName + ':' + tableNames.join(','); + }; + + me.generateMD5 = function(data){ + var hash = crypto.createHash('md5'); + hash.update(data); + return hash.digest('hex'); + } + + me.generateCacheChannel = function(req, callback){ + + // use key to call sql api with sql request if present, else + // just return dbname and table name base key + var dbName = req.params.dbname; + + var cacheKey = [ dbName ]; + if ( req.params.token ) cacheKey.push(req.params.token); + else if ( req.params.sql ) cacheKey.push( me.generateMD5(req.params.sql) ); + cacheKey = cacheKey.join(':'); + + if ( me.channelCache.hasOwnProperty(cacheKey) ) { + callback(null, me.channelCache[cacheKey]); + return; + } + + if ( req.params.token ) { + if ( ! me.channelCache.hasOwnProperty(cacheKey) ) { + callback(new Error('missing channel cache for token ' + req.params.token)); + } else { + callback(null, me.channelCache[cacheKey]); + } + return; + } + + if ( ! req.params.sql && ! req.params.token ) { + var cacheChannel = me.buildCacheChannel(dbName, [req.params.table]); + // not worth caching this + callback(null, cacheChannel); + return; + } + + if ( ! req.params.sql ) { + callback(new Error("this request doesn't need an X-Cache-Channel generated")); + return; + } + + var dbName = req.params.dbname; + var username = req.headers.host.split('.')[0]; + + // strip out windshaft/mapnik inserted sql if present + var sql = req.params.sql.match(/^\((.*)\)\sas\scdbq$/); + sql = (sql != null) ? sql[1] : req.params.sql; + + me.affectedTables(username, req.params.map_key, sql, function(err, tableNames) { + if ( err ) { callback(err); return; } + var cacheChannel = me.buildCacheChannel(dbName,tableNames); + me.channelCache[cacheKey] = cacheChannel; // store for caching + callback(null, cacheChannel); + }); + }; + // Set the cache chanel info to invalidate the cache on the frontend server // // @param req The request object. @@ -57,19 +195,57 @@ module.exports = function(){ // skip non-GET requests, or requests for which there's no response if ( req.method != 'GET' || ! req.res ) { cb(null, null); return; } var res = req.res; - var ttl = global.environment.varnish.ttl || 86400; - Cache.generateCacheChannel(req, function(channel){ - res.header('X-Cache-Channel', channel); - var cache_policy = req.query.cache_policy; - if ( cache_policy == 'persist' ) { - res.header('Cache-Control', 'public,max-age=31536000'); // 1 year + var cache_policy = req.query.cache_policy; + if ( cache_policy == 'persist' ) { + res.header('Cache-Control', 'public,max-age=31536000'); // 1 year + } else { + var ttl = global.environment.varnish.ttl || 86400; + res.header('Last-Modified', new Date().toUTCString()); + res.header('Cache-Control', 'no-cache,max-age='+ttl+',must-revalidate, public'); + } + + me.generateCacheChannel(req, function(err, channel){ + if ( ! err ) { + res.header('X-Cache-Channel', channel); + cb(null, channel); } else { - res.header('Last-Modified', new Date().toUTCString()); - res.header('Cache-Control', 'no-cache,max-age='+ttl+',must-revalidate, public'); + console.log('ERROR generating cache channel: ' + ( err.message ? err.message : err )); + // TODO: evaluate if we should bubble up the error instead + cb(null, 'ERROR'); } - cb(null, channel); // add last-modified too ? }); - } + }; + + me.afterLayergroupCreate = function(req, mapconfig, response, callback) { + var token = response.layergroupid; + + var sql = []; + _.each(mapconfig.layers, function(lyr) { + sql.push(lyr.options.sql); + }); + sql = sql.join(';'); + + var dbName = req.params.dbname; + var usr = req.headers.host.split('.')[0]; + var key = req.params.map_key; + + var cacheKey = dbName + ':' + token; + + me.affectedTables(usr, key, sql, function(err, tableNames) { + + if ( err ) { callback(err); return; } + var cacheChannel = me.buildCacheChannel(dbName,tableNames); + me.channelCache[cacheKey] = cacheChannel; // store for caching + // find last updated + me.findLastUpdated(usr, key, tableNames, function(err, lastUpdated) { + if ( err ) { callback(err); return; } + response.last_updated = lastUpdated; + callback(null); + }); + }); + }; + +/* X-Cache-Channel generation } */ /** * Whitelist input and get database name & default geometry type from @@ -79,6 +255,42 @@ module.exports = function(){ */ me.req2params = function(req, callback){ + if ( req.query.lzma ) { + + // TODO: check ? + //console.log("type of req.query.lzma is " + typeof(req.query.lzma)); + //console.log("req.query.lzma is " + req.query.lzma); + + // Decode + var lzma = []; + for (var i=0; i 127 ) num = 127-num; + //console.log(i + " hex: " + hex + " decodes as " + num); + lzma.push( num ); + } + + // Decompress + //console.log("LZMA decompression starts with " + lzma); + LZMA.decompress( + lzma, + function(result) { + //console.log("LZMA decompression completed, payload: "); console.dir(result); + try { + req.query = JSON.parse(result); + me.req2params(req, callback); + } catch (err) { + callback(new Error('Error parsing lzma as JSON: ' + err)); + } + }, + function(percent) { // progress + //console.log("LZMA decompression " + percent + "%"); + } + ); + return; + } + // Whitelist query parameters and attach format var good_query = ['sql', 'geom_type', 'cache_buster', 'cache_policy', 'callback', 'interactivity', 'map_key', 'api_key', 'style', 'style_version', 'style_convert' ]; var bad_query = _.difference(_.keys(req.query), good_query); @@ -90,7 +302,8 @@ module.exports = function(){ _.extend(req.params, req.query); // for cartodb, ensure interactivity is cartodb_id or user specified - req.params.interactivity = req.params.interactivity || 'cartodb_id'; + // Don't: https://github.com/Vizzuality/Windshaft-cartodb/issues/69 + //req.params.interactivity = req.params.interactivity || 'cartodb_id'; req.params.processXML = function(req, xml, callback) { var dbuser = req.dbuser ? req.dbuser : global.settings.postgres.user; @@ -127,7 +340,7 @@ module.exports = function(){ if (!_.isNull(data)) _.extend(req.params, {geom_type: data}); - that.addCacheChannel(req, function(err, chan) { + that.addCacheChannel(req, function(err) { callback(err, req); }); } @@ -181,11 +394,17 @@ module.exports = function(){ var that = this; Step( - function(){ + function getParams(){ + // this is mostly to compute req.params.dbname that.req2params(req, this); }, - function(err, data){ - if (err) throw err; + function flushInternalCache(err){ + // TODO: implement this, see + // http://github.com/Vizzuality/Windshaft-cartodb/issues/73 + return true; + }, + function flushVarnishCache(err){ + if (err) { callback(err); return; } if(Cache) { Cache.invalidate_db(req.params.dbname, req.params.table); } diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 3ef03a44..f720c85a 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -1,6 +1,6 @@ { "name": "windshaft-cartodb", - "version": "1.1.7", + "version": "1.1.9", "dependencies": { "cluster2": { "version": "0.3.5-cdb02", @@ -13,7 +13,7 @@ "version": "1.9.2", "dependencies": { "formidable": { - "version": "1.0.12" + "version": "1.0.13" } } }, @@ -32,10 +32,10 @@ "version": "0.8.3" }, "npm": { - "version": "1.2.12", + "version": "1.2.16", "dependencies": { "semver": { - "version": "1.1.3" + "version": "1.1.4" }, "ini": { "version": "1.1.0" @@ -50,7 +50,7 @@ "version": "1.2.0" }, "minimatch": { - "version": "0.2.9", + "version": "0.2.11", "dependencies": { "sigmund": { "version": "1.0.0" @@ -71,7 +71,7 @@ "version": "1.0.5" }, "tar": { - "version": "0.1.16" + "version": "0.1.17" }, "fstream": { "version": "0.1.22" @@ -84,7 +84,7 @@ "from": "git://github.com/isaacs/inherits" }, "mkdirp": { - "version": "0.3.4" + "version": "0.3.5" }, "read": { "version": "1.0.4", @@ -95,13 +95,13 @@ } }, "lru-cache": { - "version": "2.0.4" + "version": "2.3.0" }, "node-gyp": { - "version": "0.8.4" + "version": "0.9.5" }, "fstream-npm": { - "version": "0.1.3", + "version": "0.1.4", "dependencies": { "fstream-ignore": { "version": "0.0.6" @@ -124,7 +124,7 @@ "version": "0.1.2" }, "npm-registry-client": { - "version": "0.2.17", + "version": "0.2.18", "dependencies": { "couch-login": { "version": "0.1.15" @@ -132,13 +132,13 @@ } }, "read-package-json": { - "version": "0.2.0" + "version": "0.3.0" }, "read-installed": { "version": "0.1.1" }, "glob": { - "version": "3.1.20" + "version": "3.1.21" }, "init-package-json": { "version": "0.0.6", @@ -175,6 +175,9 @@ }, "opener": { "version": "1.3.0" + }, + "chmodr": { + "version": "0.1.0" } } } @@ -186,79 +189,82 @@ "underscore": { "version": "1.3.3" }, - "grainstore": { + "windshaft": { "version": "0.11.1", "dependencies": { - "carto": { - "version": "0.9.3-cdb2", - "from": "git://github.com/CartoDB/carto.git#cdb-0.9", + "grainstore": { + "version": "0.12.0", "dependencies": { - "mapnik-reference": { - "version": "5.0.0-cdb1", - "from": "git://github.com/CartoDB/mapnik-reference.git#cdb-5.0" - }, - "xml2js": { - "version": "0.1.14", + "carto": { + "version": "0.9.3-cdb3", + "from": "git://github.com/CartoDB/carto.git#cdb-0.9.3-cdb3", "dependencies": { - "sax": { - "version": "0.5.2" + "mapnik-reference": { + "version": "5.0.0-cdb1", + "from": "git://github.com/CartoDB/mapnik-reference.git#cdb-5.0" + }, + "xml2js": { + "version": "0.1.14", + "dependencies": { + "sax": { + "version": "0.5.2" + } + } } } - } - } - }, - "mapnik-reference": { - "version": "5.0.4" - }, - "millstone": { - "version": "0.5.15", - "dependencies": { - "request": { - "version": "2.11.4", + }, + "mapnik-reference": { + "version": "5.0.4" + }, + "millstone": { + "version": "0.5.15", "dependencies": { - "form-data": { - "version": "0.0.3", + "generic-pool": { + "version": "2.0.3" + }, + "request": { + "version": "2.12.0", "dependencies": { - "combined-stream": { + "form-data": { "version": "0.0.3", "dependencies": { - "delayed-stream": { - "version": "0.0.5" + "combined-stream": { + "version": "0.0.3", + "dependencies": { + "delayed-stream": { + "version": "0.0.5" + } + } + }, + "async": { + "version": "0.1.9" } } }, - "async": { - "version": "0.1.9" + "mime": { + "version": "1.2.7" } } }, + "srs": { + "version": "0.2.20" + }, + "zipfile": { + "version": "0.3.4" + }, + "sqlite3": { + "version": "2.1.7" + }, "mime": { - "version": "1.2.7" + "version": "1.2.9" + }, + "mkdirp": { + "version": "0.3.5" } } - }, - "srs": { - "version": "0.2.20" - }, - "zipfile": { - "version": "0.3.4" - }, - "sqlite3": { - "version": "2.1.5" - }, - "mime": { - "version": "1.2.9" - }, - "mkdirp": { - "version": "0.3.5" } } - } - } - }, - "windshaft": { - "version": "0.9.1", - "dependencies": { + }, "express": { "version": "2.5.11", "dependencies": { @@ -266,7 +272,7 @@ "version": "1.9.2", "dependencies": { "formidable": { - "version": "1.0.12" + "version": "1.0.13" } } }, @@ -298,12 +304,18 @@ } }, "tilelive-mapnik": { - "version": "0.3.3-cdb2", - "from": "git://github.com/Vizzuality/tilelive-mapnik.git#6061c65a", + "version": "0.5.0", + "from": "git://github.com/Vizzuality/tilelive-mapnik.git#6a360ee50", "dependencies": { + "generic-pool": { + "version": "2.0.3" + }, "eio": { "version": "0.1.0" }, + "mime": { + "version": "1.2.9" + }, "sphericalmercator": { "version": "1.0.2" } @@ -321,7 +333,12 @@ "version": "0.7.2" }, "hiredis": { - "version": "0.1.14" + "version": "0.1.15", + "dependencies": { + "bindings": { + "version": "1.1.0" + } + } }, "request": { "version": "2.9.202" @@ -329,8 +346,11 @@ "mapnik": { "version": "0.7.22" }, + "lzma": { + "version": "1.2.2" + }, "semver": { - "version": "1.1.3" + "version": "1.1.4" }, "mocha": { "version": "1.2.1", diff --git a/package.json b/package.json index fab65f18..e87e1a2c 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "1.1.8", + "version": "1.1.9", "description": "A map tile server for CartoDB", "url": "https://github.com/Vizzuality/Windshaft-cartodb", "licenses": [{ @@ -21,14 +21,14 @@ "cluster2": "git://github.com/CartoDB/cluster2.git#cdb_production", "node-varnish": "0.1.1", "underscore" : "~1.3.3", - "grainstore" : "~0.11.1", - "windshaft" : "~0.9.1", + "windshaft" : "~0.11.1", "step": "0.0.x", "generic-pool": "~1.0.12", "redis": "0.7.2", "hiredis": "~0.1.14", "request": "2.9.202", - "mapnik": "~0.7.14" + "mapnik": "~0.7.14", + "lzma": "~1.2.2" }, "devDependencies": { "mocha": "1.2.1", diff --git a/test/acceptance/multilayer.js b/test/acceptance/multilayer.js index 91e37b34..259c0c6e 100644 --- a/test/acceptance/multilayer.js +++ b/test/acceptance/multilayer.js @@ -6,6 +6,7 @@ var querystring = require('querystring'); var semver = require('semver'); var mapnik = require('mapnik'); var Step = require('step'); +var SQLAPIEmu = require(__dirname + '/../support/SQLAPIEmu.js'); require(__dirname + '/../support/test_helper'); @@ -19,8 +20,10 @@ server.setMaxListeners(0); suite('multilayer', function() { var redis_client = redis.createClient(global.environment.redis.port); + var sqlapi_server; - suiteSetup(function(){ + suiteSetup(function(done){ + sqlapi_server = new SQLAPIEmu(global.environment.sqlapi.port, done); }); test("layergroup with 2 layers, each with its style", function(done) { @@ -41,7 +44,7 @@ suite('multilayer', function() { ] }; - var expected_token = "d442ca6d3ece793b9c16c02a1d1ea5f2"; + var expected_token = "d55208dccb30b5ff972562f563db0d22"; Step( function do_post() { @@ -55,9 +58,23 @@ suite('multilayer', function() { assert.equal(res.statusCode, 200, res.body); var parsedBody = JSON.parse(res.body); var expectedBody = { layergroupid: expected_token }; - // TODO: check last modified - //expectedBody.layercount = 2; - if ( expected_token ) assert.deepEqual(parsedBody, expectedBody); + // check last modified + var qTables = JSON.stringify({ + 'q': 'SELECT CDB_QueryTables($windshaft$' + + layergroup.layers[0].options.sql + ';' + + layergroup.layers[1].options.sql + + '$windshaft$)' + }); + expectedBody.last_updated = JSON.stringify({ + 'q': 'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max ' + + 'FROM CDB_TableMetadata m WHERE m.tabname::name = any (\'{' + + qTables + '}\')' + }); + if ( expected_token ) { + //assert.equal(parsedBody.layergroupid, expectedBody.layergroupid); + //assert.equal(parsedBody.last_updated, expectedBody.last_updated); + assert.deepEqual(parsedBody, expectedBody); + } else expected_token = parsedBody.layergroupid; next(null, res); }); @@ -74,6 +91,19 @@ suite('multilayer', function() { }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); assert.equal(res.headers['content-type'], "image/png"); + + // Check X-Cache-Channel + var cc = res.headers['x-cache-channel']; + assert.ok(cc); + var dbname = 'cartodb_test_user_1_db' + assert.equal(cc.substring(0, dbname.length), dbname); + var jsonquery = cc.substring(dbname.length+1); + var sentquery = JSON.parse(jsonquery); + assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$' + + layergroup.layers[0].options.sql + ';' + + layergroup.layers[1].options.sql + + '$windshaft$)'); + assert.imageEqualsFile(res.body, 'test/fixtures/test_table_0_0_0_multilayer1.png', 2, function(err, similarity) { next(err); @@ -135,12 +165,177 @@ suite('multilayer', function() { ); }); + + test("layergroup can hold substitution tokens", function(done) { + + var layergroup = { + version: '1.0.0', + layers: [ + { options: { + sql: 'select 1 as cartodb_id, ' + + 'ST_Buffer(!bbox!, -32*greatest(!pixel_width!,!pixel_height!)) as the_geom_webmercator', + cartocss: '#layer { polygon-fill:red; }', + cartocss_version: '2.0.1' + } } + ] + }; + + var expected_token = "20f5710c00e3a1b0b4950de65ef0d875"; + Step( + function do_post() + { + var next = this; + assert.response(server, { + url: '/tiles/layergroup', + method: 'POST', + headers: {host: 'localhost', 'Content-Type': 'application/json' }, + data: JSON.stringify(layergroup) + }, {}, function(res) { + assert.equal(res.statusCode, 200, res.body); + var parsedBody = JSON.parse(res.body); + var expectedBody = { layergroupid: expected_token }; + // check last modified + var qTables = JSON.stringify({ + 'q': 'SELECT CDB_QueryTables($windshaft$' + + layergroup.layers[0].options.sql + + '$windshaft$)' + }); + expectedBody.last_updated = JSON.stringify({ + 'q': 'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max ' + + 'FROM CDB_TableMetadata m WHERE m.tabname::name = any (\'{' + + qTables + '}\')' + }); + if ( expected_token ) { + //assert.equal(parsedBody.layergroupid, expectedBody.layergroupid); + //assert.equal(parsedBody.last_updated, expectedBody.last_updated); + assert.deepEqual(parsedBody, expectedBody); + } + else expected_token = parsedBody.layergroupid; + next(null, res); + }); + }, + function do_get_tile1(err) + { + if ( err ) throw err; + var next = this; + assert.response(server, { + url: '/tiles/layergroup/' + expected_token + '/1/0/0.png', + method: 'GET', + headers: {host: 'localhost' }, + encoding: 'binary' + }, {}, function(res) { + assert.equal(res.statusCode, 200, res.body); + assert.equal(res.headers['content-type'], "image/png"); + + // Check X-Cache-Channel + var cc = res.headers['x-cache-channel']; + assert.ok(cc); + var dbname = 'cartodb_test_user_1_db' + assert.equal(cc.substring(0, dbname.length), dbname); + var jsonquery = cc.substring(dbname.length+1); + var sentquery = JSON.parse(jsonquery); + assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$' + + layergroup.layers[0].options.sql + + '$windshaft$)'); + + assert.imageEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.png', 2, + function(err, similarity) { + next(err); + }); + }); + }, + function do_get_tile4(err) + { + if ( err ) throw err; + var next = this; + assert.response(server, { + url: '/tiles/layergroup/' + expected_token + '/4/0/0.png', + method: 'GET', + headers: {host: 'localhost' }, + encoding: 'binary' + }, {}, function(res) { + assert.equal(res.statusCode, 200, res.body); + assert.equal(res.headers['content-type'], "image/png"); + + // Check X-Cache-Channel + var cc = res.headers['x-cache-channel']; + assert.ok(cc); + var dbname = 'cartodb_test_user_1_db' + assert.equal(cc.substring(0, dbname.length), dbname); + var jsonquery = cc.substring(dbname.length+1); + var sentquery = JSON.parse(jsonquery); + assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$' + + layergroup.layers[0].options.sql + + '$windshaft$)'); + + assert.imageEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.png', 2, + function(err, similarity) { + next(err); + }); + }); + }, + function do_get_grid1(err) + { + if ( err ) throw err; + var next = this; + assert.response(server, { + url: '/tiles/layergroup/' + expected_token + + '/layer0/1/0/0.grid.json?interactivity=cartodb_id', + headers: {host: 'localhost' }, + method: 'GET' + }, {}, function(res) { + assert.equal(res.statusCode, 200, res.body); + assert.equal(res.headers['content-type'], "text/javascript; charset=utf-8; charset=utf-8"); + assert.utfgridEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.grid.json', 2, + function(err, similarity) { + next(err); + }); + }); + }, + function do_get_grid4(err) + { + if ( err ) throw err; + var next = this; + assert.response(server, { + url: '/tiles/layergroup/' + expected_token + + '/layer0/4/0/0.grid.json?interactivity=cartodb_id', + headers: {host: 'localhost' }, + method: 'GET' + }, {}, function(res) { + assert.equal(res.statusCode, 200, res.body); + assert.equal(res.headers['content-type'], "text/javascript; charset=utf-8; charset=utf-8"); + assert.utfgridEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.grid.json', 2, + function(err, similarity) { + next(err); + }); + }); + }, + function finish(err) { + var errors = []; + if ( err ) { + errors.push(err.message); + console.log("Error: " + err); + } + redis_client.keys("map_style|cartodb_test_user_1_db|~" + expected_token, function(err, matches) { + if ( err ) errors.push(err.message); + assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches); + redis_client.del(matches, function(err) { + if ( err ) errors.push(err.message); + if ( errors.length ) done(new Error(errors)); + else done(null); + }); + }); + } + ); + }); + suiteTeardown(function(done) { // This test will add map_style records, like // 'map_style|null|publicuser|my_table', redis_client.keys("map_style|*", function(err, matches) { - _.each(matches, function(k) { redis_client.del(k); }); - done(); + redis_client.del(matches, function(err) { + sqlapi_server.close(done); + }); }); }); diff --git a/test/acceptance/server.js b/test/acceptance/server.js index 3e419015..0fa3ea12 100644 --- a/test/acceptance/server.js +++ b/test/acceptance/server.js @@ -6,6 +6,9 @@ var querystring = require('querystring'); var semver = require('semver'); var mapnik = require('mapnik'); var Step = require('step'); +var http = require('http'); +var LZMA = require('lzma/lzma_worker.js').LZMA; +var SQLAPIEmu = require(__dirname + '/../support/SQLAPIEmu.js'); require(__dirname + '/../support/test_helper'); @@ -14,9 +17,31 @@ var serverOptions = require(__dirname + '/../../lib/cartodb/server_options'); var server = new CartodbWindshaft(serverOptions); server.setMaxListeners(0); +// Utility function to compress & encode LZMA +function lzma_compress_to_hex(payload, mode, callback) { + var HEX = [ '0','1','2','3','4','5','6','7', + '8','9','a','b','c','d','e','f' ]; + LZMA.compress(payload, mode, + function(ints) { + for (var i=0; i> 4; + var lo = ints[i] & 0x0f; + ints[i] = HEX[hi] + HEX[lo]; + }; + var hex = ints.join(''); + callback(null, hex); + }, + function(percent) { + //console.log("Compressing: " + percent + "%"); + } + ); +} + suite('server', function() { var redis_client = redis.createClient(global.environment.redis.port); + var sqlapi_server; var default_style = semver.satisfies(mapnik.versions.mapnik, '<2.1.0') ? @@ -30,7 +55,8 @@ suite('server', function() { var test_style_black_200 = "#test_table{marker-fill:black;marker-line-color:red;marker-width:10}"; var test_style_black_210 = "#test_table{marker-fill:black;marker-line-color:red;marker-width:20}"; - suiteSetup(function(){ + suiteSetup(function(done){ + sqlapi_server = new SQLAPIEmu(global.environment.sqlapi.port, done); }); ///////////////////////////////////////////////////////////////////////////////// @@ -504,11 +530,24 @@ suite('server', function() { // ///////////////////////////////////////////////////////////////////////////////// - test("get'ing a json with default style should return an grid", function(done){ + test("get'ing a grid with no interactivity should fail", function(done){ assert.response(server, { headers: {host: 'localhost'}, url: '/tiles/gadm4/6/31/24.grid.json', method: 'GET' + },{}, function(res) { + assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body); + assert.deepEqual(JSON.parse(res.body), {"error":"Missing interactivity parameter"}); + done(); + }); + }); + + + test("get'ing a json with default style should return an grid", function(done){ + assert.response(server, { + headers: {host: 'localhost'}, + url: '/tiles/gadm4/6/31/24.grid.json?interactivity=cartodb_id', + method: 'GET' },{ status: 200, headers: { 'Content-Type': 'text/javascript; charset=utf-8; charset=utf-8', @@ -519,7 +558,7 @@ suite('server', function() { test("get'ing a json with default style should return an grid", function(done){ assert.response(server, { headers: {host: 'localhost'}, - url: '/tiles/gadm4/6/31/24.grid.json', + url: '/tiles/gadm4/6/31/24.grid.json?interactivity=cartodb_id', method: 'GET' },{ status: 200, @@ -528,10 +567,13 @@ suite('server', function() { }); test("get'ing a json with default style and sql should return a constrained grid", function(done){ - var sql = querystring.stringify({sql: "SELECT * FROM gadm4 WHERE codineprov = '08'"}) + var q = querystring.stringify({ + interactivity: 'cartodb_id', + sql: "SELECT * FROM gadm4 WHERE codineprov = '08'" + }) assert.response(server, { headers: {host: 'localhost'}, - url: '/tiles/gadm4/6/31/24.grid.json?' + sql, + url: '/tiles/gadm4/6/31/24.grid.json?' + q, method: 'GET' },{ status: 200, @@ -543,7 +585,7 @@ suite('server', function() { function(done) { assert.response(server, { headers: {host: 'localhost'}, - url: '/tiles/test_table_private_1/6/31/24.grid.json', + url: '/tiles/test_table_private_1/6/31/24.grid.json?interactivity=cartodb_id', method: 'GET' },{}, function(res) { // 401 Unauthorized @@ -557,7 +599,7 @@ suite('server', function() { function(done) { assert.response(server, { headers: {host: 'unknown_user'}, - url: '/tiles/test_table_private_1/6/31/24.grid.json', + url: '/tiles/test_table_private_1/6/31/24.grid.json?interactivity=cartodb_id', method: 'GET' },{ }, function(res) { @@ -573,7 +615,7 @@ suite('server', function() { function(done) { assert.response(server, { headers: {host: 'localhost'}, - url: '/tiles/test_table_private_1/6/31/24.grid.json?map_key=1234', + url: '/tiles/test_table_private_1/6/31/24.grid.json?map_key=1234&interactivity=cartodb_id', method: 'GET' },{}, function(res) { assert.equal(res.statusCode, 200, res.body); @@ -778,6 +820,41 @@ suite('server', function() { }); }); + test("get'ing a tile with url specified 2.1.0 style (lzma version)", function(done){ + var qo = { + style: test_style_black_210, + style_version: '2.1.0', + cache_buster: 5 + }; + Step ( + function compressQuery () { + //console.log("Compressing starts"); + var next = this; + lzma_compress_to_hex(JSON.stringify(qo), 1, this); + //cosole.log("compress returned " + x ); + }, + function sendRequest(err, lzma) { + //console.log("Compressing ends: " + typeof(lzma) + " - " + lzma); + assert.response(server, { + headers: {host: 'localhost'}, + url: '/tiles/test_table/15/16046/12354.png?lzma=' + lzma, + method: 'GET', + encoding: 'binary' + },{}, this); + }, + function checkResponse(res) { + assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); + var ct = res.headers['content-type']; + assert.equal(ct, 'image/png'); + assert.imageEqualsFile(res.body, './test/fixtures/test_table_15_16046_12354_styled_black.png', 2, + function(err, similarity) { + if (err) throw err; + done(); + }); + } + ); + }); + // See http://github.com/Vizzuality/Windshaft-cartodb/issues/57 test("GET'ing a tile as anonymous with style set by POST", function(done){ var style = querystring.stringify({style: test_style_black_210, style_version: '2.1.0'}); @@ -916,6 +993,66 @@ suite('server', function() { ); }); + test("uses sqlapi to figure source data of query", function(done){ + var qo = { + sql: "SELECT g.cartodb_id, g.codineprov, t.the_geom_webmercator " + + "FROM gadm4 g, test_table t " + + "WHERE g.cartodb_id = t.cartodb_id", + map_key: 1234 + }; + var sqlapi; + Step( + function sendRequest(err) { + assert.response(server, { + headers: {host: 'localhost'}, + url: '/tiles/gadm4/6/31/24.png?' + querystring.stringify(qo), + method: 'GET' + },{}, this); + }, + function checkResponse(res) { + assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); + var ct = res.headers['content-type']; + assert.equal(ct, 'image/png'); + var cc = res.headers['x-cache-channel']; + var dbname = 'cartodb_test_user_1_db' + assert.equal(cc.substring(0, dbname.length), dbname); + var jsonquery = cc.substring(dbname.length+1); + var sentquery = JSON.parse(jsonquery); + assert.equal(sentquery.api_key, qo.map_key); + assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$' + qo.sql + '$windshaft$)'); + done(); + } + ); + }); + + test("requests to skip cache on sqlapi error", function(done){ + var qo = { + sql: "SELECT g.cartodb_id, g.codineprov, t.the_geom_webmercator " + + ", 'SQLAPIERROR' is not null " + + "FROM gadm4 g, test_table t " + + "WHERE g.cartodb_id = t.cartodb_id", + map_key: 1234 + }; + var sqlapi; + Step( + function sendRequest(err) { + assert.response(server, { + headers: {host: 'localhost'}, + url: '/tiles/gadm4/6/31/24.png?' + querystring.stringify(qo), + method: 'GET' + },{}, this); + }, + function checkResponse(res) { + assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body); + var ct = res.headers['content-type']; + assert.equal(ct, 'image/png'); + // does NOT send an x-cache-channel + assert.ok(!res.headers.hasOwnProperty('x-cache-channel')); + done(); + } + ); + }); + ///////////////////////////////////////////////////////////////////////////////// // // DELETE CACHE @@ -1040,7 +1177,7 @@ suite('server', function() { // 'map_style|null|publicuser|my_table', redis_client.keys("map_style|*", function(err, matches) { _.each(matches, function(k) { redis_client.del(k); }); - done(); + sqlapi_server.close(done); }); }); diff --git a/test/support/SQLAPIEmu.js b/test/support/SQLAPIEmu.js new file mode 100644 index 00000000..68ab26db --- /dev/null +++ b/test/support/SQLAPIEmu.js @@ -0,0 +1,29 @@ +var http = require('http'); +var url = require('url'); + +var o = function(port, cb) { + + this.sqlapi_server = http.createServer(function(req,res) { + var query = url.parse(req.url, true).query; + if ( query.q.match('SQLAPIERROR') ) { + res.statusCode = 400; + res.write(JSON.stringify({'error':'Some error occurred'})); + } else { + var qs = JSON.stringify(query); + var row = { + // This is the structure of the known query sent by tiler + 'cdb_querytables': '{' + qs + '}', + 'max': qs + }; + res.write(JSON.stringify({rows: [ row ]})); + } + res.end(); + }).listen(port, cb); +}; + +o.prototype.close = function(cb) { + this.sqlapi_server.close(cb); +}; + +module.exports = o; + diff --git a/test/support/sql/gadm4.sql b/test/support/sql/gadm4.sql index 4f257192..525053ae 100644 --- a/test/support/sql/gadm4.sql +++ b/test/support/sql/gadm4.sql @@ -97,8 +97,6 @@ ALTER TABLE ONLY gadm4 CREATE INDEX bdll25_provincias_4326_2_the_geom_webmercator_idx ON gadm4 USING gist (the_geom_webmercator); --- development_cartodb_user_3 role -CREATE USER development_cartodb_user_3; -GRANT ALL ON TABLE gadm4 TO development_cartodb_user_3; +GRANT ALL ON TABLE gadm4 TO test_cartodb_user_1; GRANT SELECT ON TABLE gadm4 TO publicuser; diff --git a/test/unit/cartodb/req2params.test.js b/test/unit/cartodb/req2params.test.js index 2562b076..a29e9708 100644 --- a/test/unit/cartodb/req2params.test.js +++ b/test/unit/cartodb/req2params.test.js @@ -19,7 +19,7 @@ suite('req2params', function() { assert.ok(_.isObject(req.query), 'request has query'); assert.ok(!req.query.hasOwnProperty('dbuser'), 'dbuser was removed from query'); assert.ok(req.hasOwnProperty('params'), 'request has params'); - assert.ok(req.params.hasOwnProperty('interactivity'), 'request params have interactivity'); + assert.ok(!req.params.hasOwnProperty('interactivity'), 'request params have interactivity'); assert.equal(req.params.dbname, 'cartodb_test_user_1_db', 'could forge dbname: '+ req.params.dbname); assert.ok(!req.hasOwnProperty('dbuser'), 'could inject dbuser ('+req.params.dbuser+')'); done(); @@ -33,7 +33,7 @@ suite('req2params', function() { assert.ok(_.isObject(req.query), 'request has query'); assert.ok(!req.query.hasOwnProperty('dbuser'), 'dbuser was removed from query'); assert.ok(req.hasOwnProperty('params'), 'request has params'); - assert.ok(req.params.hasOwnProperty('interactivity'), 'request params have interactivity'); + assert.ok(!req.params.hasOwnProperty('interactivity'), 'request params have interactivity'); // database_name for user "localhost" (see test/support/prepare_db.sh) assert.equal(req.params.dbname, 'cartodb_test_user_1_db'); // unauthenticated request gets no dbuser @@ -49,7 +49,7 @@ suite('req2params', function() { assert.ok(_.isObject(req.query), 'request has query'); assert.ok(!req.query.hasOwnProperty('dbuser'), 'dbuser was removed from query'); assert.ok(req.hasOwnProperty('params'), 'request has params'); - assert.ok(req.params.hasOwnProperty('interactivity'), 'request params have interactivity'); + assert.ok(!req.params.hasOwnProperty('interactivity'), 'request params have interactivity'); // database_name for user "localhost" (see test/support/prepare_db.sh) assert.equal(req.params.dbname, 'cartodb_test_user_1_db'); // id for user "localhost" (see test/support/prepare_db.sh) @@ -62,5 +62,17 @@ suite('req2params', function() { }); }); }); + + test('retains interactivity', function(done){ + opts.req2params({headers: { host:'localhost' }, query: {interactivity: 'fld'} }, function(err, req) { + if ( err ) { console.log(err); throw new Error(err); } + //console.dir(req); + assert.ok(_.isObject(req.query), 'request has query'); + assert.ok(!req.query.hasOwnProperty('dbuser'), 'dbuser was removed from query'); + assert.ok(req.hasOwnProperty('params'), 'request has params'); + assert.ok(req.params.hasOwnProperty('interactivity'), 'request params has no interactivity'); + done(); + }); + }); }); diff --git a/tools/munin/windshaft b/tools/munin/windshaft index 77adcb23..30adaf15 100755 --- a/tools/munin/windshaft +++ b/tools/munin/windshaft @@ -67,7 +67,7 @@ for pid in ${pids}; do log=$(grep "${pid}" "${tmpreport}" | grep -w 1w | awk '{print $9}') if test -e "${log}"; then - kill -USR1 "${pid}" + kill -USR2 "${pid}" cnt=$(tac ${log} | sed -n -e '/ItemKey/p;/^RenderCache/q' | wc -l) if test $cnt -gt $maxcache; then maxcache=$cnt; fi else