diff --git a/NEWS.md b/NEWS.md index dd2322a0..89a5770f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,11 @@ +1.3.1 +------ +* Fix deadlock on new style creation +* Fix database authentication with multi-table layergroups +* Add tile and grid fetching checks at layergroup creation time +* Fix SQL error reporting to NOT split on newline +* Fix support for CartoCSS attachments + 1.3.0 ------ * Change stats format for multilayer map token request, see diff --git a/cluster.js b/cluster.js deleted file mode 100755 index 84ac8040..00000000 --- a/cluster.js +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Windshaft-CartoDB - * =============== - * - * ./app.js [environment] - * - * environments: [development, production] - */ - -var Cluster = require('cluster2'); - -// sanity check -var ENV = process.argv[2] -if (ENV != 'development' && ENV != 'production' && ENV != 'staging'){ - console.error("\nnode app.js [environment]"); - console.error("environments: [development, production, staging]\n"); - process.exit(1); -} - -var _ = require('underscore') - , Step = require('step') - , CartodbWindshaft = require('./lib/cartodb/cartodb_windshaft'); - - -// set environment specific variables -global.settings = require(__dirname + '/config/settings'); -global.environment = require(__dirname + '/config/environments/' + ENV); -_.extend(global.settings, global.environment); - -// Include cart_data.js only _after_ the "global" variable is set -// See https://github.com/Vizzuality/Windshaft-cartodb/issues/28 -var cartoData = require('./lib/cartodb/carto_data'); - -var Windshaft = require('windshaft'); -var serverOptions = require('./lib/cartodb/server_options'); - -var ws = CartodbWindshaft(serverOptions); - -// Maximum number of connections for one process (see apps.js) -ws.maxConnections = global.environment.maxConnections || 128; - -//.use(cluster.logger('logs')) -//.use(cluster.stats()) -//.use(cluster.pidfiles('pids')) -var cluster = new Cluster({ - port: global.environment.port, - host: global.environment.host, - monPort: global.environment.port+1, - monHost: global.environment.host, - timeout: global.environment.socket_timeout, - noWorkers: 1 -}); - -console.log(new Date().toISOString() + " - " + process.pid + " - Windshaft cluster starting"); - -cluster.listen(function(cb) { - cb(ws); -}, function() { - console.log("Windshaft tileserver started on port " + global.environment.port); -}); - -process.on('SIGUSR1', function() { - ws.dumpCacheStats(); -}); diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index 6693ece7..6cf28c66 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -11,7 +11,7 @@ var _ = require('underscore') module.exports = function(){ var rendererConfig = _.defaults(global.environment.renderer || {}, { - cache_ttl: 60000, + cache_ttl: 60000, // milliseconds metatile: 4, bufferSize: 64 }); @@ -22,7 +22,8 @@ module.exports = function(){ grainstore: { datasource: global.environment.postgres, cachedir: global.environment.millstone.cache_basedir, - mapnik_version: global.environment.mapnik_version || mapnik.versions.mapnik + mapnik_version: global.environment.mapnik_version || mapnik.versions.mapnik, + default_layergroup_ttl: 300 // seconds }, mapnik: { metatile: rendererConfig.metatile, @@ -341,7 +342,7 @@ module.exports = function(){ req.params.processXML = function(req, xml, callback) { var dbuser = req.dbuser ? req.dbuser : global.settings.postgres.user; - if ( ! me.rx_dbuser ) me.rx_dbuser = /(<\/Parameter>)/; + if ( ! me.rx_dbuser ) me.rx_dbuser = /(<\/Parameter>)/g; xml = xml.replace(me.rx_dbuser, "$1" + dbuser + "$2"); callback(null, xml); } diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 1ae33564..68e156f4 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -1,191 +1,7 @@ { "name": "windshaft-cartodb", - "version": "1.3.0", + "version": "1.3.1", "dependencies": { - "cluster2": { - "version": "0.3.5-cdb02", - "from": "git://github.com/CartoDB/cluster2.git#cdb_production", - "dependencies": { - "express": { - "version": "2.5.11", - "dependencies": { - "connect": { - "version": "1.9.2", - "dependencies": { - "formidable": { - "version": "1.0.13" - } - } - }, - "mime": { - "version": "1.2.4" - }, - "qs": { - "version": "0.4.2" - }, - "mkdirp": { - "version": "0.3.0" - } - } - }, - "ejs": { - "version": "0.8.3" - }, - "npm": { - "version": "1.2.17", - "dependencies": { - "semver": { - "version": "1.1.4" - }, - "ini": { - "version": "1.1.0" - }, - "slide": { - "version": "1.1.3" - }, - "abbrev": { - "version": "1.0.4" - }, - "graceful-fs": { - "version": "1.2.0" - }, - "minimatch": { - "version": "0.2.11", - "dependencies": { - "sigmund": { - "version": "1.0.0" - } - } - }, - "nopt": { - "version": "2.1.1" - }, - "rimraf": { - "version": "2.1.4" - }, - "request": { - "version": "2.9.203", - "from": "git://github.com/isaacs/request" - }, - "which": { - "version": "1.0.5" - }, - "tar": { - "version": "0.1.17" - }, - "fstream": { - "version": "0.1.22" - }, - "block-stream": { - "version": "0.0.6" - }, - "inherits": { - "version": "1.0.0", - "from": "git://github.com/isaacs/inherits" - }, - "mkdirp": { - "version": "0.3.5" - }, - "read": { - "version": "1.0.4", - "dependencies": { - "mute-stream": { - "version": "0.0.3" - } - } - }, - "lru-cache": { - "version": "2.3.0" - }, - "node-gyp": { - "version": "0.9.5" - }, - "fstream-npm": { - "version": "0.1.4", - "dependencies": { - "fstream-ignore": { - "version": "0.0.6" - } - } - }, - "uid-number": { - "version": "0.0.3" - }, - "archy": { - "version": "0.0.2" - }, - "chownr": { - "version": "0.0.1" - }, - "npmlog": { - "version": "0.0.2" - }, - "ansi": { - "version": "0.1.2" - }, - "npm-registry-client": { - "version": "0.2.18", - "dependencies": { - "couch-login": { - "version": "0.1.15" - } - } - }, - "read-package-json": { - "version": "0.3.0" - }, - "read-installed": { - "version": "0.1.1" - }, - "glob": { - "version": "3.1.21" - }, - "init-package-json": { - "version": "0.0.6", - "dependencies": { - "promzard": { - "version": "0.2.0" - } - } - }, - "osenv": { - "version": "0.0.3" - }, - "lockfile": { - "version": "0.3.0" - }, - "retry": { - "version": "0.6.0" - }, - "once": { - "version": "1.1.1" - }, - "npmconf": { - "version": "0.0.23", - "dependencies": { - "config-chain": { - "version": "1.1.5", - "dependencies": { - "proto-list": { - "version": "1.2.2" - } - } - } - } - }, - "opener": { - "version": "1.3.0" - }, - "chmodr": { - "version": "0.1.0" - }, - "cmd-shim": { - "version": "1.1.0" - } - } - } - } - }, "node-varnish": { "version": "0.1.1" }, @@ -193,10 +9,10 @@ "version": "1.3.3" }, "windshaft": { - "version": "0.12.4", + "version": "0.12.9", "dependencies": { "grainstore": { - "version": "0.13.3", + "version": "0.13.5", "dependencies": { "carto": { "version": "0.9.3-cdb3", @@ -256,7 +72,47 @@ "version": "0.3.4" }, "sqlite3": { - "version": "2.1.7" + "version": "2.1.10", + "dependencies": { + "tar.gz": { + "version": "0.1.1", + "dependencies": { + "fstream": { + "version": "0.1.22", + "dependencies": { + "rimraf": { + "version": "2.2.0" + }, + "graceful-fs": { + "version": "1.2.2" + }, + "inherits": { + "version": "1.0.0" + } + } + }, + "tar": { + "version": "0.1.17", + "dependencies": { + "inherits": { + "version": "1.0.0" + }, + "block-stream": { + "version": "0.0.6" + } + } + }, + "commander": { + "version": "1.1.1", + "dependencies": { + "keypress": { + "version": "0.1.0" + } + } + } + } + } + } }, "mime": { "version": "1.2.9" @@ -320,6 +176,9 @@ "version": "1.0.2" } } + }, + "lru-cache": { + "version": "2.3.0" } } }, @@ -330,7 +189,7 @@ "version": "2.0.3" }, "redis": { - "version": "0.7.2" + "version": "0.8.3" }, "hiredis": { "version": "0.1.15", diff --git a/package.json b/package.json index 332f5591..44ca3ec7 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "1.3.0", + "version": "1.3.1", "description": "A map tile server for CartoDB", "url": "https://github.com/Vizzuality/Windshaft-cartodb", "licenses": [{ @@ -18,13 +18,12 @@ "email": "simon@vizzuality.com" }, "dependencies": { - "cluster2": "git://github.com/CartoDB/cluster2.git#cdb_production", "node-varnish": "0.1.1", "underscore" : "~1.3.3", - "windshaft" : "~0.12.4", + "windshaft" : "~0.12.9", "step": "0.0.x", "generic-pool": "~2.0.3", - "redis": "0.7.2", + "redis": "~0.8.3", "hiredis": "~0.1.14", "request": "2.9.202", "mapnik": "~0.7.14", diff --git a/test/acceptance/multilayer.js b/test/acceptance/multilayer.js index 4bd08e8d..c6daa746 100644 --- a/test/acceptance/multilayer.js +++ b/test/acceptance/multilayer.js @@ -336,7 +336,7 @@ suite('multilayer', function() { version: '1.0.0', layers: [ { options: { - sql: 'select 1 as cartodb_id, !pixel_height! as h' + sql: 'select 1 as cartodb_id, !pixel_height! as h, ' + 'ST_Buffer(!bbox!, -32*greatest(!pixel_width!,!pixel_height!)) as the_geom_webmercator', cartocss: '#layer { polygon-fill:red; }', cartocss_version: '2.0.1' @@ -456,6 +456,124 @@ suite('multilayer', function() { }); }); + test("layergroup with 2 private-table layers", function(done) { + + var layergroup = { + version: '1.0.0', + layers: [ + { options: { + sql: 'select * from test_table_private_1 where cartodb_id=1', + cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', + cartocss_version: '2.1.1', + interactivity: 'cartodb_id' + } }, + { options: { + sql: 'select * from test_table_private_1 where cartodb_id=2', + cartocss: '#layer { marker-fill:blue; marker-allow-overlap:true; }', + cartocss_version: '2.1.1', + interactivity: 'cartodb_id' + } } + ] + }; + + var expected_token = "50cb56d0ebe9142ca4ed97bc8dac3ee1"; + Step( + function do_post() + { + var next = this; + assert.response(server, { + url: '/tiles/layergroup?map_key=1234', + method: 'POST', + headers: {host: 'localhost', 'Content-Type': 'application/json' }, + data: JSON.stringify(layergroup) + }, {}, function(res) { + assert.equal(res.statusCode, 200, res.body); + var parsedBody = JSON.parse(res.body); + var expectedBody = { layergroupid: expected_token }; + // check last modified + var qTables = JSON.stringify({ + 'q': 'SELECT CDB_QueryTables($windshaft$' + + layergroup.layers[0].options.sql + ';' + + layergroup.layers[1].options.sql + + '$windshaft$)' + }); + assert.equal(parsedBody.last_updated, expected_last_updated); + if ( expected_token ) { + assert.equal(parsedBody.layergroupid, expected_token + ':' + expected_last_updated_epoch); + } + else expected_token = parsedBody.layergroupid; + next(null, res); + }); + }, + function do_get_tile(err) + { + if ( err ) throw err; + var next = this; + assert.response(server, { + url: '/tiles/layergroup/' + expected_token + ':cb0/0/0/0.png?map_key=1234', + method: 'GET', + headers: {host: 'localhost' }, + encoding: 'binary' + }, {}, function(res) { + assert.equal(res.statusCode, 200, res.body); + assert.equal(res.headers['content-type'], "image/png"); + + // Check X-Cache-Channel + var cc = res.headers['x-cache-channel']; + assert.ok(cc); + var dbname = 'cartodb_test_user_1_db' + assert.equal(cc.substring(0, dbname.length), dbname); + next(err); + }); + }, + function do_get_grid_layer0(err) + { + if ( err ) throw err; + var next = this; + assert.response(server, { + url: '/tiles/layergroup/' + expected_token + + '/0/0/0/0.grid.json?map_key=1234', + headers: {host: 'localhost' }, + method: 'GET' + }, {}, function(res) { + assert.equal(res.statusCode, 200, res.body); + next(err); + }); + }, + function do_get_grid_layer1(err) + { + if ( err ) throw err; + var next = this; + assert.response(server, { + url: '/tiles/layergroup/' + expected_token + + '/1/0/0/0.grid.json?map_key=1234', + headers: {host: 'localhost' }, + method: 'GET' + }, {}, function(res) { + assert.equal(res.statusCode, 200, res.body); + assert.equal(res.headers['content-type'], "text/javascript; charset=utf-8; charset=utf-8"); + next(err); + }); + }, + function finish(err) { + var errors = []; + if ( err ) { + errors.push(err.message); + console.log("Error: " + err); + } + redis_client.keys("map_style|cartodb_test_user_1_db|~" + expected_token, function(err, matches) { + if ( err ) errors.push(err.message); + assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches); + redis_client.del(matches, function(err) { + if ( err ) errors.push(err.message); + if ( errors.length ) done(new Error(errors)); + else done(null); + }); + }); + } + ); + }); + suiteTeardown(function(done) { // This test will add map_style records, like diff --git a/tools/multilayer_token b/tools/multilayer_token new file mode 100755 index 00000000..a37170c0 --- /dev/null +++ b/tools/multilayer_token @@ -0,0 +1,40 @@ +#!/bin/sh + +verbose=no +tiler_url=http://dev.localhost.lan:8181/tiles/layergroup + +# This is for direct windshaft connection +#tiler_url=http://dev.localhost.lan:8083/database/cartodb_dev_user_1_db/layergroup + +while test -n "$1"; do + if test "$1" = "-v"; then + verbose=yes + elif test -z "$cfg"; then + cfg="$1" + else + tiler_url="$1" + fi + shift +done + +if test -z "$cfg"; then + echo "Usage: $0 [-v] []" >&2 + echo "Default is ${tiler_url}" >&2 + exit 1 +fi + +cmd="curl -skH Content-Type:application/json --data-binary @${cfg} ${tiler_url}" +if test x${verbose} = xyes; then + cmd="${cmd} -v" +fi +res=`${cmd}` +if test $? -gt 0; then + echo "curl command failed: ${cmd}" +fi + +if test x${verbose} = xyes; then + echo "${res}" +fi + +tok=`echo "$res" | sed 's/.*"layergroupid":"\([^"]*\)".*/\1/'` +echo $tok diff --git a/tools/show_style b/tools/show_style index d3b9163b..b073f78a 100755 --- a/tools/show_style +++ b/tools/show_style @@ -5,13 +5,25 @@ REDIS_PORT=6379 # default port -if test -z "$2"; then - echo "Usage: $0 " >&2 +if test -z "$1"; then + echo "Usage: $0 [|~]" >&2 exit 1 fi username="$1" -tabname="$2" +token="$2" dbname=`redis-cli -p ${REDIS_PORT} -n 5 hget "rails:users:${username}" "database_name"` -redis-cli get "map_style|${dbname}|${tabname}" | sed -e 's/\\n/\n/g' -e 's/\\//g' +if test $? -ne 0; then + exit 1 +fi +if test -z "${dbname}"; then + echo "Username ${username} unknown by redis (try CARTODB/script/restore_redis?)" >&2 + exit 1 +fi +echo "Database name for user ${username}: ${dbname}" # only if verbose? +if test -n "$token"; then + redis-cli get "map_style|${dbname}|${token}" | sed -e 's/\\n/\n/g' -e 's/\\//g' +else + redis-cli keys "map_style|${dbname}|*" +fi