From 9807a5e12bec216b433a35b5cbaba038c1084225 Mon Sep 17 00:00:00 2001 From: Luis Bosque Date: Tue, 11 Jun 2013 10:53:07 +0200 Subject: [PATCH 01/18] Target v1.3.1 --- NEWS.md | 3 +++ package.json | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index dd2322a0..d53adbdc 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,6 @@ +1.3.1 +------ + 1.3.0 ------ * Change stats format for multilayer map token request, see diff --git a/package.json b/package.json index 332f5591..1635b448 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "1.3.0", + "version": "1.3.1", "description": "A map tile server for CartoDB", "url": "https://github.com/Vizzuality/Windshaft-cartodb", "licenses": [{ From d53327bc332c1cbe7515e5894bac906981e5e8f0 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Tue, 11 Jun 2013 17:13:50 +0200 Subject: [PATCH 02/18] Utility to fetch a token from a layergroup config It's in its infancy, still need to edit the script to use against remote service or non-standard local deploys --- tools/multilayer_token | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100755 tools/multilayer_token diff --git a/tools/multilayer_token b/tools/multilayer_token new file mode 100755 index 00000000..3a44e6e0 --- /dev/null +++ b/tools/multilayer_token @@ -0,0 +1,15 @@ +#!/bin/sh + +if test -z "$1"; then + echo "Usage: $0 " >&2 + exit 1 +fi + +cfg=$1 +port=8181 +user=dev +domain=localhost.lan +proto=http + +curl -k -H Content-Type:application/json --data-binary "@${cfg}" ${proto}://${user}.${domain}:${port}/tiles/layergroup/ + From c07616f889e4f52dd9c6af7c7c2d606ea68488b2 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Wed, 12 Jun 2013 18:30:32 +0200 Subject: [PATCH 03/18] Run silently, prepare for using against windshaft direct --- tools/multilayer_token | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/multilayer_token b/tools/multilayer_token index 3a44e6e0..6316be65 100755 --- a/tools/multilayer_token +++ b/tools/multilayer_token @@ -10,6 +10,10 @@ port=8181 user=dev domain=localhost.lan proto=http +path="tiles/layergroup" -curl -k -H Content-Type:application/json --data-binary "@${cfg}" ${proto}://${user}.${domain}:${port}/tiles/layergroup/ +# This is for direct windshaft connection +#port=8083 +#path="database/cartodb_dev_user_1_db/layergroup" +curl -skH Content-Type:application/json --data-binary "@${cfg}" ${proto}://${user}.${domain}:${port}/${path} From 3d4af14315edf64a759eeec3058bfa0e0d4d5e93 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Wed, 12 Jun 2013 18:31:38 +0200 Subject: [PATCH 04/18] Fix deadlock on layergroup post Required upgrading windshaft to 0.12.5. Took the chance to also upgrade redis dependency to latest stable. --- NEWS.md | 1 + npm-shrinkwrap.json | 149 +++++++++++++++++++++++++++++++++++++------- package.json | 4 +- 3 files changed, 131 insertions(+), 23 deletions(-) diff --git a/NEWS.md b/NEWS.md index d53adbdc..65c50384 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,6 @@ 1.3.1 ------ +* Fix deadlock on new style creation 1.3.0 ------ diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 1ae33564..c9d21768 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -1,6 +1,6 @@ { "name": "windshaft-cartodb", - "version": "1.3.0", + "version": "1.3.1", "dependencies": { "cluster2": { "version": "0.3.5-cdb02", @@ -13,7 +13,7 @@ "version": "1.9.2", "dependencies": { "formidable": { - "version": "1.0.13" + "version": "1.0.14" } } }, @@ -29,10 +29,10 @@ } }, "ejs": { - "version": "0.8.3" + "version": "0.8.4" }, "npm": { - "version": "1.2.17", + "version": "1.2.28", "dependencies": { "semver": { "version": "1.1.4" @@ -41,16 +41,16 @@ "version": "1.1.0" }, "slide": { - "version": "1.1.3" + "version": "1.1.4" }, "abbrev": { "version": "1.0.4" }, "graceful-fs": { - "version": "1.2.0" + "version": "1.2.1" }, "minimatch": { - "version": "0.2.11", + "version": "0.2.12", "dependencies": { "sigmund": { "version": "1.0.0" @@ -64,8 +64,93 @@ "version": "2.1.4" }, "request": { - "version": "2.9.203", - "from": "git://github.com/isaacs/request" + "version": "2.21.0", + "dependencies": { + "qs": { + "version": "0.6.5" + }, + "json-stringify-safe": { + "version": "4.0.0" + }, + "forever-agent": { + "version": "0.5.0" + }, + "tunnel-agent": { + "version": "0.3.0" + }, + "http-signature": { + "version": "0.9.11", + "dependencies": { + "assert-plus": { + "version": "0.1.2" + }, + "asn1": { + "version": "0.1.11" + }, + "ctype": { + "version": "0.5.2" + } + } + }, + "hawk": { + "version": "0.13.1", + "dependencies": { + "hoek": { + "version": "0.8.5" + }, + "boom": { + "version": "0.4.2", + "dependencies": { + "hoek": { + "version": "0.9.1" + } + } + }, + "cryptiles": { + "version": "0.2.1" + }, + "sntp": { + "version": "0.2.4", + "dependencies": { + "hoek": { + "version": "0.9.1" + } + } + } + } + }, + "aws-sign": { + "version": "0.3.0" + }, + "oauth-sign": { + "version": "0.3.0" + }, + "cookie-jar": { + "version": "0.3.0" + }, + "node-uuid": { + "version": "1.4.0" + }, + "mime": { + "version": "1.2.9" + }, + "form-data": { + "version": "0.0.8", + "dependencies": { + "combined-stream": { + "version": "0.0.4", + "dependencies": { + "delayed-stream": { + "version": "0.0.5" + } + } + }, + "async": { + "version": "0.2.9" + } + } + } + } }, "which": { "version": "1.0.5" @@ -98,7 +183,7 @@ "version": "2.3.0" }, "node-gyp": { - "version": "0.9.5" + "version": "0.10.0" }, "fstream-npm": { "version": "0.1.4", @@ -124,24 +209,34 @@ "version": "0.1.2" }, "npm-registry-client": { - "version": "0.2.18", + "version": "0.2.24", "dependencies": { "couch-login": { - "version": "0.1.15" + "version": "0.1.16" } } }, "read-package-json": { - "version": "0.3.0" + "version": "0.4.1", + "dependencies": { + "normalize-package-data": { + "version": "0.1.6", + "dependencies": { + "github-url-from-git": { + "version": "1.1.1" + } + } + } + } }, "read-installed": { "version": "0.1.1" }, "glob": { - "version": "3.1.21" + "version": "3.2.1" }, "init-package-json": { - "version": "0.0.6", + "version": "0.0.8", "dependencies": { "promzard": { "version": "0.2.0" @@ -152,7 +247,7 @@ "version": "0.0.3" }, "lockfile": { - "version": "0.3.0" + "version": "0.3.4" }, "retry": { "version": "0.6.0" @@ -161,10 +256,10 @@ "version": "1.1.1" }, "npmconf": { - "version": "0.0.23", + "version": "0.1.0", "dependencies": { "config-chain": { - "version": "1.1.5", + "version": "1.1.7", "dependencies": { "proto-list": { "version": "1.2.2" @@ -181,6 +276,18 @@ }, "cmd-shim": { "version": "1.1.0" + }, + "sha": { + "version": "1.0.1" + }, + "editor": { + "version": "0.0.4" + }, + "child-process-close": { + "version": "0.1.1" + }, + "npm-user-validate": { + "version": "0.0.1" } } } @@ -193,10 +300,10 @@ "version": "1.3.3" }, "windshaft": { - "version": "0.12.4", + "version": "0.12.5", "dependencies": { "grainstore": { - "version": "0.13.3", + "version": "0.13.4", "dependencies": { "carto": { "version": "0.9.3-cdb3", @@ -330,7 +437,7 @@ "version": "2.0.3" }, "redis": { - "version": "0.7.2" + "version": "0.8.3" }, "hiredis": { "version": "0.1.15", diff --git a/package.json b/package.json index 1635b448..82f37c9b 100644 --- a/package.json +++ b/package.json @@ -21,10 +21,10 @@ "cluster2": "git://github.com/CartoDB/cluster2.git#cdb_production", "node-varnish": "0.1.1", "underscore" : "~1.3.3", - "windshaft" : "~0.12.4", + "windshaft" : "~0.12.5", "step": "0.0.x", "generic-pool": "~2.0.3", - "redis": "0.7.2", + "redis": "~0.8.3", "hiredis": "~0.1.14", "request": "2.9.202", "mapnik": "~0.7.14", From 6cb8c85da037d92980bcffd8c9dabacfa9b50002 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Thu, 13 Jun 2013 10:43:09 +0200 Subject: [PATCH 05/18] Only return token by default and full response on request (-v) --- tools/multilayer_token | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/tools/multilayer_token b/tools/multilayer_token index 6316be65..8faa999a 100755 --- a/tools/multilayer_token +++ b/tools/multilayer_token @@ -1,11 +1,21 @@ #!/bin/sh -if test -z "$1"; then - echo "Usage: $0 " >&2 +verbose=no + +while test -n "$1"; do + if test "$1" = "-v"; then + verbose=yes + else + cfg=$1 + fi + shift +done + +if test -z "$cfg"; then + echo "Usage: $0 [-v] " >&2 exit 1 fi -cfg=$1 port=8181 user=dev domain=localhost.lan @@ -16,4 +26,11 @@ path="tiles/layergroup" #port=8083 #path="database/cartodb_dev_user_1_db/layergroup" -curl -skH Content-Type:application/json --data-binary "@${cfg}" ${proto}://${user}.${domain}:${port}/${path} +res=`curl -skH Content-Type:application/json --data-binary "@${cfg}" ${proto}://${user}.${domain}:${port}/${path}` + +if test x${verbose} = xyes; then + echo "${res}" +fi + +tok=`echo "$res" | sed 's/.*"layergroupid":"\([^"]*\)".*/\1/'` +echo $tok From 850d4cd6ba6bf617d01f823172ce93a09fdc0efb Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Mon, 17 Jun 2013 11:57:35 +0200 Subject: [PATCH 06/18] Drop unused cluster support --- cluster.js | 64 ---------- npm-shrinkwrap.json | 291 -------------------------------------------- package.json | 1 - 3 files changed, 356 deletions(-) delete mode 100755 cluster.js diff --git a/cluster.js b/cluster.js deleted file mode 100755 index 84ac8040..00000000 --- a/cluster.js +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Windshaft-CartoDB - * =============== - * - * ./app.js [environment] - * - * environments: [development, production] - */ - -var Cluster = require('cluster2'); - -// sanity check -var ENV = process.argv[2] -if (ENV != 'development' && ENV != 'production' && ENV != 'staging'){ - console.error("\nnode app.js [environment]"); - console.error("environments: [development, production, staging]\n"); - process.exit(1); -} - -var _ = require('underscore') - , Step = require('step') - , CartodbWindshaft = require('./lib/cartodb/cartodb_windshaft'); - - -// set environment specific variables -global.settings = require(__dirname + '/config/settings'); -global.environment = require(__dirname + '/config/environments/' + ENV); -_.extend(global.settings, global.environment); - -// Include cart_data.js only _after_ the "global" variable is set -// See https://github.com/Vizzuality/Windshaft-cartodb/issues/28 -var cartoData = require('./lib/cartodb/carto_data'); - -var Windshaft = require('windshaft'); -var serverOptions = require('./lib/cartodb/server_options'); - -var ws = CartodbWindshaft(serverOptions); - -// Maximum number of connections for one process (see apps.js) -ws.maxConnections = global.environment.maxConnections || 128; - -//.use(cluster.logger('logs')) -//.use(cluster.stats()) -//.use(cluster.pidfiles('pids')) -var cluster = new Cluster({ - port: global.environment.port, - host: global.environment.host, - monPort: global.environment.port+1, - monHost: global.environment.host, - timeout: global.environment.socket_timeout, - noWorkers: 1 -}); - -console.log(new Date().toISOString() + " - " + process.pid + " - Windshaft cluster starting"); - -cluster.listen(function(cb) { - cb(ws); -}, function() { - console.log("Windshaft tileserver started on port " + global.environment.port); -}); - -process.on('SIGUSR1', function() { - ws.dumpCacheStats(); -}); diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index c9d21768..4e67ce78 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -2,297 +2,6 @@ "name": "windshaft-cartodb", "version": "1.3.1", "dependencies": { - "cluster2": { - "version": "0.3.5-cdb02", - "from": "git://github.com/CartoDB/cluster2.git#cdb_production", - "dependencies": { - "express": { - "version": "2.5.11", - "dependencies": { - "connect": { - "version": "1.9.2", - "dependencies": { - "formidable": { - "version": "1.0.14" - } - } - }, - "mime": { - "version": "1.2.4" - }, - "qs": { - "version": "0.4.2" - }, - "mkdirp": { - "version": "0.3.0" - } - } - }, - "ejs": { - "version": "0.8.4" - }, - "npm": { - "version": "1.2.28", - "dependencies": { - "semver": { - "version": "1.1.4" - }, - "ini": { - "version": "1.1.0" - }, - "slide": { - "version": "1.1.4" - }, - "abbrev": { - "version": "1.0.4" - }, - "graceful-fs": { - "version": "1.2.1" - }, - "minimatch": { - "version": "0.2.12", - "dependencies": { - "sigmund": { - "version": "1.0.0" - } - } - }, - "nopt": { - "version": "2.1.1" - }, - "rimraf": { - "version": "2.1.4" - }, - "request": { - "version": "2.21.0", - "dependencies": { - "qs": { - "version": "0.6.5" - }, - "json-stringify-safe": { - "version": "4.0.0" - }, - "forever-agent": { - "version": "0.5.0" - }, - "tunnel-agent": { - "version": "0.3.0" - }, - "http-signature": { - "version": "0.9.11", - "dependencies": { - "assert-plus": { - "version": "0.1.2" - }, - "asn1": { - "version": "0.1.11" - }, - "ctype": { - "version": "0.5.2" - } - } - }, - "hawk": { - "version": "0.13.1", - "dependencies": { - "hoek": { - "version": "0.8.5" - }, - "boom": { - "version": "0.4.2", - "dependencies": { - "hoek": { - "version": "0.9.1" - } - } - }, - "cryptiles": { - "version": "0.2.1" - }, - "sntp": { - "version": "0.2.4", - "dependencies": { - "hoek": { - "version": "0.9.1" - } - } - } - } - }, - "aws-sign": { - "version": "0.3.0" - }, - "oauth-sign": { - "version": "0.3.0" - }, - "cookie-jar": { - "version": "0.3.0" - }, - "node-uuid": { - "version": "1.4.0" - }, - "mime": { - "version": "1.2.9" - }, - "form-data": { - "version": "0.0.8", - "dependencies": { - "combined-stream": { - "version": "0.0.4", - "dependencies": { - "delayed-stream": { - "version": "0.0.5" - } - } - }, - "async": { - "version": "0.2.9" - } - } - } - } - }, - "which": { - "version": "1.0.5" - }, - "tar": { - "version": "0.1.17" - }, - "fstream": { - "version": "0.1.22" - }, - "block-stream": { - "version": "0.0.6" - }, - "inherits": { - "version": "1.0.0", - "from": "git://github.com/isaacs/inherits" - }, - "mkdirp": { - "version": "0.3.5" - }, - "read": { - "version": "1.0.4", - "dependencies": { - "mute-stream": { - "version": "0.0.3" - } - } - }, - "lru-cache": { - "version": "2.3.0" - }, - "node-gyp": { - "version": "0.10.0" - }, - "fstream-npm": { - "version": "0.1.4", - "dependencies": { - "fstream-ignore": { - "version": "0.0.6" - } - } - }, - "uid-number": { - "version": "0.0.3" - }, - "archy": { - "version": "0.0.2" - }, - "chownr": { - "version": "0.0.1" - }, - "npmlog": { - "version": "0.0.2" - }, - "ansi": { - "version": "0.1.2" - }, - "npm-registry-client": { - "version": "0.2.24", - "dependencies": { - "couch-login": { - "version": "0.1.16" - } - } - }, - "read-package-json": { - "version": "0.4.1", - "dependencies": { - "normalize-package-data": { - "version": "0.1.6", - "dependencies": { - "github-url-from-git": { - "version": "1.1.1" - } - } - } - } - }, - "read-installed": { - "version": "0.1.1" - }, - "glob": { - "version": "3.2.1" - }, - "init-package-json": { - "version": "0.0.8", - "dependencies": { - "promzard": { - "version": "0.2.0" - } - } - }, - "osenv": { - "version": "0.0.3" - }, - "lockfile": { - "version": "0.3.4" - }, - "retry": { - "version": "0.6.0" - }, - "once": { - "version": "1.1.1" - }, - "npmconf": { - "version": "0.1.0", - "dependencies": { - "config-chain": { - "version": "1.1.7", - "dependencies": { - "proto-list": { - "version": "1.2.2" - } - } - } - } - }, - "opener": { - "version": "1.3.0" - }, - "chmodr": { - "version": "0.1.0" - }, - "cmd-shim": { - "version": "1.1.0" - }, - "sha": { - "version": "1.0.1" - }, - "editor": { - "version": "0.0.4" - }, - "child-process-close": { - "version": "0.1.1" - }, - "npm-user-validate": { - "version": "0.0.1" - } - } - } - } - }, "node-varnish": { "version": "0.1.1" }, diff --git a/package.json b/package.json index 82f37c9b..65306cd8 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,6 @@ "email": "simon@vizzuality.com" }, "dependencies": { - "cluster2": "git://github.com/CartoDB/cluster2.git#cdb_production", "node-varnish": "0.1.1", "underscore" : "~1.3.3", "windshaft" : "~0.12.5", From eb83851bb7aed11e99e606cd8d960132a35998e3 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Mon, 17 Jun 2013 17:24:09 +0200 Subject: [PATCH 07/18] Fix database authentication with multi-table layergroups --- NEWS.md | 1 + lib/cartodb/server_options.js | 2 +- test/acceptance/multilayer.js | 118 ++++++++++++++++++++++++++++++++++ 3 files changed, 120 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 65c50384..eb630b09 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,7 @@ 1.3.1 ------ * Fix deadlock on new style creation +* Fix database authentication with multi-table layergroups 1.3.0 ------ diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index 6693ece7..80c2521d 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -341,7 +341,7 @@ module.exports = function(){ req.params.processXML = function(req, xml, callback) { var dbuser = req.dbuser ? req.dbuser : global.settings.postgres.user; - if ( ! me.rx_dbuser ) me.rx_dbuser = /(<\/Parameter>)/; + if ( ! me.rx_dbuser ) me.rx_dbuser = /(<\/Parameter>)/g; xml = xml.replace(me.rx_dbuser, "$1" + dbuser + "$2"); callback(null, xml); } diff --git a/test/acceptance/multilayer.js b/test/acceptance/multilayer.js index 4bd08e8d..993708d0 100644 --- a/test/acceptance/multilayer.js +++ b/test/acceptance/multilayer.js @@ -456,6 +456,124 @@ suite('multilayer', function() { }); }); + test("layergroup with 2 private-table layers", function(done) { + + var layergroup = { + version: '1.0.0', + layers: [ + { options: { + sql: 'select * from test_table_private_1 where cartodb_id=1', + cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', + cartocss_version: '2.1.1', + interactivity: 'cartodb_id' + } }, + { options: { + sql: 'select * from test_table_private_1 where cartodb_id=2', + cartocss: '#layer { marker-fill:blue; marker-allow-overlap:true; }', + cartocss_version: '2.1.1', + interactivity: 'cartodb_id' + } } + ] + }; + + var expected_token = "50cb56d0ebe9142ca4ed97bc8dac3ee1"; + Step( + function do_post() + { + var next = this; + assert.response(server, { + url: '/tiles/layergroup?map_key=1234', + method: 'POST', + headers: {host: 'localhost', 'Content-Type': 'application/json' }, + data: JSON.stringify(layergroup) + }, {}, function(res) { + assert.equal(res.statusCode, 200, res.body); + var parsedBody = JSON.parse(res.body); + var expectedBody = { layergroupid: expected_token }; + // check last modified + var qTables = JSON.stringify({ + 'q': 'SELECT CDB_QueryTables($windshaft$' + + layergroup.layers[0].options.sql + ';' + + layergroup.layers[1].options.sql + + '$windshaft$)' + }); + assert.equal(parsedBody.last_updated, expected_last_updated); + if ( expected_token ) { + assert.equal(parsedBody.layergroupid, expected_token + ':' + expected_last_updated_epoch); + } + else expected_token = parsedBody.layergroupid; + next(null, res); + }); + }, + function do_get_tile(err) + { + if ( err ) throw err; + var next = this; + assert.response(server, { + url: '/tiles/layergroup/' + expected_token + ':cb0/0/0/0.png?map_key=1234', + method: 'GET', + headers: {host: 'localhost' }, + encoding: 'binary' + }, {}, function(res) { + assert.equal(res.statusCode, 200, res.body); + assert.equal(res.headers['content-type'], "image/png"); + + // Check X-Cache-Channel + var cc = res.headers['x-cache-channel']; + assert.ok(cc); + var dbname = 'cartodb_test_user_1_db' + assert.equal(cc.substring(0, dbname.length), dbname); + next(err); + }); + }, + function do_get_grid_layer0(err) + { + if ( err ) throw err; + var next = this; + assert.response(server, { + url: '/tiles/layergroup/' + expected_token + + '/0/0/0/0.grid.json?map_key=1234', + headers: {host: 'localhost' }, + method: 'GET' + }, {}, function(res) { + assert.equal(res.statusCode, 200, res.body); + next(err); + }); + }, + function do_get_grid_layer1(err) + { + if ( err ) throw err; + var next = this; + assert.response(server, { + url: '/tiles/layergroup/' + expected_token + + '/1/0/0/0.grid.json?map_key=1234', + headers: {host: 'localhost' }, + method: 'GET' + }, {}, function(res) { + assert.equal(res.statusCode, 200, res.body); + assert.equal(res.headers['content-type'], "text/javascript; charset=utf-8; charset=utf-8"); + next(err); + }); + }, + function finish(err) { + var errors = []; + if ( err ) { + errors.push(err.message); + console.log("Error: " + err); + } + redis_client.keys("map_style|cartodb_test_user_1_db|~" + expected_token, function(err, matches) { + if ( err ) errors.push(err.message); + assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches); + redis_client.del(matches, function(err) { + if ( err ) errors.push(err.message); + if ( errors.length ) done(new Error(errors)); + else done(null); + }); + }); + } + ); + }); + suiteTeardown(function(done) { // This test will add map_style records, like From e984811eaea54b0f040b7eecd8a54aa68ed302fa Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Wed, 19 Jun 2013 17:24:01 +0200 Subject: [PATCH 08/18] Fix SQL bug in testcase --- test/acceptance/multilayer.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/multilayer.js b/test/acceptance/multilayer.js index 993708d0..c6daa746 100644 --- a/test/acceptance/multilayer.js +++ b/test/acceptance/multilayer.js @@ -336,7 +336,7 @@ suite('multilayer', function() { version: '1.0.0', layers: [ { options: { - sql: 'select 1 as cartodb_id, !pixel_height! as h' + sql: 'select 1 as cartodb_id, !pixel_height! as h, ' + 'ST_Buffer(!bbox!, -32*greatest(!pixel_width!,!pixel_height!)) as the_geom_webmercator', cartocss: '#layer { polygon-fill:red; }', cartocss_version: '2.0.1' From 4b1d6cd729e6aff6d33ca6614ed354f29f83fa70 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Fri, 21 Jun 2013 12:46:13 +0200 Subject: [PATCH 09/18] Add tile and grid fetching checks at layergroup creation time Basically requires windshaft 0.12.6, which implements this --- NEWS.md | 1 + npm-shrinkwrap.json | 44 ++++++++++++++++++++++++++++++++++++++++++-- package.json | 2 +- 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/NEWS.md b/NEWS.md index eb630b09..3b451a59 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,6 +2,7 @@ ------ * Fix deadlock on new style creation * Fix database authentication with multi-table layergroups +* Add tile and grid fetching checks at layergroup creation time 1.3.0 ------ diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 4e67ce78..1e952a06 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -9,7 +9,7 @@ "version": "1.3.3" }, "windshaft": { - "version": "0.12.5", + "version": "0.12.6", "dependencies": { "grainstore": { "version": "0.13.4", @@ -72,7 +72,47 @@ "version": "0.3.4" }, "sqlite3": { - "version": "2.1.7" + "version": "2.1.10", + "dependencies": { + "tar.gz": { + "version": "0.1.1", + "dependencies": { + "fstream": { + "version": "0.1.22", + "dependencies": { + "rimraf": { + "version": "2.1.4" + }, + "graceful-fs": { + "version": "1.2.2" + }, + "inherits": { + "version": "1.0.0" + } + } + }, + "tar": { + "version": "0.1.17", + "dependencies": { + "inherits": { + "version": "1.0.0" + }, + "block-stream": { + "version": "0.0.6" + } + } + }, + "commander": { + "version": "1.1.1", + "dependencies": { + "keypress": { + "version": "0.1.0" + } + } + } + } + } + } }, "mime": { "version": "1.2.9" diff --git a/package.json b/package.json index 65306cd8..771aedb6 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "dependencies": { "node-varnish": "0.1.1", "underscore" : "~1.3.3", - "windshaft" : "~0.12.5", + "windshaft" : "~0.12.6", "step": "0.0.x", "generic-pool": "~2.0.3", "redis": "~0.8.3", From 10a66fbd660f8012cdb84296c8ec2629f614dfe7 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Wed, 26 Jun 2013 13:26:53 +0200 Subject: [PATCH 10/18] Fix SQL error reporting to NOT split on newline --- NEWS.md | 1 + npm-shrinkwrap.json | 4 ++-- package.json | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/NEWS.md b/NEWS.md index 3b451a59..49c05f99 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,6 +3,7 @@ * Fix deadlock on new style creation * Fix database authentication with multi-table layergroups * Add tile and grid fetching checks at layergroup creation time +* Fix SQL error reporting to NOT split on newline 1.3.0 ------ diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 1e952a06..801c08bf 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -9,7 +9,7 @@ "version": "1.3.3" }, "windshaft": { - "version": "0.12.6", + "version": "0.12.7", "dependencies": { "grainstore": { "version": "0.13.4", @@ -81,7 +81,7 @@ "version": "0.1.22", "dependencies": { "rimraf": { - "version": "2.1.4" + "version": "2.2.0" }, "graceful-fs": { "version": "1.2.2" diff --git a/package.json b/package.json index 771aedb6..2d4b33a0 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "dependencies": { "node-varnish": "0.1.1", "underscore" : "~1.3.3", - "windshaft" : "~0.12.6", + "windshaft" : "~0.12.7", "step": "0.0.x", "generic-pool": "~2.0.3", "redis": "~0.8.3", From d7b1ff9a8081bde64f4d3a6129e12ff2ee93b678 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Wed, 26 Jun 2013 16:26:02 +0200 Subject: [PATCH 11/18] Set default layergroup ttl locally --- lib/cartodb/server_options.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index 80c2521d..4938c709 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -22,7 +22,8 @@ module.exports = function(){ grainstore: { datasource: global.environment.postgres, cachedir: global.environment.millstone.cache_basedir, - mapnik_version: global.environment.mapnik_version || mapnik.versions.mapnik + mapnik_version: global.environment.mapnik_version || mapnik.versions.mapnik, + default_layergroup_ttl: 300 // seconds }, mapnik: { metatile: rendererConfig.metatile, From 632d75a7c84db7e40f04a143c6576116cc0e4a56 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Fri, 28 Jun 2013 17:58:11 +0200 Subject: [PATCH 12/18] specify units for rendererConfig.cache_ttl --- lib/cartodb/server_options.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index 4938c709..6cf28c66 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -11,7 +11,7 @@ var _ = require('underscore') module.exports = function(){ var rendererConfig = _.defaults(global.environment.renderer || {}, { - cache_ttl: 60000, + cache_ttl: 60000, // milliseconds metatile: 4, bufferSize: 64 }); From 00c18ab8dd2fdb1d298cd014531b7328cf007936 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Fri, 28 Jun 2013 19:05:39 +0200 Subject: [PATCH 13/18] Raise windshaft dependency importing following fixes: - support for CartoCSS attachments (#layer0::label) - only check layergroup validity once - use higher zoom level for checking layergroup validity --- NEWS.md | 1 + npm-shrinkwrap.json | 7 +++++-- package.json | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/NEWS.md b/NEWS.md index 49c05f99..89a5770f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -4,6 +4,7 @@ * Fix database authentication with multi-table layergroups * Add tile and grid fetching checks at layergroup creation time * Fix SQL error reporting to NOT split on newline +* Fix support for CartoCSS attachments 1.3.0 ------ diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 801c08bf..a4ce046a 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -9,10 +9,10 @@ "version": "1.3.3" }, "windshaft": { - "version": "0.12.7", + "version": "0.12.8", "dependencies": { "grainstore": { - "version": "0.13.4", + "version": "0.13.5", "dependencies": { "carto": { "version": "0.9.3-cdb3", @@ -176,6 +176,9 @@ "version": "1.0.2" } } + }, + "lru-cache": { + "version": "2.3.0" } } }, diff --git a/package.json b/package.json index 2d4b33a0..4b4828fb 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "dependencies": { "node-varnish": "0.1.1", "underscore" : "~1.3.3", - "windshaft" : "~0.12.7", + "windshaft" : "~0.12.8", "step": "0.0.x", "generic-pool": "~2.0.3", "redis": "~0.8.3", From 6416af3f169db6919a482bf78a202df40aa6d189 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Wed, 3 Jul 2013 09:59:42 +0200 Subject: [PATCH 14/18] Add support for listing all map styles that belong to a user Also report user database name, as a facility --- tools/show_style | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/tools/show_style b/tools/show_style index d3b9163b..b073f78a 100755 --- a/tools/show_style +++ b/tools/show_style @@ -5,13 +5,25 @@ REDIS_PORT=6379 # default port -if test -z "$2"; then - echo "Usage: $0 " >&2 +if test -z "$1"; then + echo "Usage: $0 [|~]" >&2 exit 1 fi username="$1" -tabname="$2" +token="$2" dbname=`redis-cli -p ${REDIS_PORT} -n 5 hget "rails:users:${username}" "database_name"` -redis-cli get "map_style|${dbname}|${tabname}" | sed -e 's/\\n/\n/g' -e 's/\\//g' +if test $? -ne 0; then + exit 1 +fi +if test -z "${dbname}"; then + echo "Username ${username} unknown by redis (try CARTODB/script/restore_redis?)" >&2 + exit 1 +fi +echo "Database name for user ${username}: ${dbname}" # only if verbose? +if test -n "$token"; then + redis-cli get "map_style|${dbname}|${token}" | sed -e 's/\\n/\n/g' -e 's/\\//g' +else + redis-cli keys "map_style|${dbname}|*" +fi From 8c51c97102c4f2279ed82e55640df6dc989c6d02 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Thu, 4 Jul 2013 11:04:12 +0200 Subject: [PATCH 15/18] Be more verbose about curl errors --- tools/multilayer_token | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/multilayer_token b/tools/multilayer_token index 8faa999a..e9d88c53 100755 --- a/tools/multilayer_token +++ b/tools/multilayer_token @@ -26,7 +26,11 @@ path="tiles/layergroup" #port=8083 #path="database/cartodb_dev_user_1_db/layergroup" -res=`curl -skH Content-Type:application/json --data-binary "@${cfg}" ${proto}://${user}.${domain}:${port}/${path}` +cmd="curl -vskH Content-Type:application/json --data-binary @${cfg} ${proto}://${user}.${domain}:${port}/${path}" +res=`${cmd}` +if test $? -gt 0; then + echo "curl command failed: ${cmd}" +fi if test x${verbose} = xyes; then echo "${res}" From 563764dc4ff0ee9ca45230a87878d0d61629b7ed Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Thu, 4 Jul 2013 11:15:21 +0200 Subject: [PATCH 16/18] Only make curl verbose when -v is passed to commandline --- tools/multilayer_token | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/multilayer_token b/tools/multilayer_token index e9d88c53..727fbf94 100755 --- a/tools/multilayer_token +++ b/tools/multilayer_token @@ -26,7 +26,10 @@ path="tiles/layergroup" #port=8083 #path="database/cartodb_dev_user_1_db/layergroup" -cmd="curl -vskH Content-Type:application/json --data-binary @${cfg} ${proto}://${user}.${domain}:${port}/${path}" +cmd="curl -skH Content-Type:application/json --data-binary @${cfg} ${proto}://${user}.${domain}:${port}/${path}" +if test x${verbose} = xyes; then + cmd="${cmd} -v" +fi res=`${cmd}` if test $? -gt 0; then echo "curl command failed: ${cmd}" From 316c9c209d174f49e7ce743dd93181d8ff67d9f3 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Thu, 4 Jul 2013 12:50:57 +0200 Subject: [PATCH 17/18] Add support for specifying tiler url --- tools/multilayer_token | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/tools/multilayer_token b/tools/multilayer_token index 727fbf94..a37170c0 100755 --- a/tools/multilayer_token +++ b/tools/multilayer_token @@ -1,32 +1,29 @@ #!/bin/sh verbose=no +tiler_url=http://dev.localhost.lan:8181/tiles/layergroup + +# This is for direct windshaft connection +#tiler_url=http://dev.localhost.lan:8083/database/cartodb_dev_user_1_db/layergroup while test -n "$1"; do if test "$1" = "-v"; then verbose=yes + elif test -z "$cfg"; then + cfg="$1" else - cfg=$1 + tiler_url="$1" fi shift done if test -z "$cfg"; then - echo "Usage: $0 [-v] " >&2 + echo "Usage: $0 [-v] []" >&2 + echo "Default is ${tiler_url}" >&2 exit 1 fi -port=8181 -user=dev -domain=localhost.lan -proto=http -path="tiles/layergroup" - -# This is for direct windshaft connection -#port=8083 -#path="database/cartodb_dev_user_1_db/layergroup" - -cmd="curl -skH Content-Type:application/json --data-binary @${cfg} ${proto}://${user}.${domain}:${port}/${path}" +cmd="curl -skH Content-Type:application/json --data-binary @${cfg} ${tiler_url}" if test x${verbose} = xyes; then cmd="${cmd} -v" fi From 1c65cec6edeac23e9973f81b1905c387d3a0b110 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Thu, 4 Jul 2013 17:05:30 +0200 Subject: [PATCH 18/18] Do not consider broken layergroup configs as good on second look The fix is really in Windshaft, this commit simply requires a newer version of it. --- npm-shrinkwrap.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index a4ce046a..68e156f4 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -9,7 +9,7 @@ "version": "1.3.3" }, "windshaft": { - "version": "0.12.8", + "version": "0.12.9", "dependencies": { "grainstore": { "version": "0.13.5", diff --git a/package.json b/package.json index 4b4828fb..44ca3ec7 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "dependencies": { "node-varnish": "0.1.1", "underscore" : "~1.3.3", - "windshaft" : "~0.12.8", + "windshaft" : "~0.12.9", "step": "0.0.x", "generic-pool": "~2.0.3", "redis": "~0.8.3",