Merge branch 'release/staging'
This commit is contained in:
Luis Bosque
@@ -1,3 +1,11 @@
|
||||
1.3.1
|
||||
------
|
||||
* Fix deadlock on new style creation
|
||||
* Fix database authentication with multi-table layergroups
|
||||
* Add tile and grid fetching checks at layergroup creation time
|
||||
* Fix SQL error reporting to NOT split on newline
|
||||
* Fix support for CartoCSS attachments
|
||||
|
||||
1.3.0
|
||||
------
|
||||
* Change stats format for multilayer map token request, see
|
||||
|
||||
-64
@@ -1,64 +0,0 @@
|
||||
/*
|
||||
* Windshaft-CartoDB
|
||||
* ===============
|
||||
*
|
||||
* ./app.js [environment]
|
||||
*
|
||||
* environments: [development, production]
|
||||
*/
|
||||
|
||||
var Cluster = require('cluster2');
|
||||
|
||||
// sanity check
|
||||
var ENV = process.argv[2]
|
||||
if (ENV != 'development' && ENV != 'production' && ENV != 'staging'){
|
||||
console.error("\nnode app.js [environment]");
|
||||
console.error("environments: [development, production, staging]\n");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
var _ = require('underscore')
|
||||
, Step = require('step')
|
||||
, CartodbWindshaft = require('./lib/cartodb/cartodb_windshaft');
|
||||
|
||||
|
||||
// set environment specific variables
|
||||
global.settings = require(__dirname + '/config/settings');
|
||||
global.environment = require(__dirname + '/config/environments/' + ENV);
|
||||
_.extend(global.settings, global.environment);
|
||||
|
||||
// Include cart_data.js only _after_ the "global" variable is set
|
||||
// See https://github.com/Vizzuality/Windshaft-cartodb/issues/28
|
||||
var cartoData = require('./lib/cartodb/carto_data');
|
||||
|
||||
var Windshaft = require('windshaft');
|
||||
var serverOptions = require('./lib/cartodb/server_options');
|
||||
|
||||
var ws = CartodbWindshaft(serverOptions);
|
||||
|
||||
// Maximum number of connections for one process (see apps.js)
|
||||
ws.maxConnections = global.environment.maxConnections || 128;
|
||||
|
||||
//.use(cluster.logger('logs'))
|
||||
//.use(cluster.stats())
|
||||
//.use(cluster.pidfiles('pids'))
|
||||
var cluster = new Cluster({
|
||||
port: global.environment.port,
|
||||
host: global.environment.host,
|
||||
monPort: global.environment.port+1,
|
||||
monHost: global.environment.host,
|
||||
timeout: global.environment.socket_timeout,
|
||||
noWorkers: 1
|
||||
});
|
||||
|
||||
console.log(new Date().toISOString() + " - " + process.pid + " - Windshaft cluster starting");
|
||||
|
||||
cluster.listen(function(cb) {
|
||||
cb(ws);
|
||||
}, function() {
|
||||
console.log("Windshaft tileserver started on port " + global.environment.port);
|
||||
});
|
||||
|
||||
process.on('SIGUSR1', function() {
|
||||
ws.dumpCacheStats();
|
||||
});
|
||||
@@ -11,7 +11,7 @@ var _ = require('underscore')
|
||||
module.exports = function(){
|
||||
|
||||
var rendererConfig = _.defaults(global.environment.renderer || {}, {
|
||||
cache_ttl: 60000,
|
||||
cache_ttl: 60000, // milliseconds
|
||||
metatile: 4,
|
||||
bufferSize: 64
|
||||
});
|
||||
@@ -22,7 +22,8 @@ module.exports = function(){
|
||||
grainstore: {
|
||||
datasource: global.environment.postgres,
|
||||
cachedir: global.environment.millstone.cache_basedir,
|
||||
mapnik_version: global.environment.mapnik_version || mapnik.versions.mapnik
|
||||
mapnik_version: global.environment.mapnik_version || mapnik.versions.mapnik,
|
||||
default_layergroup_ttl: 300 // seconds
|
||||
},
|
||||
mapnik: {
|
||||
metatile: rendererConfig.metatile,
|
||||
@@ -341,7 +342,7 @@ module.exports = function(){
|
||||
|
||||
req.params.processXML = function(req, xml, callback) {
|
||||
var dbuser = req.dbuser ? req.dbuser : global.settings.postgres.user;
|
||||
if ( ! me.rx_dbuser ) me.rx_dbuser = /(<Parameter name="user"><!\[CDATA\[)[^\]]*(]]><\/Parameter>)/;
|
||||
if ( ! me.rx_dbuser ) me.rx_dbuser = /(<Parameter name="user"><!\[CDATA\[)[^\]]*(]]><\/Parameter>)/g;
|
||||
xml = xml.replace(me.rx_dbuser, "$1" + dbuser + "$2");
|
||||
callback(null, xml);
|
||||
}
|
||||
|
||||
Generated
+48
-189
@@ -1,191 +1,7 @@
|
||||
{
|
||||
"name": "windshaft-cartodb",
|
||||
"version": "1.3.0",
|
||||
"version": "1.3.1",
|
||||
"dependencies": {
|
||||
"cluster2": {
|
||||
"version": "0.3.5-cdb02",
|
||||
"from": "git://github.com/CartoDB/cluster2.git#cdb_production",
|
||||
"dependencies": {
|
||||
"express": {
|
||||
"version": "2.5.11",
|
||||
"dependencies": {
|
||||
"connect": {
|
||||
"version": "1.9.2",
|
||||
"dependencies": {
|
||||
"formidable": {
|
||||
"version": "1.0.13"
|
||||
}
|
||||
}
|
||||
},
|
||||
"mime": {
|
||||
"version": "1.2.4"
|
||||
},
|
||||
"qs": {
|
||||
"version": "0.4.2"
|
||||
},
|
||||
"mkdirp": {
|
||||
"version": "0.3.0"
|
||||
}
|
||||
}
|
||||
},
|
||||
"ejs": {
|
||||
"version": "0.8.3"
|
||||
},
|
||||
"npm": {
|
||||
"version": "1.2.17",
|
||||
"dependencies": {
|
||||
"semver": {
|
||||
"version": "1.1.4"
|
||||
},
|
||||
"ini": {
|
||||
"version": "1.1.0"
|
||||
},
|
||||
"slide": {
|
||||
"version": "1.1.3"
|
||||
},
|
||||
"abbrev": {
|
||||
"version": "1.0.4"
|
||||
},
|
||||
"graceful-fs": {
|
||||
"version": "1.2.0"
|
||||
},
|
||||
"minimatch": {
|
||||
"version": "0.2.11",
|
||||
"dependencies": {
|
||||
"sigmund": {
|
||||
"version": "1.0.0"
|
||||
}
|
||||
}
|
||||
},
|
||||
"nopt": {
|
||||
"version": "2.1.1"
|
||||
},
|
||||
"rimraf": {
|
||||
"version": "2.1.4"
|
||||
},
|
||||
"request": {
|
||||
"version": "2.9.203",
|
||||
"from": "git://github.com/isaacs/request"
|
||||
},
|
||||
"which": {
|
||||
"version": "1.0.5"
|
||||
},
|
||||
"tar": {
|
||||
"version": "0.1.17"
|
||||
},
|
||||
"fstream": {
|
||||
"version": "0.1.22"
|
||||
},
|
||||
"block-stream": {
|
||||
"version": "0.0.6"
|
||||
},
|
||||
"inherits": {
|
||||
"version": "1.0.0",
|
||||
"from": "git://github.com/isaacs/inherits"
|
||||
},
|
||||
"mkdirp": {
|
||||
"version": "0.3.5"
|
||||
},
|
||||
"read": {
|
||||
"version": "1.0.4",
|
||||
"dependencies": {
|
||||
"mute-stream": {
|
||||
"version": "0.0.3"
|
||||
}
|
||||
}
|
||||
},
|
||||
"lru-cache": {
|
||||
"version": "2.3.0"
|
||||
},
|
||||
"node-gyp": {
|
||||
"version": "0.9.5"
|
||||
},
|
||||
"fstream-npm": {
|
||||
"version": "0.1.4",
|
||||
"dependencies": {
|
||||
"fstream-ignore": {
|
||||
"version": "0.0.6"
|
||||
}
|
||||
}
|
||||
},
|
||||
"uid-number": {
|
||||
"version": "0.0.3"
|
||||
},
|
||||
"archy": {
|
||||
"version": "0.0.2"
|
||||
},
|
||||
"chownr": {
|
||||
"version": "0.0.1"
|
||||
},
|
||||
"npmlog": {
|
||||
"version": "0.0.2"
|
||||
},
|
||||
"ansi": {
|
||||
"version": "0.1.2"
|
||||
},
|
||||
"npm-registry-client": {
|
||||
"version": "0.2.18",
|
||||
"dependencies": {
|
||||
"couch-login": {
|
||||
"version": "0.1.15"
|
||||
}
|
||||
}
|
||||
},
|
||||
"read-package-json": {
|
||||
"version": "0.3.0"
|
||||
},
|
||||
"read-installed": {
|
||||
"version": "0.1.1"
|
||||
},
|
||||
"glob": {
|
||||
"version": "3.1.21"
|
||||
},
|
||||
"init-package-json": {
|
||||
"version": "0.0.6",
|
||||
"dependencies": {
|
||||
"promzard": {
|
||||
"version": "0.2.0"
|
||||
}
|
||||
}
|
||||
},
|
||||
"osenv": {
|
||||
"version": "0.0.3"
|
||||
},
|
||||
"lockfile": {
|
||||
"version": "0.3.0"
|
||||
},
|
||||
"retry": {
|
||||
"version": "0.6.0"
|
||||
},
|
||||
"once": {
|
||||
"version": "1.1.1"
|
||||
},
|
||||
"npmconf": {
|
||||
"version": "0.0.23",
|
||||
"dependencies": {
|
||||
"config-chain": {
|
||||
"version": "1.1.5",
|
||||
"dependencies": {
|
||||
"proto-list": {
|
||||
"version": "1.2.2"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"opener": {
|
||||
"version": "1.3.0"
|
||||
},
|
||||
"chmodr": {
|
||||
"version": "0.1.0"
|
||||
},
|
||||
"cmd-shim": {
|
||||
"version": "1.1.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"node-varnish": {
|
||||
"version": "0.1.1"
|
||||
},
|
||||
@@ -193,10 +9,10 @@
|
||||
"version": "1.3.3"
|
||||
},
|
||||
"windshaft": {
|
||||
"version": "0.12.4",
|
||||
"version": "0.12.9",
|
||||
"dependencies": {
|
||||
"grainstore": {
|
||||
"version": "0.13.3",
|
||||
"version": "0.13.5",
|
||||
"dependencies": {
|
||||
"carto": {
|
||||
"version": "0.9.3-cdb3",
|
||||
@@ -256,7 +72,47 @@
|
||||
"version": "0.3.4"
|
||||
},
|
||||
"sqlite3": {
|
||||
"version": "2.1.7"
|
||||
"version": "2.1.10",
|
||||
"dependencies": {
|
||||
"tar.gz": {
|
||||
"version": "0.1.1",
|
||||
"dependencies": {
|
||||
"fstream": {
|
||||
"version": "0.1.22",
|
||||
"dependencies": {
|
||||
"rimraf": {
|
||||
"version": "2.2.0"
|
||||
},
|
||||
"graceful-fs": {
|
||||
"version": "1.2.2"
|
||||
},
|
||||
"inherits": {
|
||||
"version": "1.0.0"
|
||||
}
|
||||
}
|
||||
},
|
||||
"tar": {
|
||||
"version": "0.1.17",
|
||||
"dependencies": {
|
||||
"inherits": {
|
||||
"version": "1.0.0"
|
||||
},
|
||||
"block-stream": {
|
||||
"version": "0.0.6"
|
||||
}
|
||||
}
|
||||
},
|
||||
"commander": {
|
||||
"version": "1.1.1",
|
||||
"dependencies": {
|
||||
"keypress": {
|
||||
"version": "0.1.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"mime": {
|
||||
"version": "1.2.9"
|
||||
@@ -320,6 +176,9 @@
|
||||
"version": "1.0.2"
|
||||
}
|
||||
}
|
||||
},
|
||||
"lru-cache": {
|
||||
"version": "2.3.0"
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -330,7 +189,7 @@
|
||||
"version": "2.0.3"
|
||||
},
|
||||
"redis": {
|
||||
"version": "0.7.2"
|
||||
"version": "0.8.3"
|
||||
},
|
||||
"hiredis": {
|
||||
"version": "0.1.15",
|
||||
|
||||
+3
-4
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"private": true,
|
||||
"name": "windshaft-cartodb",
|
||||
"version": "1.3.0",
|
||||
"version": "1.3.1",
|
||||
"description": "A map tile server for CartoDB",
|
||||
"url": "https://github.com/Vizzuality/Windshaft-cartodb",
|
||||
"licenses": [{
|
||||
@@ -18,13 +18,12 @@
|
||||
"email": "simon@vizzuality.com"
|
||||
},
|
||||
"dependencies": {
|
||||
"cluster2": "git://github.com/CartoDB/cluster2.git#cdb_production",
|
||||
"node-varnish": "0.1.1",
|
||||
"underscore" : "~1.3.3",
|
||||
"windshaft" : "~0.12.4",
|
||||
"windshaft" : "~0.12.9",
|
||||
"step": "0.0.x",
|
||||
"generic-pool": "~2.0.3",
|
||||
"redis": "0.7.2",
|
||||
"redis": "~0.8.3",
|
||||
"hiredis": "~0.1.14",
|
||||
"request": "2.9.202",
|
||||
"mapnik": "~0.7.14",
|
||||
|
||||
@@ -336,7 +336,7 @@ suite('multilayer', function() {
|
||||
version: '1.0.0',
|
||||
layers: [
|
||||
{ options: {
|
||||
sql: 'select 1 as cartodb_id, !pixel_height! as h'
|
||||
sql: 'select 1 as cartodb_id, !pixel_height! as h, '
|
||||
+ 'ST_Buffer(!bbox!, -32*greatest(!pixel_width!,!pixel_height!)) as the_geom_webmercator',
|
||||
cartocss: '#layer { polygon-fill:red; }',
|
||||
cartocss_version: '2.0.1'
|
||||
@@ -456,6 +456,124 @@ suite('multilayer', function() {
|
||||
});
|
||||
});
|
||||
|
||||
test("layergroup with 2 private-table layers", function(done) {
|
||||
|
||||
var layergroup = {
|
||||
version: '1.0.0',
|
||||
layers: [
|
||||
{ options: {
|
||||
sql: 'select * from test_table_private_1 where cartodb_id=1',
|
||||
cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
|
||||
cartocss_version: '2.1.1',
|
||||
interactivity: 'cartodb_id'
|
||||
} },
|
||||
{ options: {
|
||||
sql: 'select * from test_table_private_1 where cartodb_id=2',
|
||||
cartocss: '#layer { marker-fill:blue; marker-allow-overlap:true; }',
|
||||
cartocss_version: '2.1.1',
|
||||
interactivity: 'cartodb_id'
|
||||
} }
|
||||
]
|
||||
};
|
||||
|
||||
var expected_token = "50cb56d0ebe9142ca4ed97bc8dac3ee1";
|
||||
Step(
|
||||
function do_post()
|
||||
{
|
||||
var next = this;
|
||||
assert.response(server, {
|
||||
url: '/tiles/layergroup?map_key=1234',
|
||||
method: 'POST',
|
||||
headers: {host: 'localhost', 'Content-Type': 'application/json' },
|
||||
data: JSON.stringify(layergroup)
|
||||
}, {}, function(res) {
|
||||
assert.equal(res.statusCode, 200, res.body);
|
||||
var parsedBody = JSON.parse(res.body);
|
||||
var expectedBody = { layergroupid: expected_token };
|
||||
// check last modified
|
||||
var qTables = JSON.stringify({
|
||||
'q': 'SELECT CDB_QueryTables($windshaft$'
|
||||
+ layergroup.layers[0].options.sql + ';'
|
||||
+ layergroup.layers[1].options.sql
|
||||
+ '$windshaft$)'
|
||||
});
|
||||
assert.equal(parsedBody.last_updated, expected_last_updated);
|
||||
if ( expected_token ) {
|
||||
assert.equal(parsedBody.layergroupid, expected_token + ':' + expected_last_updated_epoch);
|
||||
}
|
||||
else expected_token = parsedBody.layergroupid;
|
||||
next(null, res);
|
||||
});
|
||||
},
|
||||
function do_get_tile(err)
|
||||
{
|
||||
if ( err ) throw err;
|
||||
var next = this;
|
||||
assert.response(server, {
|
||||
url: '/tiles/layergroup/' + expected_token + ':cb0/0/0/0.png?map_key=1234',
|
||||
method: 'GET',
|
||||
headers: {host: 'localhost' },
|
||||
encoding: 'binary'
|
||||
}, {}, function(res) {
|
||||
assert.equal(res.statusCode, 200, res.body);
|
||||
assert.equal(res.headers['content-type'], "image/png");
|
||||
|
||||
// Check X-Cache-Channel
|
||||
var cc = res.headers['x-cache-channel'];
|
||||
assert.ok(cc);
|
||||
var dbname = 'cartodb_test_user_1_db'
|
||||
assert.equal(cc.substring(0, dbname.length), dbname);
|
||||
next(err);
|
||||
});
|
||||
},
|
||||
function do_get_grid_layer0(err)
|
||||
{
|
||||
if ( err ) throw err;
|
||||
var next = this;
|
||||
assert.response(server, {
|
||||
url: '/tiles/layergroup/' + expected_token
|
||||
+ '/0/0/0/0.grid.json?map_key=1234',
|
||||
headers: {host: 'localhost' },
|
||||
method: 'GET'
|
||||
}, {}, function(res) {
|
||||
assert.equal(res.statusCode, 200, res.body);
|
||||
next(err);
|
||||
});
|
||||
},
|
||||
function do_get_grid_layer1(err)
|
||||
{
|
||||
if ( err ) throw err;
|
||||
var next = this;
|
||||
assert.response(server, {
|
||||
url: '/tiles/layergroup/' + expected_token
|
||||
+ '/1/0/0/0.grid.json?map_key=1234',
|
||||
headers: {host: 'localhost' },
|
||||
method: 'GET'
|
||||
}, {}, function(res) {
|
||||
assert.equal(res.statusCode, 200, res.body);
|
||||
assert.equal(res.headers['content-type'], "text/javascript; charset=utf-8; charset=utf-8");
|
||||
next(err);
|
||||
});
|
||||
},
|
||||
function finish(err) {
|
||||
var errors = [];
|
||||
if ( err ) {
|
||||
errors.push(err.message);
|
||||
console.log("Error: " + err);
|
||||
}
|
||||
redis_client.keys("map_style|cartodb_test_user_1_db|~" + expected_token, function(err, matches) {
|
||||
if ( err ) errors.push(err.message);
|
||||
assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches);
|
||||
redis_client.del(matches, function(err) {
|
||||
if ( err ) errors.push(err.message);
|
||||
if ( errors.length ) done(new Error(errors));
|
||||
else done(null);
|
||||
});
|
||||
});
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
suiteTeardown(function(done) {
|
||||
|
||||
// This test will add map_style records, like
|
||||
|
||||
Executable
+40
@@ -0,0 +1,40 @@
|
||||
#!/bin/sh
|
||||
|
||||
verbose=no
|
||||
tiler_url=http://dev.localhost.lan:8181/tiles/layergroup
|
||||
|
||||
# This is for direct windshaft connection
|
||||
#tiler_url=http://dev.localhost.lan:8083/database/cartodb_dev_user_1_db/layergroup
|
||||
|
||||
while test -n "$1"; do
|
||||
if test "$1" = "-v"; then
|
||||
verbose=yes
|
||||
elif test -z "$cfg"; then
|
||||
cfg="$1"
|
||||
else
|
||||
tiler_url="$1"
|
||||
fi
|
||||
shift
|
||||
done
|
||||
|
||||
if test -z "$cfg"; then
|
||||
echo "Usage: $0 [-v] <config_file> [<tiler_url>]" >&2
|
||||
echo "Default <tiler_url> is ${tiler_url}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cmd="curl -skH Content-Type:application/json --data-binary @${cfg} ${tiler_url}"
|
||||
if test x${verbose} = xyes; then
|
||||
cmd="${cmd} -v"
|
||||
fi
|
||||
res=`${cmd}`
|
||||
if test $? -gt 0; then
|
||||
echo "curl command failed: ${cmd}"
|
||||
fi
|
||||
|
||||
if test x${verbose} = xyes; then
|
||||
echo "${res}"
|
||||
fi
|
||||
|
||||
tok=`echo "$res" | sed 's/.*"layergroupid":"\([^"]*\)".*/\1/'`
|
||||
echo $tok
|
||||
+16
-4
@@ -5,13 +5,25 @@
|
||||
REDIS_PORT=6379 # default port
|
||||
|
||||
|
||||
if test -z "$2"; then
|
||||
echo "Usage: $0 <username> <tablename>" >&2
|
||||
if test -z "$1"; then
|
||||
echo "Usage: $0 <username> [<tablename>|~<token>]" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
username="$1"
|
||||
tabname="$2"
|
||||
token="$2"
|
||||
|
||||
dbname=`redis-cli -p ${REDIS_PORT} -n 5 hget "rails:users:${username}" "database_name"`
|
||||
redis-cli get "map_style|${dbname}|${tabname}" | sed -e 's/\\n/\n/g' -e 's/\\//g'
|
||||
if test $? -ne 0; then
|
||||
exit 1
|
||||
fi
|
||||
if test -z "${dbname}"; then
|
||||
echo "Username ${username} unknown by redis (try CARTODB/script/restore_redis?)" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Database name for user ${username}: ${dbname}" # only if verbose?
|
||||
if test -n "$token"; then
|
||||
redis-cli get "map_style|${dbname}|${token}" | sed -e 's/\\n/\n/g' -e 's/\\//g'
|
||||
else
|
||||
redis-cli keys "map_style|${dbname}|*"
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user