Compare commits

..

16 Commits
1.6.0 ... 1.6.2

Author SHA1 Message Date
Sandro Santilli
99ef396aeb Release 1.6.2 2014-01-23 12:25:34 +01:00
javi
69d7fb0344 fixed news #113 2014-01-22 19:12:17 +01:00
javi
e4e08db0b4 Merge branch 'master' of github.com:Vizzuality/Windshaft-cartodb 2014-01-22 19:10:37 +01:00
javi
164d952e56 support CORS in template instanciation endpoint, fixes #113 2014-01-22 19:10:09 +01:00
Sandro Santilli
c711dc328e Fix XML print from in show_style for token styles (#110) 2014-01-17 17:47:37 +01:00
Sandro Santilli
8b80ad8ba1 Restore XML print from the show_style tool
Closes #110
2014-01-16 18:51:02 +01:00
Sandro Santilli
5772c81590 Fix support for long (>64k chars) queries in layergroup creation
Closes #111. Includes testcase.
2014-01-16 17:20:30 +01:00
Sandro Santilli
09d4467e22 Prepare for 1.6.2 2014-01-16 17:19:55 +01:00
Sandro Santilli
d22f399f18 Release 1.6.1 2014-01-15 19:23:20 +01:00
Sandro Santilli
f89fd98ed7 Expect malformed response objects (#109)
Include test for sql errors on layergroup creation
Closes #109
2014-01-15 11:53:19 +01:00
Sandro Santilli
b01ce9d4cc Regenerate shrinkwrap for 1.6.1 2014-01-14 18:09:36 +01:00
Sandro Santilli
18ccd3cbaf Localize external CartoCSS resources at renderer creation time
Closes #108. JIRA CDB-1422 #resolve
2014-01-14 16:20:06 +01:00
Sandro Santilli
d6fe5339cf Do not choke on headers cleanup when response headers are not set
Raise a WARNING instead.
See #107 (github) and CDB-1438 (JIRA)
2014-01-13 18:56:09 +01:00
Sandro Santilli
2690ef3f05 Drop cache headers from error responses.
Closes #107 (github), #resolve CDB-1423 (JIRA)
2014-01-13 11:20:02 +01:00
Sandro Santilli
ae82d0ab47 Expect overrides of mapnik_version to be honoured
Reported on http://gis.stackexchange.com/questions/81450/cartodb-windshaft-error
2014-01-10 13:20:26 +01:00
Sandro Santilli
90e0a5dc30 Prepare for 1.6.1 2014-01-10 11:32:03 +01:00
10 changed files with 439 additions and 80 deletions

22
NEWS.md
View File

@@ -1,3 +1,24 @@
1.6.2 -- 2014-01-23
-------------------
Bug fixes:
* Fix support for long (>64k chars) queries in layergroup creation (#111)
Enhancements:
* Enhance tools/show_style to accept an environment parameter and
print XML style now it is not in redis anymore (#110)
* Support CORS in template instanciation endpoint (#113)
1.6.1 -- 2014-01-15
-------------------
Bug fixes:
* Drop cache headers from error responses (#107)
* Localize external CartoCSS resources at renderer creation time (#108)
1.6.0 -- 2014-01-10
-------------------
@@ -6,6 +27,7 @@ New features:
* Add 'user_from_host' directive to generalize username extraction (#100)
* Implement signed template maps (#98)
Other changes:
* Update cartodb-redis dependency to "~0.3.0"

View File

@@ -42,6 +42,22 @@ var CartodbWindshaft = function(serverOptions) {
return version;
}
// Override sendError to drop added cache headers (if any)
// See http://github.com/CartoDB/Windshaft-cartodb/issues/107
var ws_sendError = ws.sendError;
ws.sendError = function(res) {
// NOTE: the "res" object will have no _headers when
// faked by Windshaft, see
// http://github.com/CartoDB/Windshaft-cartodb/issues/109
//
if ( res._headers ) {
delete res._headers['cache-control'];
delete res._headers['last-modified'];
delete res._headers['x-cache-channel'];
}
ws_sendError.apply(this, arguments);
};
/**
* Helper to allow access to the layer to be used in the maps infowindow popup.
*/
@@ -371,6 +387,11 @@ var CartodbWindshaft = function(serverOptions) {
);
};
ws.options(template_baseurl + '/:template_id', function(req, res) {
ws.doCORS(res, "Content-Type");
return next();
});
// Instantiate a template
ws.post(template_baseurl + '/:template_id', function(req, res) {
ws.doCORS(res);

View File

@@ -78,7 +78,15 @@ module.exports = function(){
if (_.isString(api_key) && api_key != '') { qs.api_key = api_key; }
// call sql api
request.get({url:sqlapi, qs:qs, json:true}, function(err, res, body){
//
// NOTE: using POST to avoid size limits:
// Seehttp://github.com/CartoDB/Windshaft-cartodb/issues/111
//
// TODO: use "host" header to allow IP based specification
// of sqlapi address (and avoid a DNS lookup)
//
request.post({url:sqlapi, body:qs, json:true},
function(err, res, body){
if (err){
console.log('ERROR connecting to SQL API on ' + sqlapi + ': ' + err);
callback(err);

168
npm-shrinkwrap.json generated
View File

@@ -1,6 +1,6 @@
{
"name": "windshaft-cartodb",
"version": "1.6.0",
"version": "1.6.2",
"dependencies": {
"node-varnish": {
"version": "0.1.1"
@@ -9,14 +9,14 @@
"version": "1.3.3"
},
"windshaft": {
"version": "0.14.5",
"version": "0.15.0",
"dependencies": {
"grainstore": {
"version": "0.15.2",
"version": "0.16.0",
"dependencies": {
"carto": {
"version": "0.9.5-cdb2",
"from": "git://github.com/CartoDB/carto.git#0.9.5-cdb2",
"from": "http://github.com/CartoDB/carto/tarball/0.9.5-cdb2",
"dependencies": {
"underscore": {
"version": "1.4.4"
@@ -25,7 +25,7 @@
"version": "0.2.8",
"dependencies": {
"sax": {
"version": "0.5.5"
"version": "0.5.8"
}
}
},
@@ -131,46 +131,136 @@
"version": "0.3.8"
},
"zipfile": {
"version": "0.4.2"
"version": "0.4.3"
},
"sqlite3": {
"version": "2.1.19",
"version": "2.2.0",
"dependencies": {
"tar.gz": {
"version": "0.1.1",
"node-pre-gyp": {
"version": "0.2.5",
"dependencies": {
"fstream": {
"version": "0.1.25",
"nopt": {
"version": "2.1.2",
"dependencies": {
"rimraf": {
"version": "2.2.4"
},
"graceful-fs": {
"version": "2.0.1"
},
"inherits": {
"version": "2.0.1"
"abbrev": {
"version": "1.0.4"
}
}
},
"npmlog": {
"version": "0.0.6",
"dependencies": {
"ansi": {
"version": "0.2.1"
}
}
},
"semver": {
"version": "2.1.0"
},
"tar": {
"version": "0.1.18",
"version": "0.1.19",
"dependencies": {
"inherits": {
"version": "2.0.1"
},
"block-stream": {
"version": "0.0.7"
},
"fstream": {
"version": "0.1.25",
"dependencies": {
"graceful-fs": {
"version": "2.0.1"
}
}
}
}
},
"commander": {
"version": "1.1.1",
"tar-pack": {
"version": "2.0.0",
"dependencies": {
"keypress": {
"version": "0.1.0"
"uid-number": {
"version": "0.0.3"
},
"once": {
"version": "1.1.1"
},
"debug": {
"version": "0.7.4"
},
"fstream": {
"version": "0.1.25",
"dependencies": {
"graceful-fs": {
"version": "2.0.1"
},
"inherits": {
"version": "2.0.1"
}
}
},
"fstream-ignore": {
"version": "0.0.7",
"dependencies": {
"minimatch": {
"version": "0.2.14",
"dependencies": {
"sigmund": {
"version": "1.0.0"
}
}
},
"inherits": {
"version": "2.0.1"
}
}
},
"readable-stream": {
"version": "1.0.24"
},
"graceful-fs": {
"version": "1.2.3"
}
}
},
"aws-sdk": {
"version": "2.0.0-rc8",
"dependencies": {
"xml2js": {
"version": "0.2.4",
"dependencies": {
"sax": {
"version": "0.6.0"
}
}
},
"xmlbuilder": {
"version": "0.4.2"
}
}
},
"rc": {
"version": "0.3.2",
"dependencies": {
"optimist": {
"version": "0.3.7",
"dependencies": {
"wordwrap": {
"version": "0.0.2"
}
}
},
"deep-extend": {
"version": "0.2.6"
},
"ini": {
"version": "1.1.0"
}
}
},
"rimraf": {
"version": "2.2.5"
}
}
}
@@ -239,7 +329,7 @@
}
},
"tilelive-mapnik": {
"version": "0.6.4",
"version": "0.6.5",
"dependencies": {
"eio": {
"version": "0.2.2"
@@ -267,10 +357,18 @@
"version": "0.3.0"
},
"redis-mpool": {
"version": "0.0.2",
"version": "0.0.3",
"dependencies": {
"generic-pool": {
"version": "2.0.4"
},
"hiredis": {
"version": "0.1.16",
"dependencies": {
"bindings": {
"version": "1.1.1"
}
}
}
}
},
@@ -289,22 +387,6 @@
"redis": {
"version": "0.8.6"
},
"redis-mpool": {
"version": "0.0.2",
"dependencies": {
"generic-pool": {
"version": "2.0.4"
}
}
},
"hiredis": {
"version": "0.1.15",
"dependencies": {
"bindings": {
"version": "1.1.0"
}
}
},
"mocha": {
"version": "1.14.0",
"dependencies": {
@@ -338,7 +420,7 @@
"version": "3.2.3",
"dependencies": {
"minimatch": {
"version": "0.2.12",
"version": "0.2.14",
"dependencies": {
"lru-cache": {
"version": "2.5.0"

View File

@@ -1,7 +1,7 @@
{
"private": true,
"name": "windshaft-cartodb",
"version": "1.6.0",
"version": "1.6.2",
"description": "A map tile server for CartoDB",
"keywords": [
"cartodb"
@@ -24,7 +24,7 @@
"dependencies": {
"node-varnish": "0.1.1",
"underscore" : "~1.3.3",
"windshaft" : "~0.14.5",
"windshaft" : "~0.15.0",
"step": "0.0.x",
"request": "2.9.202",
"cartodb-redis": "~0.3.0",

View File

@@ -19,6 +19,14 @@ var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions);
server.setMaxListeners(0);
// Check that the response headers do not request caching
// Throws on failure
function checkNoCache(res) {
assert.ok(!res.headers.hasOwnProperty('x-cache-channel'));
assert.ok(!res.headers.hasOwnProperty('cache-control')); // is this correct ?
assert.ok(!res.headers.hasOwnProperty('last-modified')); // is this correct ?
}
suite('multilayer', function() {
var redis_client = redis.createClient(global.environment.redis.port);
@@ -460,6 +468,35 @@ suite('multilayer', function() {
});
});
// Also tests that server doesn't crash:
// see http://github.com/CartoDB/Windshaft-cartodb/issues/109
test("layergroup creation fails if sql is bogus", function(done) {
var layergroup = {
stat_tag: 'random_tag',
version: '1.0.0',
layers: [
{ options: {
sql: 'select bogus(0,0) as the_geom_webmercator',
cartocss: '#layer { polygon-fill:red; }',
cartocss_version: '2.0.1'
} }
]
};
assert.response(server, {
url: '/tiles/layergroup',
method: 'POST',
headers: {host: 'localhost', 'Content-Type': 'application/json' },
data: JSON.stringify(layergroup)
}, {}, function(res) {
assert.equal(res.statusCode, 400, res.body);
var parsed = JSON.parse(res.body);
var msg = parsed.errors[0];
assert.ok(msg.match(/bogus.*exist/), msg);
checkNoCache(res);
done();
});
});
test("layergroup with 2 private-table layers", function(done) {
var layergroup = {
@@ -863,6 +900,71 @@ suite('multilayer', function() {
);
});
// SQL strings can be of arbitrary length, when using POST
// See https://github.com/CartoDB/Windshaft-cartodb/issues/111
test("sql string can be very long", function(done){
var long_val = 'pretty';
for (var i=0; i<1024; ++i) long_val += ' long'
long_val += ' string';
var sql = "SELECT ";
for (var i=0; i<16; ++i)
sql += "'" + long_val + "'::text as pretty_long_field_name_" + i + ", ";
sql += "cartodb_id, the_geom_webmercator FROM gadm4 g";
var layergroup = {
version: '1.0.0',
layers: [
{ options: {
sql: sql,
cartocss: '#layer { marker-fill:red; }',
cartocss_version: '2.0.1'
} }
]
};
var errors = [];
var expected_token;
Step(
function do_post()
{
var data = JSON.stringify(layergroup);
assert.ok(data.length > 1024*64);
var next = this;
assert.response(server, {
url: '/tiles/layergroup?api_key=1234',
method: 'POST',
headers: {host: 'localhost', 'Content-Type': 'application/json' },
data: data
}, {}, function(res) { next(null, res); });
},
function check_result(err, res) {
if ( err ) throw err;
assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
var parsedBody = JSON.parse(res.body);
var token_components = parsedBody.layergroupid.split(':');
expected_token = token_components[0];
return null;
},
function cleanup(err) {
if ( err ) errors.push(err.message);
if ( ! expected_token ) return null;
var next = this;
redis_client.keys("map_style|test_cartodb_user_1_db|~" + expected_token, function(err, matches) {
if ( err ) errors.push(err.message);
assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches);
redis_client.del(matches, function(err) {
if ( err ) errors.push(err.message);
next();
});
});
},
function finish(err) {
if ( err ) errors.push('' + err);
if ( errors.length ) done(new Error(errors.join(',')));
else done(null);
}
);
});
suiteTeardown(function(done) {
// This test will add map_style records, like

View File

@@ -107,7 +107,7 @@ suite('server', function() {
}, function(res) {
var parsed = JSON.parse(res.body);
assert.equal(parsed.style, _.template(default_style, {table: 'my_table'}));
assert.equal(parsed.style_version, mapnik.versions.mapnik);
assert.equal(parsed.style_version, mapnik_version);
done();
});
});
@@ -125,6 +125,7 @@ suite('server', function() {
assert.equal(res.statusCode, 400, res.body);
assert.deepEqual(JSON.parse(res.body),
{error: 'Sorry, you are unauthorized (permission denied)'});
assert.ok(!res.headers.hasOwnProperty('cache-control'));
done();
});
});
@@ -142,6 +143,7 @@ suite('server', function() {
assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body);
assert.deepEqual(JSON.parse(res.body),
{error:"missing unknown_user's database_name in redis (try CARTODB/script/restore_redis)"});
assert.ok(!res.headers.hasOwnProperty('cache-control'));
done();
});
});
@@ -158,7 +160,7 @@ suite('server', function() {
var parsed = JSON.parse(res.body);
var style = _.template(default_style, {table: 'test_table_private_1'});
assert.equal(parsed.style, style);
assert.equal(parsed.style_version, mapnik.versions.mapnik);
assert.equal(parsed.style_version, mapnik_version);
done();
});
});
@@ -212,9 +214,12 @@ suite('server', function() {
url: '/tiles/my_table/style',
method: 'POST'
},{
status: 400,
body: '{"error":"must send style information"}'
}, function() { done(); });
}, function(res) {
assert.equal(res.statusCode, 400);
assert.ok(!res.headers.hasOwnProperty('cache-control'));
done();
});
});
test("post'ing bad style returns 400 with error", function(done){
@@ -351,7 +356,7 @@ suite('server', function() {
assert.equal(res.statusCode, 200, res.body);
var parsed = JSON.parse(res.body);
assert.equal(parsed.style, style);
assert.equal(parsed.style_version, mapnik.versions.mapnik);
assert.equal(parsed.style_version, mapnik_version);
done();
});
});
@@ -379,7 +384,7 @@ suite('server', function() {
var parsed = JSON.parse(res.body);
// NOTE: no transform expected for the specific style
assert.equal(parsed.style, style);
assert.equal(parsed.style_version, mapnik.versions.mapnik);
assert.equal(parsed.style_version, mapnik_version);
done();
});
});
@@ -766,6 +771,8 @@ suite('server', function() {
assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body);
assert.deepEqual(JSON.parse(res.body),
{error:"missing unknown_user's database_name in redis (try CARTODB/script/restore_redis)"});
assert.ok(!res.headers.hasOwnProperty('cache-control'),
"Unexpected Cache-Control: " + res.headers['cache-control']);
done();
});
});
@@ -786,6 +793,9 @@ suite('server', function() {
}, function(res) {
// 401 Unauthorized
assert.equal(res.statusCode, 401, res.statusCode + ': ' + res.body);
// Failed in 1.6.0 of https://github.com/CartoDB/Windshaft-cartodb/issues/107
assert.ok(!res.headers.hasOwnProperty('cache-control'),
"Unexpected Cache-Control: " + res.headers['cache-control']);
done();
});
});
@@ -1179,6 +1189,7 @@ suite('server', function() {
method: 'DELETE'
},{}, function(res) {
assert.equal(res.statusCode, 404, res.statusCode + ': ' + res.body);
assert.ok(!res.headers.hasOwnProperty('cache-control'));
done();
});
});
@@ -1210,6 +1221,7 @@ suite('server', function() {
},{}, function(res) {
// FIXME: should be 401 instead
assert.equal(res.statusCode, 500, res.statusCode + ': ' + res.body);
assert.ok(!res.headers.hasOwnProperty('cache-control'));
done();
});
});
@@ -1262,6 +1274,7 @@ suite('server', function() {
method: 'DELETE'
},{}, function(res) {
assert.equal(res.statusCode, 404, res.statusCode + ': ' + res.body);
assert.ok(!res.headers.hasOwnProperty('cache-control'));
done();
});
});

View File

@@ -131,6 +131,32 @@ suite('template_api', function() {
);
});
test("instance endpoint should return CORS headers", function(done){
Step(function postTemplate1(err, res) {
var next = this;
var post_request = {
url: '/tiles/template?api_key=1234',
method: 'POST',
headers: {host: 'localhost.localhost', 'Content-Type': 'application/json' },
data: JSON.stringify(template_acceptance1)
};
assert.response(server, post_request, {}, function(res) { next(null, res); });
},
function testCORS() {
assert.response(server, {
url: '/tiles/template/acceptance1',
method: 'OPTIONS'
},{
status: 200,
headers: {
'Access-Control-Allow-Headers': 'X-Requested-With, X-Prototype-Version, X-CSRF-Token, Content-Type',
'Access-Control-Allow-Origin': '*'
}
}, function() { done(); });
});
});
test("can list templates", function(done) {
var errors = [];

View File

@@ -5,31 +5,60 @@ var o = function(port, cb) {
this.queries = [];
var that = this;
this.sqlapi_server = http.createServer(function(req,res) {
var query = url.parse(req.url, true).query;
that.queries.push(query);
if ( query.q.match('SQLAPIERROR') ) {
res.statusCode = 400;
res.write(JSON.stringify({'error':'Some error occurred'}));
} else if ( query.q.match('EPOCH.* as max') ) {
// This is the structure of the known query sent by tiler
var row = {
'max': 1234567890.123
};
res.write(JSON.stringify({rows: [ row ]}));
} else {
var qs = JSON.stringify(query);
var row = {
// This is the structure of the known query sent by tiler
'cdb_querytables': '{' + qs + '}',
'max': qs
};
res.write(JSON.stringify({rows: [ row ]}));
}
res.end();
//console.log("server got request with method " + req.method);
var query;
if ( req.method == 'GET' ) {
query = url.parse(req.url, true).query;
that.handleQuery(query, res);
}
else if ( req.method == 'POST') {
var data = '';
req.on('data', function(chunk) {
//console.log("GOT Chunk " + chunk);
data += chunk;
});
req.on('end', function() {
//console.log("Data is: "); console.dir(data);
query = JSON.parse(data);
//console.log("Parsed is: "); console.dir(query);
//console.log("handleQuery is " + that.handleQuery);
that.handleQuery(query, res);
});
}
else {
that.handleQuery('SQLAPIEmu does not support method' + req.method, res);
}
}).listen(port, cb);
};
o.prototype.handleQuery = function(query, res) {
this.queries.push(query);
if ( query.q.match('SQLAPIERROR') ) {
res.statusCode = 400;
res.write(JSON.stringify({'error':'Some error occurred'}));
} else if ( query.q.match('EPOCH.* as max') ) {
// This is the structure of the known query sent by tiler
var row = {
'max': 1234567890.123
};
res.write(JSON.stringify({rows: [ row ]}));
} else {
var qs = JSON.stringify(query);
var row = {
// This is the structure of the known query sent by tiler
'cdb_querytables': '{' + qs + '}',
'max': qs
};
var out_obj = {rows: [ row ]};
var out = JSON.stringify(out_obj);
res.write(out);
}
res.end();
};
o.prototype.close = function(cb) {
this.sqlapi_server.close(cb);
};

View File

@@ -2,28 +2,84 @@
# TODO: port to node, if you really need it
REDIS_PORT=6379 # default port
ENV='development';
BASEDIR=`cd $(dirname $0)/../; pwd`
if test -z "$1"; then
echo "Usage: $0 <username> [<tablename>|~<token>]" >&2
echo "Usage: $0 [--env <environment>] <username> [<tablename>|~<token>]" >&2
echo " environment defaults to 'development'"
exit 1
fi
username=""
token=""
while test -n "$1"; do
if test "$1" = "--env"; then
shift; ENV="$1"; shift
elif test -z "$username"; then
username="$1"; shift
elif test -z "$token"; then
token="$1"; shift
else
echo "Unused option $1" >&2
shift
fi
done
echo "Using environment '${ENV}'"
CONFIG="${BASEDIR}/config/environments/${ENV}.js"
REDIS_PORT=`node -e "console.log(require('${CONFIG}').redis.port)"`
if test $? -ne 0; then
exit 1
fi
username="$1"
token="$2"
dbname=`redis-cli -p ${REDIS_PORT} -n 5 hget "rails:users:${username}" "database_name"`
if test $? -ne 0; then
exit 1
fi
if test -z "${dbname}"; then
echo "Username ${username} unknown by redis (try CARTODB/script/restore_redis?)" >&2
echo "Username ${username} unknown by redis on port ${REDIS_PORT} (try CARTODB/script/restore_redis?)" >&2
exit 1
fi
echo "Database name for user ${username}: ${dbname}" # only if verbose?
if test -n "$token"; then
redis-cli get "map_style|${dbname}|${token}" | sed -e 's/\\n/\n/g' -e 's/\\//g'
rec=`redis-cli get "map_style|${dbname}|${token}"`
if test -z "${rec}"; then
echo "${token}: no such map style known by redis on port ${REDIS_PORT}" >&2
exit 1
fi
#echo "${rec}"
escrec=`echo "${rec}" | sed -e 's/\\\\/\\\\\\\\/g'`
#echo "${escrec}"
node <<EOF
var x=JSON.parse('${escrec}');
console.log('style: ' + x.style);
console.log('version: ' + x.version);
global.environment = require('${CONFIG}');
var serverOptions = require('${BASEDIR}/lib/cartodb/server_options'); // _after_ setting global.environment
var grainstore = require('${BASEDIR}/node_modules/windshaft/node_modules/grainstore/lib/grainstore');
var mml_store = new grainstore.MMLStore(serverOptions.redis, serverOptions.grainstore);
var builderconfig = {dbname:'${dbname}'};
if ( '${token}'.match(/^~/) ) {
builderconfig.token = '${token}'.substring(1);
} else {
builderconfig.table = '${token}';
}
var mml_builder = mml_store.mml_builder(builderconfig,
function(err, payload) {
if ( err ) throw err;
mml_builder.toXML(function(err, xml) {
if ( err ) throw err;
console.log('- XML - ');
console.log(xml);
});
});
EOF
#echo "${rec}" | sed -e 's/\\n/\n/g' -e 's/\\//g'
else
redis-cli keys "map_style|${dbname}|*"
fi