cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'release/v2.0'

Browse Source
This commit is contained in:
Luis Bosque
2012-11-30 16:52:55 +01:00
parent 597008d9d4 e813209768
commit 7231864e1c
13 changed files with 347 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
NEWS.md
+9
View File
@@ -1,3 +1,12 @@
1.1.3 (30//11//12)
-----
* Fix reset_styles script to really skip extended keys
* CartoCSS versioning
* Mapnik-version dependent default styles
* Enhance 2.0 -> 2.1 transforms:
* styles with conditional markers
* scale arrow markers by 50%
1.1.2 (DD//MM//YY)
-----
* CartoCSS versioning
README.md
+2
View File
@@ -101,6 +101,8 @@ Args:
Args:
* style - the style in CartoCSS you want to set
* style_version - the version of the style for POST
* style_convert - request conversion to target version (both POST and GET)
**INFOWINDOW**
config/environments/development.js.example
+3 -2
View File
@@ -11,13 +11,14 @@ var config = {
user: "publicuser",
host: '127.0.0.1',
port: 5432,
extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188",
extent: "-20037508.5,-20037508.5,20037508.5,20037508.5",
/* experimental
geometry_field: "the_geom",
extent: "-180,-90,180,90",
srid: 4326,
*/
simplify: true
simplify: true,
max_size: 500
}
,mapnik_version: undefined
,millstone: {
config/environments/production.js.example
+3 -2
View File
@@ -10,8 +10,9 @@ var config = {
user: "publicuser",
host: '127.0.0.1',
port: 6432,
extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188",
simplify: true
extent: "-20037508.5,-20037508.5,20037508.5,20037508.5",
simplify: true,
max_size: 500
}
,mapnik_version: undefined
,millstone: {
config/environments/staging.js.example
+3 -2
View File
@@ -10,8 +10,9 @@ var config = {
user: "publicuser",
host: '127.0.0.1',
port: 6432,
extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188",
simplify: true
extent: "-20037508.5,-20037508.5,20037508.5,20037508.5",
simplify: true,
max_size: 500
}
,mapnik_version: undefined
,millstone: {
config/environments/test.js.example
+3 -2
View File
@@ -10,8 +10,9 @@ var config = {
user: "publicuser",
host: '127.0.0.1',
port: 5432,
extent: "-20005048.4188,-20005048.4188,20005048.4188,20005048.4188",
simplify: true
extent: "-20037508.5,-20037508.5,20037508.5,20037508.5",
simplify: true,
max_size: 500
}
,mapnik_version: '2.0.2'
,millstone: {
configure
Vendored
+15 -5
View File
@@ -17,16 +17,18 @@
# --strk(2012-07-23)
#
PGPORT=5432
MAPNIK_VERSION=
usage() {
echo "Usage: $0 [OPTION]"
echo
echo "Configuration:"
echo " --help display this help and exit"
echo " --with-pgport=NUM access PostgreSQL server on TCP port NUM"
echo " --help display this help and exit"
echo " --with-pgport=NUM access PostgreSQL server on TCP port NUM [$PGPORT]"
echo " --with-mapnik-version=STRING set mapnik version string [$MAPNIK_VERSION]"
}
PGPORT=5432
while test -n "$1"; do
case "$1" in
--help|-h)
@@ -36,6 +38,9 @@ while test -n "$1"; do
--with-pgport=*)
PGPORT=`echo "$1" | cut -d= -f2`
;;
--with-mapnik-version=*)
MAPNIK_VERSION=`echo "$1" | cut -d= -f2`
;;
*)
echo "Unknown option '$1'" >&2
usage >&2
@@ -45,11 +50,16 @@ while test -n "$1"; do
done
echo "PGPORT: $PGPORT"
echo "MAPNIK_VERSION: $MAPNIK_VERSION"
# TODO: allow specifying configuration settings !
for f in config/environments/*.example; do
o=`dirname "$f"`/`basename "$f" .example`
echo "Writing $o"
# See http://austinmatzko.com/2008/04/26/sed-multi-line-search-and-replace/
sed -n "1h;1!H;\${;g;s/\(,postgres: {[^}]*port: *'\?\)[^',]*\('\?,\)/\1$PGPORT\2/;p;}" < "$f" > "$o"
sed -n "1h;1!H;\${;g;s/\(,postgres: {[^}]*port: *'\?\)[^',]*\('\?,\)/\1$PGPORT\2/;p;}" < "$f" \
| sed "s/mapnik_version:.*/mapnik_version: '$MAPNIK_VERSION'/" \
> "$o"
done
lib/cartodb/server_options.js
+1 -1
View File
@@ -65,7 +65,7 @@ module.exports = function(){
me.req2params = function(req, callback){
// Whitelist query parameters and attach format
var good_query = ['sql', 'geom_type', 'cache_buster', 'cache_policy', 'callback', 'interactivity', 'map_key', 'api_key', 'style', 'style_version'];
var good_query = ['sql', 'geom_type', 'cache_buster', 'cache_policy', 'callback', 'interactivity', 'map_key', 'api_key', 'style', 'style_version', 'style_convert' ];
var bad_query = _.difference(_.keys(req.query), good_query);
_.each(bad_query, function(key){ delete req.query[key]; });
npm-shrinkwrap.json
Generated
+12 -11
View File
@@ -1,6 +1,6 @@
{
"name": "windshaft-cartodb",
"version": "1.1.0",
"version": "1.1.3",
"dependencies": {
"cluster2": {
"version": "0.3.5-cdb02",
@@ -182,18 +182,12 @@
"version": "1.3.3"
},
"grainstore": {
"version": "0.9.6",
"version": "0.10.8",
"dependencies": {
"semver": {
"version": "1.1.0"
},
"carto": {
"version": "0.9.3-cdb1",
"version": "0.9.3-cdb2",
"from": "git://github.com/CartoDB/carto.git#cdb-0.9",
"dependencies": {
"mapnik-reference": {
"version": "5.0.0"
},
"xml2js": {
"version": "0.1.14",
"dependencies": {
@@ -204,6 +198,10 @@
}
}
},
"mapnik-reference": {
"version": "5.0.0-cdb1",
"from": "git://github.com/CartoDB/mapnik-reference.git#cdb-5.0"
},
"millstone": {
"version": "0.5.11",
"dependencies": {
@@ -251,7 +249,7 @@
}
},
"windshaft": {
"version": "0.7.1",
"version": "0.8.0",
"dependencies": {
"express": {
"version": "2.5.11",
@@ -276,7 +274,7 @@
}
},
"tilelive": {
"version": "4.3.1",
"version": "4.3.3",
"dependencies": {
"optimist": {
"version": "0.3.5",
@@ -323,6 +321,9 @@
"mapnik": {
"version": "0.7.16"
},
"semver": {
"version": "1.1.0"
},
"mocha": {
"version": "1.2.1",
"dependencies": {
package.json
+5 -4
View File
@@ -1,7 +1,7 @@
{
"private": true,
"name": "windshaft-cartodb",
"version": "1.1.2",
"version": "1.1.3",
"description": "A map tile server for CartoDB",
"url": "https://github.com/Vizzuality/Windshaft-cartodb",
"licenses": [{
@@ -21,8 +21,8 @@
"cluster2": "git://github.com/CartoDB/cluster2.git#cdb_production",
"node-varnish": "0.1.1",
"underscore" : "~1.3.3",
"grainstore" : "~0.9.6",
"windshaft" : "~0.7.0",
"grainstore" : "~0.10.8",
"windshaft" : "~0.8.0",
"step": "0.0.x",
"generic-pool": "1.0.x",
"redis": "0.7.2",
@@ -31,7 +31,8 @@
"mapnik": "~0.7.14"
},
"devDependencies": {
"mocha": "1.2.1"
"mocha": "1.2.1",
"semver": "~1.1.0"
},
"scripts": {
"test": "make check"
test/acceptance/server.js
+207 -21
View File
@@ -3,6 +3,10 @@ var tests = module.exports = {};
var _ = require('underscore');
var redis = require('redis');
var querystring = require('querystring');
var semver = require('semver');
var mapnik = require('mapnik');
var Step = require('step');
require(__dirname + '/../support/test_helper');
var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft');
@@ -13,6 +17,18 @@ server.setMaxListeners(0);
suite('server', function() {
var redis_client = redis.createClient(global.environment.redis.port);
var default_style = semver.satisfies(mapnik.versions.mapnik, '<2.1.0')
?
// 2.0.0 default
'#<%= table %>{marker-fill: #FF6600;marker-opacity: 1;marker-width: 8;marker-line-color: white;marker-line-width: 3;marker-line-opacity: 0.9;marker-placement: point;marker-type: ellipse;marker-allow-overlap: true;}'
:
// 2.1.0 default
'#<%= table %>[mapnik-geometry-type=1] {marker-fill: #FF6600;marker-opacity: 1;marker-width: 16;marker-line-color: white;marker-line-width: 3;marker-line-opacity: 0.9;marker-placement: point;marker-type: ellipse;marker-allow-overlap: true;}#<%= table %>[mapnik-geometry-type=2] {line-color:#FF6600; line-width:1; line-opacity: 0.7;}#<%= table %>[mapnik-geometry-type=3] {polygon-fill:#FF6600; polygon-opacity: 0.7; line-opacity:1; line-color: #FFFFFF;}';
// A couple of styles to use during testing
var test_style_black_200 = "#test_table{marker-fill:black;marker-line-color:red;marker-width:10}";
var test_style_black_210 = "#test_table{marker-fill:black;marker-line-color:red;marker-width:20}";
suiteSetup(function(){
});
@@ -78,8 +94,8 @@ suite('server', function() {
headers: { 'X-Cache-Channel': 'cartodb_test_user_1_db:my_table' },
}, function(res) {
var parsed = JSON.parse(res.body);
assert.equal(parsed.style, "#my_table {marker-fill: #FF6600;marker-opacity: 1;marker-width: 8;marker-line-color: white;marker-line-width: 3;marker-line-opacity: 0.9;marker-placement: point;marker-type: ellipse;marker-allow-overlap: true;}");
assert.equal(parsed.style_version, '2.0.0');
assert.equal(parsed.style, _.template(default_style, {table: 'my_table'}));
assert.equal(parsed.style_version, mapnik.versions.mapnik);
done();
});
});
@@ -128,8 +144,9 @@ suite('server', function() {
}, function(res) {
assert.equal(res.statusCode, 200, res.body);
var parsed = JSON.parse(res.body);
assert.equal(parsed.style, "#test_table_private_1 {marker-fill: #FF6600;marker-opacity: 1;marker-width: 8;marker-line-color: white;marker-line-width: 3;marker-line-opacity: 0.9;marker-placement: point;marker-type: ellipse;marker-allow-overlap: true;}");
assert.equal(parsed.style_version, '2.0.0');
var style = _.template(default_style, {table: 'test_table_private_1'});
assert.equal(parsed.style, style);
assert.equal(parsed.style_version, mapnik.versions.mapnik);
done();
});
});
@@ -197,7 +214,7 @@ suite('server', function() {
url: '/tiles/my_table5/style?map_key=1234',
method: 'POST',
headers: {host: 'localhost', 'Content-Type': 'application/x-www-form-urlencoded' },
data: querystring.stringify({style: 'Map {background-color:#fff;}'})
data: querystring.stringify({style: 'Map { background-color:#fff; }'})
},{
}, function(res) {
assert.equal(res.statusCode, 200, res.body);
@@ -211,7 +228,7 @@ suite('server', function() {
url: '/tiles/my_table5/style?api_key=1234',
method: 'POST',
headers: {host: 'localhost', 'Content-Type': 'application/x-www-form-urlencoded' },
data: querystring.stringify({style: 'Map {background-color:#fff;}'})
data: querystring.stringify({style: 'Map { background-color:#fff; }'})
},{}, function(res) {
assert.equal(res.statusCode, 200, res.body);
done();
@@ -224,7 +241,7 @@ suite('server', function() {
url: '/tiles/my_table5/style?map_key=1234',
method: 'POST',
headers: {host: 'localhost', 'Content-Type': 'application/x-www-form-urlencoded' },
data: querystring.stringify({style: 'Map {background-color:#fff;}'})
data: querystring.stringify({style: 'Map { background-color:#fff; }'})
},{
}, function(res) {
assert.equal(res.statusCode, 200, res.body);
@@ -232,7 +249,7 @@ suite('server', function() {
url: '/tiles/my_table5/style',
method: 'POST',
headers: {host: 'localhost', 'Content-Type': 'application/x-www-form-urlencoded' },
data: querystring.stringify({style: 'Map {background-color:#aaa;}'})
data: querystring.stringify({style: 'Map { background-color:#aaa; }'})
},{}, function(res) {
// FIXME: should be 401 Unauthorized
assert.equal(res.statusCode, 500, res.body);
@@ -246,7 +263,7 @@ suite('server', function() {
status: 200,
}, function(res) {
var parsed = JSON.parse(res.body);
assert.equal(parsed.style, 'Map {background-color:#fff;}');
assert.equal(parsed.style, 'Map { background-color:#fff; }');
assert.equal(parsed.style_version, '2.0.0');
done();
});
@@ -256,14 +273,13 @@ suite('server', function() {
});
test("post'ing good style returns 200 then getting returns original style", function(done){
var style = 'Map {background-color:#fff;}';
var style = 'Map { background-color:#fff; }';
assert.response(server, {
url: '/tiles/my_table5/style?map_key=1234',
method: 'POST',
headers: {host: 'localhost', 'Content-Type': 'application/x-www-form-urlencoded' },
data: querystring.stringify({style: style, style_version: '2.0.2'})
},{
}, function(res) {
},{}, function(res) {
assert.equal(res.statusCode, 200, res.body);
@@ -271,19 +287,54 @@ suite('server', function() {
headers: {host: 'localhost'},
url: '/tiles/my_table5/style',
method: 'GET'
},{
status: 200,
}, function(res) {
},{}, function(res) {
assert.equal(res.statusCode, 200, res.body);
var parsed = JSON.parse(res.body);
assert.equal(parsed.style, style);
assert.equal(parsed.style_version, '2.0.2');
done();
assert.response(server, {
headers: {host: 'localhost'},
url: '/tiles/my_table5/style?style_convert=true',
method: 'GET'
},{}, function(res) {
assert.equal(res.statusCode, 200, res.body);
var parsed = JSON.parse(res.body);
assert.equal(parsed.style, style);
assert.equal(parsed.style_version, mapnik.versions.mapnik);
done();
});
});
});
});
test("post'ing good style with style_convert returns 200 then getting returns converted style", function(done){
var style = 'Map { background-color:#fff; }';
assert.response(server, {
url: '/tiles/my_table5/style?map_key=1234',
method: 'POST',
headers: {host: 'localhost', 'Content-Type': 'application/x-www-form-urlencoded' },
data: querystring.stringify({style: style, style_version: '2.0.2', style_convert: true})
},{}, function(res) {
assert.equal(res.statusCode, 200, res.body);
assert.response(server, {
headers: {host: 'localhost'},
url: '/tiles/my_table5/style',
method: 'GET'
},{}, function(res) {
assert.equal(res.statusCode, 200, res.body);
var parsed = JSON.parse(res.body);
// NOTE: no transform expected for the specific style
assert.equal(parsed.style, style);
assert.equal(parsed.style_version, mapnik.versions.mapnik);
done();
});
});
});
/////////////////////////////////////////////////////////////////////////////////
//
// DELETE STYLE
@@ -310,7 +361,7 @@ suite('server', function() {
status: 200,
}, function(res) {
var parsed = JSON.parse(res.body);
assert.equal(parsed.style, 'Map {background-color:#fff;}');
assert.equal(parsed.style, 'Map { background-color:#fff; }');
//assert.equal(parsed.version, '2.0.0');
done();
});
@@ -319,7 +370,7 @@ suite('server', function() {
test("delete'ing style returns 200 then getting returns default style", function(done){
// this is the default style
var style = '#my_table5 {marker-fill: #FF6600;marker-opacity: 1;marker-width: 8;marker-line-color: white;marker-line-width: 3;marker-line-opacity: 0.9;marker-placement: point;marker-type: ellipse;marker-allow-overlap: true;}'
var style = _.template(default_style, {table: 'my_table5'});
assert.response(server, {
url: '/tiles/my_table5/style?map_key=1234',
method: 'DELETE',
@@ -689,9 +740,6 @@ suite('server', function() {
});
});
var test_style_black_200 = "#test_table{marker-fill:black;marker-line-color:red;marker-width:10}";
var test_style_black_210 = "#test_table{marker-fill:black;marker-line-color:red;marker-width:20}";
test("get'ing a tile with url specified 2.0.0 style should return an expected tile", function(done){
var style = querystring.stringify({style: test_style_black_200, style_version: '2.0.0'});
assert.response(server, {
@@ -730,6 +778,144 @@ suite('server', function() {
});
});
// See http://github.com/Vizzuality/Windshaft-cartodb/issues/57
test("GET'ing a tile as anonymous with style set by POST", function(done){
var style = querystring.stringify({style: test_style_black_210, style_version: '2.1.0'});
Step (
function postStyle1() {
var next = this;
assert.response(server, {
method: 'POST',
url: '/tiles/test_table/style',
headers: {host: 'localhost', 'Content-Type': 'application/x-www-form-urlencoded' },
data: querystring.stringify({style: 'Map { background-color:#fff; }', map_key: 1234})
},{}, function(res) {
assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
next();
});
},
// Load the new cache with results from Style1 above
function getTileAnon1(err) {
if ( err ) throw err;
var next = this;
assert.response(server, {
headers: {host: 'localhost'},
url: '/tiles/test_table/15/16046/12354.png',
method: 'GET',
encoding: 'binary'
},{}, function(res){
assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
var ct = res.headers['content-type'];
assert.equal(ct, 'image/png');
assert.imageEqualsFile(res.body, './test/fixtures/blank.png', 0,
function(err, similarity) {
if (err) next(err);
else next();
});
});
},
// Get again with authentication
function getTileAuth1(err) {
if ( err ) throw err;
var next = this;
assert.response(server, {
headers: {host: 'localhost'},
url: '/tiles/test_table/15/16046/12354.png?map_key=1234',
method: 'GET',
encoding: 'binary'
},{}, function(res){
assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
var ct = res.headers['content-type'];
assert.equal(ct, 'image/png');
assert.imageEqualsFile(res.body, './test/fixtures/blank.png', 0,
function(err, similarity) {
if (err) next(err);
else next();
});
});
},
// Change the style
function postStyle2(err) {
if ( err ) throw err;
var next = this;
assert.response(server, {
method: 'POST',
url: '/tiles/test_table/style',
headers: {host: 'localhost', 'Content-Type': 'application/x-www-form-urlencoded' },
data: querystring.stringify({style: test_style_black_200, map_key: 1234})
},{}, function(res) {
try {
assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
next();
}
catch (err) { next(err); }
});
},
// Verify the Style2 is applied. NOTE: pass the SAME cache_buster as before!
function getTileAnon2(err) {
if ( err ) throw err;
var next = this;
assert.response(server, {
headers: {host: 'localhost'},
url: '/tiles/test_table/15/16046/12354.png',
method: 'GET',
encoding: 'binary'
},{}, function(res){
assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
var ct = res.headers['content-type'];
assert.equal(ct, 'image/png');
assert.imageEqualsFile(res.body, './test/fixtures/test_table_15_16046_12354_styled_black.png', 2,
function(err, similarity) {
// NOTE: we expect them to be EQUAL here
if (err) { next(err); return; }
next();
});
});
},
// Delete the style
function delStyle(err) {
if ( err ) throw err;
var next = this;
assert.response(server, {
method: 'DELETE',
url: '/tiles/test_table/style?map_key=1234',
headers: {host: 'localhost'}
},{}, function(res) {
try {
assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
next();
}
catch (err) { next(err); }
});
},
// Verify the default style is applied.
function getTileAnon3(err) {
if ( err ) throw err;
var next = this;
assert.response(server, {
headers: {host: 'localhost'},
url: '/tiles/test_table/15/16046/12354.png?cache_buster=2314',
method: 'GET',
encoding: 'binary'
},{}, function(res){
assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
var ct = res.headers['content-type'];
assert.equal(ct, 'image/png');
assert.imageEqualsFile(res.body, './test/fixtures/test_table_15_16046_12354_styled_black.png', 2,
function(err, similarity) {
// NOTE: we expect them to be different here
if (err) next();
else next(new Error('Last posted style still in effect after delete'));
});
});
},
function finish(err) {
if ( err ) done(err);
else done();
}
);
});
/////////////////////////////////////////////////////////////////////////////////
//
// DELETE CACHE
tools/convert_database_styles
Executable
+81
View File
@@ -0,0 +1,81 @@
#!/usr/bin/env node
var path = require('path');
var grainstore = require('grainstore');
var mapnik = require('mapnik');
var redis = require('redis');
function usage(me, exitcode) {
console.log("Usage: " + me + " <database_name> <table_name> [<target_mapnik_version>]");
process.exit(exitcode);
}
var node_path = process.argv.shift();
var script_path = process.argv.shift();
var me = path.basename(script_path);
var database_name = process.argv.shift()
var table_name = process.argv.shift()
var MAPNIK_VERSION = process.argv.shift()
if ( ! MAPNIK_VERSION ) {
MAPNIK_VERSION = mapnik.versions.mapnik;
}
if ( ! database_name || ! table_name) {
usage(me, 1);
}
var REDIS_PORT = 6379; // TODO: make a command line parameter
var dbnum = 0;
var mml_store = new grainstore.MMLStore({port:REDIS_PORT}, {mapnik_version:MAPNIK_VERSION});
var failures = [];
var client = redis.createClient(REDIS_PORT, 'localhost');
client.on('connect', function() {
client.select(dbnum);
client.keys('map_style|' + database_name + '|' + table_name, function(err, matches) {
processNext = function() {
if ( ! matches.length ) process.exit(failures.length);
var k = matches.shift();
if ( /map_style\|.*\|.*\|/.test(k) ) {
//console.warn("Key " + k + " is EXTENDED, skipping");
processNext();
}
var out = 'map_style|' + database_name + '|' + table_name + ': ';
var mml_builder = mml_store.mml_builder({
dbname:database_name,
table:table_name},
function(err, payload) {
if ( err ) {
console.warn(out + err.message);
failures.push(k); processNext();
}
else {
mml_builder.resetStyle(function(err, data) {
if ( err ) {
console.warn(out + err.message);
failures.push(k);
}
else console.log(out + 'OK');
processNext();
}, true);
}
});
};
processNext();
});
});
tools/reset_styles
+3
View File
@@ -55,7 +55,10 @@ client.on('connect', function() {
if ( /map_style\|.*\|.*\|/.test(k) ) {
//console.warn("Key " + k + " is EXTENDED, skipping");
// TODO: drop instead ?
// See https://github.com/Vizzuality/Windshaft-cartodb/issues/58
processNext();
return;
}
var params = RegExp(/map_style\|(.*)\|(.*)/).exec(k);