cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'release/staging'

Browse Source
This commit is contained in:
David Arango
2013-06-11 14:41:12 +02:00
parent 627b3f084d 70f535d13a
commit f20c98e49c
8 changed files with 125 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
NEWS.md
+5
View File
@@ -1,3 +1,8 @@
1.3.0
------
* Change stats format for multilayer map token request, see
http://github.com/Vizzuality/Windshaft-cartodb/wiki/Redis-stats-format
1.2.1
------
* Fix multilayer post from firefox
lib/cartodb/carto_data.js
+24 -8
View File
@@ -7,6 +7,8 @@
* and geometry type from the redis stores of cartodb
*/
var strftime = require('strftime');
var RedisPool = require("./redis_pool")
, _ = require('underscore')
, Step = require('step');
@@ -20,7 +22,8 @@ module.exports = function() {
table_metadata_db: 0,
user_key: "rails:users:<%= username %>",
table_key: "rails:<%= database_name %>:<%= table_name %>",
user_stats_key: "tiler:users:<%= username %>"
global_mapview_key: "user:<%= username %>:mapviews:global",
tagged_mapview_key: "user:<%= username %>:mapviews:stat_tag:<%= stat_tag %>"
};
@@ -52,15 +55,28 @@ module.exports = function() {
* Increment mapview count for a user
*
* @param username
* @param stat_tag
* @param callback will be called with the new value
*/
me.incMapviewCount = function(username, callback) {
var redisKey = _.template(this.user_stats_key, {username: username});
this.redisCmd(me.user_metadata_db, 'HINCRBY', [redisKey, 'mapviews', 1], function(err, val) {
if ( err ) callback(err, null);
else callback(err, val);
});
me.incMapviewCount = function(username, stat_tag, callback) {
var that = this;
var now = strftime("%Y%m%d", new Date());
var redisKey;
Step (
function incrementGlobal() {
redisKey = _.template(that.global_mapview_key, {username: username});
that.redisCmd(me.user_metadata_db, 'ZINCRBY', [redisKey, 1, now], this);
},
function incrementTag(err, val) {
if ( err ) throw err;
if ( _.isUndefined(stat_tag) ) return 1;
redisKey = _.template(that.tagged_mapview_key, {username: username, stat_tag: stat_tag});
that.redisCmd(me.user_metadata_db, 'ZINCRBY', [redisKey, 1, now], this);
},
function finish(err, val) {
if ( callback ) callback(err);
}
);
};
lib/cartodb/server_options.js
+8 -3
View File
@@ -76,13 +76,18 @@ module.exports = function(){
}
if (res.statusCode != 200) {
var msg = res.body.error ? res.body.error : res.body;
callback(new Error('unexpected response status (' + res.statusCode + ') for sql query: ' + sql));
callback(new Error(msg));
console.log('unexpected response status (' + res.statusCode + ') for sql query: ' + sql + ': ' + msg);
return;
}
callback(null, body.rows);
});
};
//
// Invoke callback with number of milliseconds since
// last update in any of the given tables
//
me.findLastUpdated = function (username, api_key, tableNames, callback) {
var sql = 'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max FROM CDB_TableMetadata m WHERE m.tabname::name = any (\'{'
+ tableNames.join(',') + '}\')';
@@ -99,7 +104,7 @@ module.exports = function(){
if(rows.length !== 0) {
last_updated = rows[0].max || 0;
}
callback(null, last_updated);
callback(null, last_updated*1000);
});
};
@@ -245,7 +250,7 @@ module.exports = function(){
// Don't wait for the mapview count increment to
// take place before proceeding. Error will be logged
// asyncronously
cartoData.incMapviewCount(username, function(err) {
cartoData.incMapviewCount(username, mapconfig.stat_tag, function(err) {
if ( err ) console.log("ERROR: failed to increment mapview count for user '" + username + "': " + err);
done();
});
npm-shrinkwrap.json
Generated
+6 -3
View File
@@ -1,6 +1,6 @@
{
"name": "windshaft-cartodb",
"version": "1.2.1",
"version": "1.3.0",
"dependencies": {
"cluster2": {
"version": "0.3.5-cdb02",
@@ -193,10 +193,10 @@
"version": "1.3.3"
},
"windshaft": {
"version": "0.12.3",
"version": "0.12.4",
"dependencies": {
"grainstore": {
"version": "0.13.2",
"version": "0.13.3",
"dependencies": {
"carto": {
"version": "0.9.3-cdb3",
@@ -346,6 +346,9 @@
"mapnik": {
"version": "0.7.22"
},
"strftime": {
"version": "0.6.0"
},
"lzma": {
"version": "1.2.3"
},
package.json
+3 -2
View File
@@ -1,7 +1,7 @@
{
"private": true,
"name": "windshaft-cartodb",
"version": "1.2.1",
"version": "1.3.0",
"description": "A map tile server for CartoDB",
"url": "https://github.com/Vizzuality/Windshaft-cartodb",
"licenses": [{
@@ -21,13 +21,14 @@
"cluster2": "git://github.com/CartoDB/cluster2.git#cdb_production",
"node-varnish": "0.1.1",
"underscore" : "~1.3.3",
"windshaft" : "~0.12.3",
"windshaft" : "~0.12.4",
"step": "0.0.x",
"generic-pool": "~2.0.3",
"redis": "0.7.2",
"hiredis": "~0.1.14",
"request": "2.9.202",
"mapnik": "~0.7.14",
"strftime": "~0.6.0",
"lzma": "~1.2.3"
},
"devDependencies": {
test/acceptance/multilayer.js
+77 -24
View File
@@ -6,7 +6,9 @@ var querystring = require('querystring');
var semver = require('semver');
var mapnik = require('mapnik');
var Step = require('step');
var strftime = require('strftime');
var SQLAPIEmu = require(__dirname + '/../support/SQLAPIEmu.js');
var redis_stats_db = 5;
require(__dirname + '/../support/test_helper');
@@ -330,6 +332,7 @@ suite('multilayer', function() {
test("layergroup creation raises mapviews counter", function(done) {
var layergroup = {
stat_tag: 'random_tag',
version: '1.0.0',
layers: [
{ options: {
@@ -340,18 +343,18 @@ suite('multilayer', function() {
} }
]
};
var statskey = "tiler:users:localhost";
var mapviews_field = 'mapviews';
var statskey = "user:localhost:mapviews";
var redis_stats_client = redis.createClient(global.environment.redis.port);
var redis_stats_db = 5;
var expected_token; // will be set on first post and checked on second
var now = strftime("%Y%m%d", new Date());
var errors = [];
Step(
function clean_stats()
{
var next = this;
redis_stats_client.select(redis_stats_db, function(err) {
if ( err ) next(err);
else redis_stats_client.del(statskey, next);
else redis_stats_client.del(statskey+':global', next);
});
},
function do_post_1(err)
@@ -366,12 +369,19 @@ suite('multilayer', function() {
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.body);
expected_token = JSON.parse(res.body).layergroupid;
redis_stats_client.hget(statskey, mapviews_field, next);
redis_stats_client.zscore(statskey + ":global", now, next);
});
},
function check_stats_1_do_post_2(err, val) {
function check_global_stats_1(err, val) {
if ( err ) throw err;
assert.equal(val, 1);
assert.equal(val, 1, "Expected score of " + now + " in "
+ statskey + ":global to be 1, got " + val);
redis_stats_client.zscore(statskey+':stat_tag:random_tag', now, this);
},
function check_tag_stats_1_do_post_2(err, val) {
if ( err ) throw err;
assert.equal(val, 1, "Expected score of " + now + " in "
+ statskey + ":stat_tag:" + layergroup.stat_tag + " to be 1, got " + val);
var next = this;
assert.response(server, {
url: '/tiles/layergroup',
@@ -381,44 +391,87 @@ suite('multilayer', function() {
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.body);
assert.equal(JSON.parse(res.body).layergroupid, expected_token);
redis_stats_client.hget(statskey, mapviews_field, next);
redis_stats_client.zscore(statskey+':global', now, next);
});
},
function check_stats_2(err, val)
function check_global_stats_2(err, val)
{
if ( err ) throw err;
assert.equal(val, 2);
assert.equal(val, 2, "Expected score of " + now + " in "
+ statskey + ":global to be 2, got " + val);
redis_stats_client.zscore(statskey+':stat_tag:' + layergroup.stat_tag, now, this);
},
function check_tag_stats_2(err, val)
{
if ( err ) throw err;
assert.equal(val, 2, "Expected score of " + now + " in "
+ statskey + ":stat_tag:" + layergroup.stat_tag + " to be 2, got " + val);
return 1;
},
function finish(err) {
var errors = [];
if ( err ) {
errors.push(err.message);
console.log("Error: " + err);
}
function cleanup_map_style(err) {
if ( err ) errors.push('' + err);
var next = this;
// trip epoch
expected_token = expected_token.split(':')[0];
redis_client.keys("map_style|cartodb_test_user_1_db|~" + expected_token, function(err, matches) {
if ( err ) errors.push(err.message);
assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches);
redis_client.del(matches, function(err) {
if ( err ) errors.push(err.message);
if ( errors.length ) done(new Error(errors));
else done(null);
});
redis_client.del(matches, next);
});
},
function cleanup_stats(err) {
if ( err ) errors.push('' + err);
redis_client.del([statskey+':global', statskey+':stat_tag:'+layergroup.stat_tag], this);
},
function finish(err) {
if ( err ) errors.push('' + err);
if ( errors.length ) done(new Error(errors.join(',')));
else done(null);
}
);
});
test("layergroup creation fails if CartoCSS is bogus", function(done) {
var layergroup = {
stat_tag: 'random_tag',
version: '1.0.0',
layers: [
{ options: {
sql: 'select 1 as cartodb_id, !pixel_height! as h'
+ 'ST_Buffer(!bbox!, -32*greatest(!pixel_width!,!pixel_height!)) as the_geom_webmercator',
cartocss: '#layer { polygon-fit:red; }',
cartocss_version: '2.0.1'
} }
]
};
assert.response(server, {
url: '/tiles/layergroup',
method: 'POST',
headers: {host: 'localhost', 'Content-Type': 'application/json' },
data: JSON.stringify(layergroup)
}, {}, function(res) {
assert.equal(res.statusCode, 400, res.body);
var parsed = JSON.parse(res.body);
assert.ok(parsed.errors[0].match(/^style0/));
assert.ok(parsed.errors[0].match(/Unrecognized rule: polygon-fit/));
done();
});
});
suiteTeardown(function(done) {
// This test will add map_style records, like
// 'map_style|null|publicuser|my_table',
redis_client.keys("map_style|*", function(err, matches) {
redis_client.del(matches, function(err) {
sqlapi_server.close(done);
redis_client.select(5, function(err, matches) {
redis_client.keys("user:localhost:mapviews*", function(err, matches) {
redis_client.del(matches, function(err) {
sqlapi_server.close(done);
});
});
});
});
});
});
});
test/support/SQLAPIEmu.js
+1 -1
View File
@@ -14,7 +14,7 @@ var o = function(port, cb) {
} else if ( query.q.match('EPOCH.* as max') ) {
// This is the structure of the known query sent by tiler
var row = {
'max': 1234567890123
'max': 1234567890.123
};
res.write(JSON.stringify({rows: [ row ]}));
} else {
tools/convert_database_styles
+1 -1
View File
@@ -2,7 +2,7 @@
var path = require('path');
var grainstore = require('grainstore');
var grainstore = require('../node_modules/windshaft/node_modules/grainstore');
var mapnik = require('mapnik');
var redis = require('redis');