diff --git a/NEWS.md b/NEWS.md index 2bcd5b1b..dd2322a0 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,8 @@ +1.3.0 +------ +* Change stats format for multilayer map token request, see + http://github.com/Vizzuality/Windshaft-cartodb/wiki/Redis-stats-format + 1.2.1 ------ * Fix multilayer post from firefox diff --git a/lib/cartodb/carto_data.js b/lib/cartodb/carto_data.js index a5c5939d..8b568c05 100644 --- a/lib/cartodb/carto_data.js +++ b/lib/cartodb/carto_data.js @@ -7,6 +7,8 @@ * and geometry type from the redis stores of cartodb */ +var strftime = require('strftime'); + var RedisPool = require("./redis_pool") , _ = require('underscore') , Step = require('step'); @@ -20,7 +22,8 @@ module.exports = function() { table_metadata_db: 0, user_key: "rails:users:<%= username %>", table_key: "rails:<%= database_name %>:<%= table_name %>", - user_stats_key: "tiler:users:<%= username %>" + global_mapview_key: "user:<%= username %>:mapviews:global", + tagged_mapview_key: "user:<%= username %>:mapviews:stat_tag:<%= stat_tag %>" }; @@ -52,15 +55,28 @@ module.exports = function() { * Increment mapview count for a user * * @param username + * @param stat_tag * @param callback will be called with the new value */ - me.incMapviewCount = function(username, callback) { - var redisKey = _.template(this.user_stats_key, {username: username}); - - this.redisCmd(me.user_metadata_db, 'HINCRBY', [redisKey, 'mapviews', 1], function(err, val) { - if ( err ) callback(err, null); - else callback(err, val); - }); + me.incMapviewCount = function(username, stat_tag, callback) { + var that = this; + var now = strftime("%Y%m%d", new Date()); + var redisKey; + Step ( + function incrementGlobal() { + redisKey = _.template(that.global_mapview_key, {username: username}); + that.redisCmd(me.user_metadata_db, 'ZINCRBY', [redisKey, 1, now], this); + }, + function incrementTag(err, val) { + if ( err ) throw err; + if ( _.isUndefined(stat_tag) ) return 1; + redisKey = _.template(that.tagged_mapview_key, {username: username, stat_tag: stat_tag}); + that.redisCmd(me.user_metadata_db, 'ZINCRBY', [redisKey, 1, now], this); + }, + function finish(err, val) { + if ( callback ) callback(err); + } + ); }; diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index c67b134a..6693ece7 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -76,13 +76,18 @@ module.exports = function(){ } if (res.statusCode != 200) { var msg = res.body.error ? res.body.error : res.body; - callback(new Error('unexpected response status (' + res.statusCode + ') for sql query: ' + sql)); + callback(new Error(msg)); + console.log('unexpected response status (' + res.statusCode + ') for sql query: ' + sql + ': ' + msg); return; } callback(null, body.rows); }); }; + // + // Invoke callback with number of milliseconds since + // last update in any of the given tables + // me.findLastUpdated = function (username, api_key, tableNames, callback) { var sql = 'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max FROM CDB_TableMetadata m WHERE m.tabname::name = any (\'{' + tableNames.join(',') + '}\')'; @@ -99,7 +104,7 @@ module.exports = function(){ if(rows.length !== 0) { last_updated = rows[0].max || 0; } - callback(null, last_updated); + callback(null, last_updated*1000); }); }; @@ -245,7 +250,7 @@ module.exports = function(){ // Don't wait for the mapview count increment to // take place before proceeding. Error will be logged // asyncronously - cartoData.incMapviewCount(username, function(err) { + cartoData.incMapviewCount(username, mapconfig.stat_tag, function(err) { if ( err ) console.log("ERROR: failed to increment mapview count for user '" + username + "': " + err); done(); }); diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index a3ce463a..1ae33564 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -1,6 +1,6 @@ { "name": "windshaft-cartodb", - "version": "1.2.1", + "version": "1.3.0", "dependencies": { "cluster2": { "version": "0.3.5-cdb02", @@ -193,10 +193,10 @@ "version": "1.3.3" }, "windshaft": { - "version": "0.12.3", + "version": "0.12.4", "dependencies": { "grainstore": { - "version": "0.13.2", + "version": "0.13.3", "dependencies": { "carto": { "version": "0.9.3-cdb3", @@ -346,6 +346,9 @@ "mapnik": { "version": "0.7.22" }, + "strftime": { + "version": "0.6.0" + }, "lzma": { "version": "1.2.3" }, diff --git a/package.json b/package.json index d315ace0..332f5591 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "1.2.1", + "version": "1.3.0", "description": "A map tile server for CartoDB", "url": "https://github.com/Vizzuality/Windshaft-cartodb", "licenses": [{ @@ -21,13 +21,14 @@ "cluster2": "git://github.com/CartoDB/cluster2.git#cdb_production", "node-varnish": "0.1.1", "underscore" : "~1.3.3", - "windshaft" : "~0.12.3", + "windshaft" : "~0.12.4", "step": "0.0.x", "generic-pool": "~2.0.3", "redis": "0.7.2", "hiredis": "~0.1.14", "request": "2.9.202", "mapnik": "~0.7.14", + "strftime": "~0.6.0", "lzma": "~1.2.3" }, "devDependencies": { diff --git a/test/acceptance/multilayer.js b/test/acceptance/multilayer.js index 00a01290..4bd08e8d 100644 --- a/test/acceptance/multilayer.js +++ b/test/acceptance/multilayer.js @@ -6,7 +6,9 @@ var querystring = require('querystring'); var semver = require('semver'); var mapnik = require('mapnik'); var Step = require('step'); +var strftime = require('strftime'); var SQLAPIEmu = require(__dirname + '/../support/SQLAPIEmu.js'); +var redis_stats_db = 5; require(__dirname + '/../support/test_helper'); @@ -330,6 +332,7 @@ suite('multilayer', function() { test("layergroup creation raises mapviews counter", function(done) { var layergroup = { + stat_tag: 'random_tag', version: '1.0.0', layers: [ { options: { @@ -340,18 +343,18 @@ suite('multilayer', function() { } } ] }; - var statskey = "tiler:users:localhost"; - var mapviews_field = 'mapviews'; + var statskey = "user:localhost:mapviews"; var redis_stats_client = redis.createClient(global.environment.redis.port); - var redis_stats_db = 5; var expected_token; // will be set on first post and checked on second + var now = strftime("%Y%m%d", new Date()); + var errors = []; Step( function clean_stats() { var next = this; redis_stats_client.select(redis_stats_db, function(err) { if ( err ) next(err); - else redis_stats_client.del(statskey, next); + else redis_stats_client.del(statskey+':global', next); }); }, function do_post_1(err) @@ -366,12 +369,19 @@ suite('multilayer', function() { }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); expected_token = JSON.parse(res.body).layergroupid; - redis_stats_client.hget(statskey, mapviews_field, next); + redis_stats_client.zscore(statskey + ":global", now, next); }); }, - function check_stats_1_do_post_2(err, val) { + function check_global_stats_1(err, val) { if ( err ) throw err; - assert.equal(val, 1); + assert.equal(val, 1, "Expected score of " + now + " in " + + statskey + ":global to be 1, got " + val); + redis_stats_client.zscore(statskey+':stat_tag:random_tag', now, this); + }, + function check_tag_stats_1_do_post_2(err, val) { + if ( err ) throw err; + assert.equal(val, 1, "Expected score of " + now + " in " + + statskey + ":stat_tag:" + layergroup.stat_tag + " to be 1, got " + val); var next = this; assert.response(server, { url: '/tiles/layergroup', @@ -381,44 +391,87 @@ suite('multilayer', function() { }, {}, function(res) { assert.equal(res.statusCode, 200, res.body); assert.equal(JSON.parse(res.body).layergroupid, expected_token); - redis_stats_client.hget(statskey, mapviews_field, next); + redis_stats_client.zscore(statskey+':global', now, next); }); }, - function check_stats_2(err, val) + function check_global_stats_2(err, val) { if ( err ) throw err; - assert.equal(val, 2); + assert.equal(val, 2, "Expected score of " + now + " in " + + statskey + ":global to be 2, got " + val); + redis_stats_client.zscore(statskey+':stat_tag:' + layergroup.stat_tag, now, this); + }, + function check_tag_stats_2(err, val) + { + if ( err ) throw err; + assert.equal(val, 2, "Expected score of " + now + " in " + + statskey + ":stat_tag:" + layergroup.stat_tag + " to be 2, got " + val); return 1; }, - function finish(err) { - var errors = []; - if ( err ) { - errors.push(err.message); - console.log("Error: " + err); - } + function cleanup_map_style(err) { + if ( err ) errors.push('' + err); + var next = this; // trip epoch expected_token = expected_token.split(':')[0]; redis_client.keys("map_style|cartodb_test_user_1_db|~" + expected_token, function(err, matches) { - if ( err ) errors.push(err.message); - assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches); - redis_client.del(matches, function(err) { - if ( err ) errors.push(err.message); - if ( errors.length ) done(new Error(errors)); - else done(null); - }); + redis_client.del(matches, next); }); + }, + function cleanup_stats(err) { + if ( err ) errors.push('' + err); + redis_client.del([statskey+':global', statskey+':stat_tag:'+layergroup.stat_tag], this); + }, + function finish(err) { + if ( err ) errors.push('' + err); + if ( errors.length ) done(new Error(errors.join(','))); + else done(null); } ); }); + test("layergroup creation fails if CartoCSS is bogus", function(done) { + var layergroup = { + stat_tag: 'random_tag', + version: '1.0.0', + layers: [ + { options: { + sql: 'select 1 as cartodb_id, !pixel_height! as h' + + 'ST_Buffer(!bbox!, -32*greatest(!pixel_width!,!pixel_height!)) as the_geom_webmercator', + cartocss: '#layer { polygon-fit:red; }', + cartocss_version: '2.0.1' + } } + ] + }; + assert.response(server, { + url: '/tiles/layergroup', + method: 'POST', + headers: {host: 'localhost', 'Content-Type': 'application/json' }, + data: JSON.stringify(layergroup) + }, {}, function(res) { + assert.equal(res.statusCode, 400, res.body); + var parsed = JSON.parse(res.body); + assert.ok(parsed.errors[0].match(/^style0/)); + assert.ok(parsed.errors[0].match(/Unrecognized rule: polygon-fit/)); + done(); + }); + }); + suiteTeardown(function(done) { + // This test will add map_style records, like // 'map_style|null|publicuser|my_table', redis_client.keys("map_style|*", function(err, matches) { redis_client.del(matches, function(err) { - sqlapi_server.close(done); + redis_client.select(5, function(err, matches) { + redis_client.keys("user:localhost:mapviews*", function(err, matches) { + redis_client.del(matches, function(err) { + sqlapi_server.close(done); + }); + }); + }); }); }); + }); }); diff --git a/test/support/SQLAPIEmu.js b/test/support/SQLAPIEmu.js index cbb61b7c..99e9b8c6 100644 --- a/test/support/SQLAPIEmu.js +++ b/test/support/SQLAPIEmu.js @@ -14,7 +14,7 @@ var o = function(port, cb) { } else if ( query.q.match('EPOCH.* as max') ) { // This is the structure of the known query sent by tiler var row = { - 'max': 1234567890123 + 'max': 1234567890.123 }; res.write(JSON.stringify({rows: [ row ]})); } else { diff --git a/tools/convert_database_styles b/tools/convert_database_styles index db6c14c6..e69fa37c 100755 --- a/tools/convert_database_styles +++ b/tools/convert_database_styles @@ -2,7 +2,7 @@ var path = require('path'); -var grainstore = require('grainstore'); +var grainstore = require('../node_modules/windshaft/node_modules/grainstore'); var mapnik = require('mapnik'); var redis = require('redis');