Compare commits

...

16 Commits

Author SHA1 Message Date
Raul Ochoa
723a184086 Release 1.17.2 2014-10-01 20:27:11 +02:00
Raul Ochoa
db34a4ffff Stubs next version 2014-09-30 15:09:56 +02:00
Raul Ochoa
0b896fb935 Release 1.17.1 2014-09-30 15:08:26 +02:00
Raul Ochoa
e0331ec022 Upgrades mocha 2014-09-30 15:03:14 +02:00
Raul Ochoa
e31ef916f0 Upgrades windshaft to 0.27.1 which downgrades node-mapnik to 1.4.10 2014-09-30 15:02:24 +02:00
Raul Ochoa
55669f88ff Updates news 2014-09-26 11:21:28 +02:00
Raul Ochoa
c13dbc9a57 Merge pull request #214 from CartoDB/CDB-4008
TTL for template locks so they are not kept forever
2014-09-26 11:16:05 +02:00
Raul Ochoa
e8e03585ff Adds dot.js dependency 2014-09-25 19:17:02 +02:00
Raul Ochoa
b4bee864d2 Lock now considers the creation time and compares against a ttl so
a lock is not keep forever in case of failure.

Pending: lazy removal of expired locks.
2014-09-25 19:00:35 +02:00
Raul Ochoa
b41d1e84da Stubs next version 2014-09-25 16:17:10 +02:00
Raul Ochoa
3e571b4ce8 Use object.keys to iterate over objects 2014-09-25 12:17:32 +02:00
Raul Ochoa
fb8fd5121e Do not expose internal implementation 2014-09-25 12:16:34 +02:00
Raul Ochoa
ac2a3243b5 Don't cache regexes and avoid the _re hack 2014-09-25 12:04:52 +02:00
Raul Ochoa
1c10b8193b Adds dot to compile templates 2014-09-24 19:17:51 +02:00
Raul Ochoa
abf0fa1b32 Remove unused var 2014-09-24 19:12:43 +02:00
Raul Ochoa
4c5bc13c7f Check style fixes 2014-09-24 19:11:53 +02:00
5 changed files with 200 additions and 342 deletions

18
NEWS.md
View File

@@ -1,3 +1,21 @@
1.17.2 -- 2014-10-01
--------------------
Announcements:
- Upgrades windshaft to 0.27.2 which downgrades node-mapnik to 0.7.26-cdb1
1.17.1 -- 2014-09-30
--------------------
Announcements:
- Upgrades windshaft to 0.27.1 which downgrades node-mapnik to 1.4.10
Enhancements:
- TTL for template locks so they are not kept forever
- Upgrades mocha
1.17.0 -- 2014-09-25
--------------------

View File

@@ -1,10 +1,7 @@
var crypto = require('crypto');
var Step = require('step');
var _ = require('underscore');
// Templates in this hash (keyed as <username>@<template_name>)
// are being worked on.
var user_template_locks = {};
var crypto = require('crypto'),
Step = require('step'),
_ = require('underscore'),
dot = require('dot');
// Class handling map templates
//
@@ -38,17 +35,18 @@ function TemplateMaps(redis_pool, signed_maps, opts) {
//
// We have the following datastores:
//
// 1. User teplates: set of per-user map templates
// 1. User templates: set of per-user map templates
// NOTE: each template would have an associated auth
// reference, see signed_maps.js
// User templates (HASH:tpl_id->tpl_val)
this.key_usr_tpl = "map_tpl|<%= owner %>";
this.key_usr_tpl = dot.template("map_tpl|{{=it.owner}}");
// User template locks (HASH:tpl_id->ctime)
this.key_usr_tpl_lck = "map_tpl|<%= owner %>|locks";
this.key_usr_tpl_lck = dot.template("map_tpl|{{=it.owner}}|locks");
};
this.lock_ttl = this.opts['lock_ttl'] || 5000;
}
var o = TemplateMaps.prototype;
@@ -97,36 +95,34 @@ o._redisCmd = function(redisFunc, redisArgs, callback) {
// @param callback function(err, obtained)
o._obtainTemplateLock = function(owner, tpl_id, callback) {
var usr_tpl_lck_key = _.template(this.key_usr_tpl_lck, {owner:owner});
var that = this;
var gotLock = false;
Step (
function obtainLock() {
var ctime = Date.now();
that._redisCmd('HSETNX', [usr_tpl_lck_key, tpl_id, ctime], this);
},
function checkLock(err, locked) {
if ( err ) throw err;
if ( ! locked ) {
// Already locked
// TODO: unlock if expired ?
throw new Error("Template '" + tpl_id + "' of user '" + owner + "' is locked");
}
return gotLock = true;
},
function finish(err) {
callback(err, gotLock);
}
);
var that = this,
lockKey = this.key_usr_tpl_lck({owner:owner});
Step (
function obtainLock() {
that._redisCmd('HGET', [lockKey, tpl_id], this);
},
function checkLock(err, lockTime) {
if (err) { throw err; }
var _newLockTime = Date.now();
if (!lockTime || ((_newLockTime - lockTime) > that.lock_ttl)) {
that._redisCmd('HSET', [lockKey, tpl_id, _newLockTime], this);
} else {
throw new Error("Template '" + tpl_id + "' of user '" + owner + "' is locked");
}
},
function finish(err, hsetValue) {
callback(err, !!hsetValue);
}
);
};
// @param callback function(err, deleted)
o._releaseTemplateLock = function(owner, tpl_id, callback) {
var usr_tpl_lck_key = _.template(this.key_usr_tpl_lck, {owner:owner});
this._redisCmd('HDEL', [usr_tpl_lck_key, tpl_id], callback);
this._redisCmd('HDEL', [this.key_usr_tpl_lck({owner:owner}), tpl_id], callback);
};
o._reValidIdentifier = /^[a-zA-Z][0-9a-zA-Z_]*$/;
var _reValidIdentifier = /^[a-zA-Z][0-9a-zA-Z_]*$/;
o._checkInvalidTemplate = function(template) {
if ( template.version != '0.0.1' ) {
return new Error("Unsupported template version " + template.version);
@@ -135,22 +131,26 @@ o._checkInvalidTemplate = function(template) {
if ( ! tplname ) {
return new Error("Missing template name");
}
if ( ! tplname.match(this._reValidIdentifier) ) {
if ( ! tplname.match(_reValidIdentifier) ) {
return new Error("Invalid characters in template name '" + tplname + "'");
}
var phold = template.placeholders;
for (var k in phold) {
if ( ! k.match(this._reValidIdentifier) ) {
return new Error("Invalid characters in placeholder name '" + k + "'");
}
if ( ! phold[k].hasOwnProperty('default') ) {
return new Error("Missing default for placeholder '" + k + "'");
}
if ( ! phold[k].hasOwnProperty('type') ) {
return new Error("Missing type for placeholder '" + k + "'");
}
};
var placeholders = template.placeholders || {};
var placeholderKeys = Object.keys(placeholders);
for (var i = 0, len = placeholderKeys.length; i < len; i++) {
var placeholderKey = placeholderKeys[i];
if (!placeholderKey.match(_reValidIdentifier)) {
return new Error("Invalid characters in placeholder name '" + placeholderKey + "'");
}
if ( ! placeholders[placeholderKey].hasOwnProperty('default') ) {
return new Error("Missing default for placeholder '" + placeholderKey + "'");
}
if ( ! placeholders[placeholderKey].hasOwnProperty('type') ) {
return new Error("Missing type for placeholder '" + placeholderKey + "'");
}
}
// Check certificate validity
var cert = this.getTemplateCertificate(template);
@@ -168,12 +168,11 @@ o._checkInvalidTemplate = function(template) {
// SignedMaps.addCertificate or SignedMaps.authorizedByCert
//
o.getTemplateCertificate = function(template) {
var cert = {
version: '0.0.1',
template_id: template.name,
auth: template.auth
return {
version: '0.0.1',
template_id: template.name,
auth: template.auth
};
return cert;
};
// Add a template
@@ -209,7 +208,7 @@ o.addTemplate = function(owner, template, callback) {
//
//
var usr_tpl_key = _.template(this.key_usr_tpl, {owner:owner});
var usr_tpl_key = this.key_usr_tpl({owner:owner});
var gotLock = false;
var that = this;
var limit = that._userTemplateLimit();
@@ -293,7 +292,7 @@ o.addTemplate = function(owner, template, callback) {
// @param callback function(err)
//
o.delTemplate = function(owner, tpl_id, callback) {
var usr_tpl_key = _.template(this.key_usr_tpl, {owner:owner});
var usr_tpl_key = this.key_usr_tpl({owner:owner});
var gotLock = false;
var that = this;
Step(
@@ -402,7 +401,7 @@ o.updTemplate = function(owner, tpl_id, template, callback) {
return;
}
var usr_tpl_key = _.template(this.key_usr_tpl, {owner:owner});
var usr_tpl_key = this.key_usr_tpl({owner:owner});
var gotLock = false;
var that = this;
Step(
@@ -496,8 +495,7 @@ o.updTemplate = function(owner, tpl_id, template, callback) {
// Returns a list of template identifiers
//
o.listTemplates = function(owner, callback) {
var usr_tpl_key = _.template(this.key_usr_tpl, {owner:owner});
this._redisCmd('HKEYS', [ usr_tpl_key ], callback);
this._redisCmd('HKEYS', [ this.key_usr_tpl({owner:owner}) ], callback);
};
// Get a templates
@@ -511,17 +509,15 @@ o.listTemplates = function(owner, callback) {
// Return full template definition
//
o.getTemplate = function(owner, tpl_id, callback) {
var usr_tpl_key = _.template(this.key_usr_tpl, {owner:owner});
var that = this;
Step(
function getTemplate() {
that._redisCmd('HGET', [ usr_tpl_key, tpl_id ], this);
that._redisCmd('HGET', [ that.key_usr_tpl({owner:owner}), tpl_id ], this);
},
function parseTemplate(err, tpl_val) {
if ( err ) throw err;
var tpl = JSON.parse(tpl_val);
// Should we strip auth_id ?
return tpl;
return JSON.parse(tpl_val);
},
function finish(err, tpl) {
callback(err, tpl);
@@ -541,25 +537,22 @@ o.getTemplate = function(owner, tpl_id, callback) {
//
// @throws Error on malformed template or parameter
//
o._reNumber = /^([-+]?[\d\.]?\d+([eE][+-]?\d+)?)$/;
o._reCSSColorName = /^[a-zA-Z]+$/;
o._reCSSColorVal = /^#[0-9a-fA-F]{3,6}$/;
o._replaceVars = function(str, params) {
var _reNumber = /^([-+]?[\d\.]?\d+([eE][+-]?\d+)?)$/,
_reCSSColorName = /^[a-zA-Z]+$/,
_reCSSColorVal = /^#[0-9a-fA-F]{3,6}$/;
_replaceVars = function(str, params) {
//return _.template(str, params); // lazy way, possibly dangerous
// Construct regular expressions for each param
if ( ! params._re ) {
params._re = {};
for (var k in params) {
params._re[k] = RegExp("<%=\\s*" + k + "\\s*%>", "g");
}
}
for (var k in params) str = str.replace(params._re[k], params[k]);
return str;
Object.keys(params).forEach(function(k) {
str = str.replace(new RegExp("<%=\\s*" + k + "\\s*%>", "g"), params[k]);
});
return str;
};
o.instance = function(template, params) {
var all_params = {};
var phold = template.placeholders;
for (var k in phold) {
var phold = template.placeholders || {};
Object.keys(phold).forEach(function(k) {
var val = params.hasOwnProperty(k) ? params[k] : phold[k].default;
var type = phold[k].type;
// properly escape
@@ -573,7 +566,7 @@ o.instance = function(template, params) {
}
else if ( type === 'number' ) {
// check it's a number
if ( typeof(val) !== 'number' && ! val.match(this._reNumber) ) {
if ( typeof(val) !== 'number' && ! val.match(_reNumber) ) {
throw new Error("Invalid number value for template parameter '"
+ k + "': " + val);
}
@@ -581,7 +574,7 @@ o.instance = function(template, params) {
else if ( type === 'css_color' ) {
// check it only contains letters or
// starts with # and only contains hexdigits
if ( ! val.match(this._reCSSColorName) && ! val.match(this._reCSSColorVal) ) {
if ( ! val.match(_reCSSColorName) && ! val.match(_reCSSColorVal) ) {
throw new Error("Invalid css_color value for template parameter '"
+ k + "': " + val);
}
@@ -591,14 +584,14 @@ o.instance = function(template, params) {
throw new Error("Invalid placeholder type '" + type + "'");
}
all_params[k] = val;
}
});
// NOTE: we're deep-cloning the layergroup here
var layergroup = JSON.parse(JSON.stringify(template.layergroup));
for (var i=0; i<layergroup.layers.length; ++i) {
var lyropt = layergroup.layers[i].options;
if ( lyropt.cartocss ) lyropt.cartocss = this._replaceVars(lyropt.cartocss, all_params);
if ( lyropt.sql) lyropt.sql = this._replaceVars(lyropt.sql, all_params);
if ( lyropt.cartocss ) lyropt.cartocss = _replaceVars(lyropt.cartocss, all_params);
if ( lyropt.sql) lyropt.sql = _replaceVars(lyropt.sql, all_params);
// Anything else ?
}
return layergroup;

268
npm-shrinkwrap.json generated
View File

@@ -1,6 +1,6 @@
{
"name": "windshaft-cartodb",
"version": "1.17.0",
"version": "1.17.2",
"dependencies": {
"node-varnish": {
"version": "0.3.0",
@@ -9,9 +9,12 @@
"underscore": {
"version": "1.6.0"
},
"dot": {
"version": "1.0.2"
},
"windshaft": {
"version": "0.27.0",
"from": "https://github.com/CartoDB/Windshaft/tarball/0.27.0",
"version": "0.27.2",
"from": "https://github.com/CartoDB/Windshaft/tarball/0.27.2",
"dependencies": {
"chronograph": {
"version": "0.1.0",
@@ -896,252 +899,8 @@
}
},
"mapnik": {
"version": "1.4.15",
"from": "https://github.com/CartoDB/node-mapnik/tarball/1.4.15-cdb",
"dependencies": {
"nan": {
"version": "1.2.0"
},
"mapnik-vector-tile": {
"version": "0.5.5"
},
"node-pre-gyp": {
"version": "0.5.25",
"dependencies": {
"nopt": {
"version": "3.0.1",
"dependencies": {
"abbrev": {
"version": "1.0.5"
}
}
},
"npmlog": {
"version": "0.1.1",
"dependencies": {
"ansi": {
"version": "0.3.0"
}
}
},
"request": {
"version": "2.40.0",
"dependencies": {
"qs": {
"version": "1.0.2"
},
"json-stringify-safe": {
"version": "5.0.0"
},
"mime-types": {
"version": "1.0.2"
},
"forever-agent": {
"version": "0.5.2"
},
"node-uuid": {
"version": "1.4.1"
},
"tough-cookie": {
"version": "0.12.1",
"dependencies": {
"punycode": {
"version": "1.3.1"
}
}
},
"form-data": {
"version": "0.1.4",
"dependencies": {
"combined-stream": {
"version": "0.0.5",
"dependencies": {
"delayed-stream": {
"version": "0.0.5"
}
}
},
"mime": {
"version": "1.2.11"
},
"async": {
"version": "0.9.0"
}
}
},
"tunnel-agent": {
"version": "0.4.0"
},
"http-signature": {
"version": "0.10.0",
"dependencies": {
"assert-plus": {
"version": "0.1.2"
},
"asn1": {
"version": "0.1.11"
},
"ctype": {
"version": "0.5.2"
}
}
},
"oauth-sign": {
"version": "0.3.0"
},
"hawk": {
"version": "1.1.1",
"dependencies": {
"hoek": {
"version": "0.9.1"
},
"boom": {
"version": "0.4.2"
},
"cryptiles": {
"version": "0.2.2"
},
"sntp": {
"version": "0.2.4"
}
}
},
"aws-sign2": {
"version": "0.5.0"
},
"stringstream": {
"version": "0.0.4"
}
}
},
"semver": {
"version": "3.0.1"
},
"tar": {
"version": "1.0.1",
"dependencies": {
"block-stream": {
"version": "0.0.7"
},
"fstream": {
"version": "1.0.2",
"dependencies": {
"graceful-fs": {
"version": "3.0.2"
}
}
},
"inherits": {
"version": "2.0.1"
}
}
},
"tar-pack": {
"version": "2.0.0",
"dependencies": {
"uid-number": {
"version": "0.0.3"
},
"once": {
"version": "1.1.1"
},
"debug": {
"version": "0.7.4"
},
"fstream": {
"version": "0.1.31",
"dependencies": {
"graceful-fs": {
"version": "3.0.2"
},
"inherits": {
"version": "2.0.1"
}
}
},
"tar": {
"version": "0.1.20",
"dependencies": {
"block-stream": {
"version": "0.0.7"
},
"inherits": {
"version": "2.0.1"
}
}
},
"fstream-ignore": {
"version": "0.0.7",
"dependencies": {
"minimatch": {
"version": "0.2.14",
"dependencies": {
"lru-cache": {
"version": "2.5.0"
},
"sigmund": {
"version": "1.0.0"
}
}
},
"inherits": {
"version": "2.0.1"
}
}
},
"readable-stream": {
"version": "1.0.31",
"dependencies": {
"core-util-is": {
"version": "1.0.1"
},
"isarray": {
"version": "0.0.1"
},
"string_decoder": {
"version": "0.10.31"
},
"inherits": {
"version": "2.0.1"
}
}
},
"graceful-fs": {
"version": "1.2.3"
}
}
},
"mkdirp": {
"version": "0.5.0",
"dependencies": {
"minimist": {
"version": "0.0.8"
}
}
},
"rc": {
"version": "0.5.1",
"dependencies": {
"minimist": {
"version": "0.0.10"
},
"deep-extend": {
"version": "0.2.11"
},
"strip-json-comments": {
"version": "0.1.3"
},
"ini": {
"version": "1.1.0"
}
}
},
"rimraf": {
"version": "2.2.8"
}
}
}
}
"version": "0.7.26-cdb1",
"from": "http://github.com/Vizzuality/node-mapnik/tarball/0.7.26-cdb1"
},
"tilelive": {
"version": "4.5.3",
@@ -1230,12 +989,7 @@
},
"cartodb-redis": {
"version": "0.11.0",
"from": "https://github.com/CartoDB/node-cartodb-redis/tarball/0.11.0",
"dependencies": {
"dot": {
"version": "1.0.2"
}
}
"from": "https://github.com/CartoDB/node-cartodb-redis/tarball/0.11.0"
},
"cartodb-psql": {
"version": "0.4.0",
@@ -1330,13 +1084,13 @@
"version": "0.8.2"
},
"mocha": {
"version": "1.14.0",
"version": "1.21.4",
"dependencies": {
"commander": {
"version": "2.0.0"
},
"growl": {
"version": "1.7.0"
"version": "1.8.1"
},
"jade": {
"version": "0.26.3",

View File

@@ -1,7 +1,7 @@
{
"private": true,
"name": "windshaft-cartodb",
"version": "1.17.0",
"version": "1.17.2",
"description": "A map tile server for CartoDB",
"keywords": [
"cartodb"
@@ -24,7 +24,8 @@
"dependencies": {
"node-varnish": "https://github.com/Vizzuality/node-varnish/tarball/0.3.0",
"underscore" : "~1.6.0",
"windshaft": "https://github.com/CartoDB/Windshaft/tarball/0.27.0",
"dot": "~1.0.2",
"windshaft": "https://github.com/CartoDB/Windshaft/tarball/0.27.2",
"step": "~0.0.5",
"request": "~2.9.203",
"cartodb-redis": "https://github.com/CartoDB/node-cartodb-redis/tarball/0.11.0",
@@ -35,7 +36,7 @@
"rollbar": "~0.3.13"
},
"devDependencies": {
"mocha": "1.14.0",
"mocha": "~1.21.4",
"redis": "~0.8.6",
"strftime": "~0.8.2",
"semver": "~1.1.4"

View File

@@ -12,6 +12,14 @@ suite('template_maps', function() {
// configure redis pool instance to use in tests
var redis_pool = RedisPool(global.environment.redis);
var signed_maps = new SignedMaps(redis_pool);
var validTemplate = {
version:'0.0.1',
name: 'first',
auth: {},
layergroup: {}
};
var owner = 'me';
test('does not accept template with unsupported version', function(done) {
var tmap = new TemplateMaps(redis_pool, signed_maps);
@@ -502,5 +510,89 @@ suite('template_maps', function() {
}
);
});
var redisCmdFunc = TemplateMaps.prototype._redisCmd;
function runWithRedisStubbed(stubbedCommands, func) {
TemplateMaps.prototype._redisCmd = function(redisFunc, redisArgs, callback) {
redisFunc = redisFunc.toLowerCase();
if (stubbedCommands.hasOwnProperty(redisFunc)) {
callback(null, stubbedCommands[redisFunc]);
} else {
throw 'Unknown command';
}
};
func();
TemplateMaps.prototype._redisCmd = redisCmdFunc;
}
test('_obtainTemplateLock with no previous value, happy case', function(done) {
runWithRedisStubbed({hget: null, hset: 1}, function() {
var templateMaps = new TemplateMaps(redis_pool, signed_maps);
templateMaps._obtainTemplateLock(owner, validTemplate.name, function(err, gotLock) {
assert.ok(!err);
assert.ok(gotLock);
done();
});
});
});
test('_obtainTemplateLock no lock for non expired ttl, simulates obtaining two locks at same time', function(done) {
runWithRedisStubbed({hget: Date.now()}, function() {
var templateMaps = new TemplateMaps(redis_pool, signed_maps);
templateMaps._obtainTemplateLock(owner, validTemplate.name, function(err, gotLock) {
assert.ok(!!err);
assert.equal(gotLock, false);
done();
});
});
});
test('_obtainTemplateLock no lock for non expired ttl, last millisecond of valid ttl', function(done) {
var nowValue = Date.now(),
nowFunc = Date.now;
Date.now = function() {
return nowValue;
};
var lockTtl = 1000;
runWithRedisStubbed({hget: Date.now() - lockTtl, hset: true}, function() {
var templateMaps = new TemplateMaps(redis_pool, signed_maps, {lock_ttl: lockTtl});
templateMaps._obtainTemplateLock(owner, validTemplate.name, function(err, gotLock) {
assert.ok(!!err);
assert.equal(gotLock, false);
Date.now = nowFunc;
done();
});
});
});
test('_obtainTemplateLock gets lock for expired ttl, first millisecond of invalid ttl', function(done) {
var nowValue = Date.now(),
nowFunc = Date.now;
Date.now = function() {
return nowValue;
};
var lockTtl = 1000;
runWithRedisStubbed({hget: Date.now() - lockTtl - 1, hset: true}, function() {
var templateMaps = new TemplateMaps(redis_pool, signed_maps, {lock_ttl: lockTtl});
templateMaps._obtainTemplateLock(owner, validTemplate.name, function(err, gotLock) {
assert.ok(!err);
assert.ok(gotLock);
Date.now = nowFunc;
done();
});
});
});
});