cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

CartoDB redis interaction delegated to "cartodb-redis" module

Browse Source
This commit is contained in:
Sandro Santilli
2013-12-17 17:17:14 +01:00
parent e412a0f4b6
commit a60a3adc12
8 changed files with 33 additions and 518 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/cartodb/carto_data.js
-352
View File
@@ -1,352 +0,0 @@
/**
* User: simon
* Date: 30/08/2011
* Time: 21:10
* Desc: CartoDB helper.
* Retrieves dbname (based on subdomain/username)
* and geometry type from the redis stores of cartodb
*/
var strftime = require('strftime');
var RedisPool = require("./redis_pool")
, _ = require('underscore')
, Step = require('step');
module.exports = function() {
var redis_pool = new RedisPool(global.environment.redis);
var me = {
user_metadata_db: 5,
table_metadata_db: 0,
user_key: "rails:users:<%= username %>",
table_key: "rails:<%= database_name %>:<%= table_name %>",
global_mapview_key: "user:<%= username %>:mapviews:global",
tagged_mapview_key: "user:<%= username %>:mapviews:stat_tag:<%= stat_tag %>"
};
me.userFromHostname = function(hostname) {
return hostname.split('.')[0];
}
/**
* Get the database name for this particular subdomain/username
*
* @param req - standard express req object. importantly contains host information
* @param callback - gets called with args(err, dbname)
*/
me.getDatabase = function(req, callback) {
// strip subdomain from header host
var username = me.userFromHostname(req.headers.host);
var redisKey = _.template(this.user_key, {username: username});
this.retrieve(this.user_metadata_db, redisKey, 'database_name', function(err, dbname) {
if ( err ) callback(err, null);
else if ( dbname === null ) {
callback(new Error("missing " + username + "'s database_name in redis (try CARTODB/script/restore_redis)"), null);
}
else callback(err, dbname);
});
};
/**
* Increment mapview count for a user
*
* @param username
* @param stat_tag
* @param callback will be called with the new value
*/
me.incMapviewCount = function(username, stat_tag, callback) {
var that = this;
var now = strftime("%Y%m%d", new Date());
var redisKey;
Step (
function incrementGlobal() {
redisKey = _.template(that.global_mapview_key, {username: username});
that.redisCmd(me.user_metadata_db, 'ZINCRBY', [redisKey, 1, now], this);
},
function incrementTag(err, val) {
if ( err ) throw err;
if ( _.isUndefined(stat_tag) ) return 1;
redisKey = _.template(that.tagged_mapview_key, {username: username, stat_tag: stat_tag});
that.redisCmd(me.user_metadata_db, 'ZINCRBY', [redisKey, 1, now], this);
},
function finish(err, val) {
if ( callback ) callback(err);
}
);
};
/**
* Get the user id for this particular subdomain/username
*
* @param req - standard express req object. importantly contains host information
* @param callback
*/
me.getId= function(req, callback) {
// strip subdomain from header host
var username = me.userFromHostname(req.headers.host);
var redisKey = _.template(this.user_key, {username: username});
this.retrieve(this.user_metadata_db, redisKey, 'id', function(err, dbname) {
if ( err ) callback(err, null);
else if ( dbname === null ) {
callback(new Error("missing " + username + "'s id in redis (try CARTODB/script/restore_redis)"), null);
}
else callback(err, dbname);
});
};
/**
* Get the database host this particular subdomain/username
*
* @param req - standard express req object. importantly contains host information
* @param callback
*/
me.getDatabaseHost= function(req, callback) {
// strip subdomain from header host
var username = me.userFromHostname(req.headers.host);
var redisKey = _.template(this.user_key, {username: username});
this.retrieve(this.user_metadata_db, redisKey, 'database_host', function(err, dbname) {
if ( err ) callback(err, null);
else {
if ( dbname === null ) {
/* database_host was introduced in cartodb-2.5.0,
* for older versions we'll just use configured host */
//console.log("WARNING: missing " + username + "'s database_host in redis (try CARTODB/script/restore_redis)");
}
callback(err, dbname);
}
});
};
/**
* Get the database password for this particular subdomain/username
*
* @param req - standard express req object. importantly contains host information
* @param callback
*/
me.getDatabasePassword= function(req, callback) {
// strip subdomain from header host
var username = me.userFromHostname(req.headers.host);
var redisKey = _.template(this.user_key, {username: username});
this.retrieve(this.user_metadata_db, redisKey, 'database_password', function(err, data) {
if ( err ) callback(err, null);
else {
if ( data === null ) {
/* database_password was introduced in cartodb-2.5.0,
* for older versions we'll just use configured password */
//console.log("WARNING: missing " + username + "'s database_password in redis (try CARTODB/script/restore_redis)");
}
callback(err, data);
}
});
};
/**
* Check the user map key for this particular subdomain/username
*
* @param req - standard express req object. importantly contains host information
* @param callback
*/
me.checkMapKey = function(req, callback) {
// strip subdomain from header host
var username = me.userFromHostname(req.headers.host);
var redisKey = "rails:users:" + username;
this.retrieve(this.user_metadata_db, redisKey, "map_key", function(err, val) {
var valid = 0;
if ( val ) {
if ( val == req.query.map_key ) valid = 1;
else if ( val == req.query.api_key ) valid = 1;
// check also in request body
else if ( req.body && req.body.map_key && val == req.body.map_key ) valid = 1;
else if ( req.body && req.body.api_key && val == req.body.api_key ) valid = 1;
}
callback(err, valid);
});
};
/**
* Get privacy for cartodb table
*
* @param req - standard req object. Importantly contains table and host information
* @param callback - is the table private or not?
*/
me.authorize= function(req, callback) {
var that = this;
Step(
function(){
that.checkMapKey(req, this);
},
function checkIfInternal(err, check_result){
if (err) throw err;
// if unauthorized continue to check table privacy
if (check_result !== 1) return true;
// authorized by key, login as db owner
var user_params = {};
var auth_user = global.settings.postgres_auth_user;
var auth_pass = global.settings.postgres_auth_pass;
Step(
function getId() {
that.getId(req, this);
},
function(err, user_id) {
if (err) throw err;
user_params['user_id'] = user_id;
var dbuser = _.template(auth_user, user_params);
_.extend(req.params, {dbuser:dbuser});
// skip looking up user_password if postgres_auth_pass
// doesn't contain the "user_password" label
if ( ! auth_pass.match(/\buser_password\b/) ) return null;
that.getDatabasePassword(req, this);
},
function(err, user_password) {
if (err) throw err;
user_params['user_password'] = user_password;
if ( auth_pass ) {
var dbpass = _.template(auth_pass, user_params);
_.extend(req.params, {dbpassword:dbpass});
}
return true;
},
function finish(err) {
callback(err, true); // authorized (or error)
}
);
}
,function getDatabase(err, data){
if (err) throw err;
that.getDatabase(req, this);
},
function getPrivacy(err, data){
if (err) throw err;
var redisKey = _.template(that.table_key, {database_name: data, table_name: req.params.table});
that.retrieve(that.table_metadata_db, redisKey, 'privacy', this);
},
function(err, data){
callback(err, data);
}
);
};
/**
* Get the geometry type for this particular table;
* @param req - standard req object. Importantly contains table and host information
* @param callback
*/
me.getGeometryType = function(req, callback){
var that = this;
Step(
function(){
that.getDatabase(req, this)
},
function(err, data){
if (err) throw err;
var redisKey = _.template(that.table_key, {database_name: data, table_name: req.params.table});
that.retrieve(that.table_metadata_db, redisKey, 'the_geom_type', this);
},
function(err, data){
callback(err, data);
}
);
};
me.getInfowindow = function(req, callback){
var that = this;
Step(
function(){
that.getDatabase(req, this);
},
function(err, data) {
if (err) throw err;
var redisKey = _.template(that.table_key, {database_name: data, table_name: req.params.table});
that.retrieve(that.table_metadata_db, redisKey, 'infowindow', this);
},
function(err, data){
callback(err, data);
}
);
};
me.getMapMetadata = function(req, callback){
var that = this;
Step(
function(){
that.getDatabase(req, this);
},
function(err, data) {
if (err) throw err;
var redisKey = _.template(that.table_key, {database_name: data, table_name: req.params.table});
that.retrieve(that.table_metadata_db, redisKey, 'map_metadata', this);
},
function(err, data){
callback(err, data);
}
);
};
// Redis Hash lookup
// @param callback will be invoked with args (err, reply)
// note that reply is null when the key is missing
me.retrieve = function(db, redisKey, hashKey, callback) {
this.redisCmd(db,'HGET',[redisKey, hashKey], callback);
};
// Redis Set member check
me.inSet = function(db, setKey, member, callback) {
this.redisCmd(db,'SISMEMBER',[setKey, member], callback);
};
// Redis INCREMENT
me.increment = function(db, key, callback) {
this.redisCmd(db,'INCR', key, callback);
};
/**
* Use Redis
*
* @param db - redis database number
* @param redisFunc - the redis function to execute
* @param redisArgs - the arguments for the redis function in an array
* @param callback - function to pass results too.
*/
me.redisCmd = function(db, redisFunc, redisArgs, callback) {
var redisClient;
Step(
function getRedisClient() {
redis_pool.acquire(db, this);
},
function executeQuery(err, data) {
if ( err ) throw err;
redisClient = data;
redisArgs.push(this);
redisClient[redisFunc.toUpperCase()].apply(redisClient, redisArgs);
},
function releaseRedisClient(err, data) {
if ( ! _.isUndefined(redisClient) ) redis_pool.release(db, redisClient);
callback(err, data);
}
);
};
return me;
}();
lib/cartodb/redis_pool.js
-80
View File
@@ -1,80 +0,0 @@
/**
* RedisPool. A database specific redis pooling lib
*
*/
var redis = require('redis')
, _ = require('underscore')
, Pool = require('generic-pool').Pool;
// constructor.
//
// - `opts` {Object} optional config for redis and pooling
var RedisPool = function(opts){
var opts = opts || {};
var defaults = {
host: '127.0.0.1',
port: '6379',
max: 50,
idleTimeoutMillis: 10000,
reapIntervalMillis: 1000,
log: false
};
var options = _.defaults(opts, defaults)
var me = {
pools: {} // cached pools by DB name
};
// Acquire resource.
//
// - `database` {String} redis database name
// - `callback` {Function} callback to call once acquired. Takes the form
// `callback(err, resource)`
me.acquire = function(database, callback) {
if (!this.pools[database]) {
this.pools[database] = this.makePool(database);
}
this.pools[database].acquire(function(err,resource) {
callback(err, resource);
});
};
// Release resource.
//
// - `database` {String} redis database name
// - `resource` {Object} resource object to release
me.release = function(database, resource) {
this.pools[database] && this.pools[database].release(resource);
};
// Factory for pool objects.
me.makePool = function(database) {
return Pool({
name: database,
create: function(callback){
var client = redis.createClient(options.port, options.host);
client.on('connect', function () {
client.send_anyway = true;
client.select(database);
client.send_anyway = false;
callback(null, client);
});
client.on('error', function (err) {
callback(err, null);
});
},
destroy: function(client) {
return client.quit();
},
max: options.max,
idleTimeoutMillis: options.idleTimeoutMillis,
reapIntervalMillis: options.reapIntervalMillis,
log: options.log
});
};
return me;
};
module.exports = RedisPool;
lib/cartodb/server_options.js
+1 -1
View File
@@ -1,6 +1,6 @@
var _ = require('underscore')
, Step = require('step')
, cartoData = require('./carto_data')
, cartoData = require('cartodb-redis')
, Cache = require('./cache_validator')
, mapnik = require('mapnik')
, crypto = require('crypto')