From a60a3adc12bd6a8ea147bb6835d7145bbd5e3c96 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Fri, 15 Nov 2013 15:49:04 +0100 Subject: [PATCH] CartoDB redis interaction delegated to "cartodb-redis" module --- Makefile | 1 - NEWS.md | 4 + lib/cartodb/carto_data.js | 352 --------------------------- lib/cartodb/redis_pool.js | 80 ------ lib/cartodb/server_options.js | 2 +- npm-shrinkwrap.json | 42 ++-- package.json | 7 +- test/unit/cartodb/redis_pool.test.js | 63 ----- 8 files changed, 33 insertions(+), 518 deletions(-) delete mode 100644 lib/cartodb/carto_data.js delete mode 100644 lib/cartodb/redis_pool.js delete mode 100644 test/unit/cartodb/redis_pool.test.js diff --git a/Makefile b/Makefile index 6dd172f2..efb8a107 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,6 @@ config/environments/test.js: config.status--test check-local: config/environments/test.js ./run_tests.sh ${RUNTESTFLAGS} \ - test/unit/cartodb/redis_pool.test.js \ test/unit/cartodb/req2params.test.js \ test/acceptance/cache_validator.js \ test/acceptance/server.js \ diff --git a/NEWS.md b/NEWS.md index 51258ac6..ddd45257 100644 --- a/NEWS.md +++ b/NEWS.md @@ -14,6 +14,10 @@ Bug fixes: * Fix http status on database authentication error (windshaft/#94) * Fix text-face-name error at layergroup creation (windshaft/#93) +Other changes: + +* CartoDB redis interaction delegated to "cartodb-redis" module + 1.4.1 -- 2013-11-08 ------------------- diff --git a/lib/cartodb/carto_data.js b/lib/cartodb/carto_data.js deleted file mode 100644 index fe093bfb..00000000 --- a/lib/cartodb/carto_data.js +++ /dev/null @@ -1,352 +0,0 @@ -/** - * User: simon - * Date: 30/08/2011 - * Time: 21:10 - * Desc: CartoDB helper. - * Retrieves dbname (based on subdomain/username) - * and geometry type from the redis stores of cartodb - */ - -var strftime = require('strftime'); - -var RedisPool = require("./redis_pool") - , _ = require('underscore') - , Step = require('step'); - -module.exports = function() { - var redis_pool = new RedisPool(global.environment.redis); - - - var me = { - user_metadata_db: 5, - table_metadata_db: 0, - user_key: "rails:users:<%= username %>", - table_key: "rails:<%= database_name %>:<%= table_name %>", - global_mapview_key: "user:<%= username %>:mapviews:global", - tagged_mapview_key: "user:<%= username %>:mapviews:stat_tag:<%= stat_tag %>" - }; - - me.userFromHostname = function(hostname) { - return hostname.split('.')[0]; - } - - /** - * Get the database name for this particular subdomain/username - * - * @param req - standard express req object. importantly contains host information - * @param callback - gets called with args(err, dbname) - */ - me.getDatabase = function(req, callback) { - // strip subdomain from header host - var username = me.userFromHostname(req.headers.host); - var redisKey = _.template(this.user_key, {username: username}); - - this.retrieve(this.user_metadata_db, redisKey, 'database_name', function(err, dbname) { - if ( err ) callback(err, null); - else if ( dbname === null ) { - callback(new Error("missing " + username + "'s database_name in redis (try CARTODB/script/restore_redis)"), null); - } - else callback(err, dbname); - }); - }; - - /** - * Increment mapview count for a user - * - * @param username - * @param stat_tag - * @param callback will be called with the new value - */ - me.incMapviewCount = function(username, stat_tag, callback) { - var that = this; - var now = strftime("%Y%m%d", new Date()); - var redisKey; - Step ( - function incrementGlobal() { - redisKey = _.template(that.global_mapview_key, {username: username}); - that.redisCmd(me.user_metadata_db, 'ZINCRBY', [redisKey, 1, now], this); - }, - function incrementTag(err, val) { - if ( err ) throw err; - if ( _.isUndefined(stat_tag) ) return 1; - redisKey = _.template(that.tagged_mapview_key, {username: username, stat_tag: stat_tag}); - that.redisCmd(me.user_metadata_db, 'ZINCRBY', [redisKey, 1, now], this); - }, - function finish(err, val) { - if ( callback ) callback(err); - } - ); - }; - - - /** - * Get the user id for this particular subdomain/username - * - * @param req - standard express req object. importantly contains host information - * @param callback - */ - me.getId= function(req, callback) { - // strip subdomain from header host - var username = me.userFromHostname(req.headers.host); - var redisKey = _.template(this.user_key, {username: username}); - - this.retrieve(this.user_metadata_db, redisKey, 'id', function(err, dbname) { - if ( err ) callback(err, null); - else if ( dbname === null ) { - callback(new Error("missing " + username + "'s id in redis (try CARTODB/script/restore_redis)"), null); - } - else callback(err, dbname); - }); - }; - - /** - * Get the database host this particular subdomain/username - * - * @param req - standard express req object. importantly contains host information - * @param callback - */ - me.getDatabaseHost= function(req, callback) { - // strip subdomain from header host - var username = me.userFromHostname(req.headers.host); - var redisKey = _.template(this.user_key, {username: username}); - - this.retrieve(this.user_metadata_db, redisKey, 'database_host', function(err, dbname) { - if ( err ) callback(err, null); - else { - if ( dbname === null ) { - /* database_host was introduced in cartodb-2.5.0, - * for older versions we'll just use configured host */ - //console.log("WARNING: missing " + username + "'s database_host in redis (try CARTODB/script/restore_redis)"); - } - callback(err, dbname); - } - }); - }; - - /** - * Get the database password for this particular subdomain/username - * - * @param req - standard express req object. importantly contains host information - * @param callback - */ - me.getDatabasePassword= function(req, callback) { - // strip subdomain from header host - var username = me.userFromHostname(req.headers.host); - var redisKey = _.template(this.user_key, {username: username}); - - this.retrieve(this.user_metadata_db, redisKey, 'database_password', function(err, data) { - if ( err ) callback(err, null); - else { - if ( data === null ) { - /* database_password was introduced in cartodb-2.5.0, - * for older versions we'll just use configured password */ - //console.log("WARNING: missing " + username + "'s database_password in redis (try CARTODB/script/restore_redis)"); - } - callback(err, data); - } - }); - }; - - /** - * Check the user map key for this particular subdomain/username - * - * @param req - standard express req object. importantly contains host information - * @param callback - */ - me.checkMapKey = function(req, callback) { - // strip subdomain from header host - var username = me.userFromHostname(req.headers.host); - var redisKey = "rails:users:" + username; - this.retrieve(this.user_metadata_db, redisKey, "map_key", function(err, val) { - var valid = 0; - if ( val ) { - if ( val == req.query.map_key ) valid = 1; - else if ( val == req.query.api_key ) valid = 1; - // check also in request body - else if ( req.body && req.body.map_key && val == req.body.map_key ) valid = 1; - else if ( req.body && req.body.api_key && val == req.body.api_key ) valid = 1; - } - callback(err, valid); - }); - }; - - /** - * Get privacy for cartodb table - * - * @param req - standard req object. Importantly contains table and host information - * @param callback - is the table private or not? - */ - me.authorize= function(req, callback) { - var that = this; - - Step( - function(){ - that.checkMapKey(req, this); - }, - function checkIfInternal(err, check_result){ - if (err) throw err; - - // if unauthorized continue to check table privacy - if (check_result !== 1) return true; - - // authorized by key, login as db owner - var user_params = {}; - var auth_user = global.settings.postgres_auth_user; - var auth_pass = global.settings.postgres_auth_pass; - Step( - function getId() { - that.getId(req, this); - }, - function(err, user_id) { - if (err) throw err; - user_params['user_id'] = user_id; - var dbuser = _.template(auth_user, user_params); - _.extend(req.params, {dbuser:dbuser}); - - // skip looking up user_password if postgres_auth_pass - // doesn't contain the "user_password" label - if ( ! auth_pass.match(/\buser_password\b/) ) return null; - - that.getDatabasePassword(req, this); - }, - function(err, user_password) { - if (err) throw err; - user_params['user_password'] = user_password; - if ( auth_pass ) { - var dbpass = _.template(auth_pass, user_params); - _.extend(req.params, {dbpassword:dbpass}); - } - return true; - }, - function finish(err) { - callback(err, true); // authorized (or error) - } - ); - } - ,function getDatabase(err, data){ - if (err) throw err; - that.getDatabase(req, this); - }, - function getPrivacy(err, data){ - if (err) throw err; - var redisKey = _.template(that.table_key, {database_name: data, table_name: req.params.table}); - - that.retrieve(that.table_metadata_db, redisKey, 'privacy', this); - }, - function(err, data){ - callback(err, data); - } - ); - }; - - - /** - * Get the geometry type for this particular table; - * @param req - standard req object. Importantly contains table and host information - * @param callback - */ - me.getGeometryType = function(req, callback){ - var that = this; - - Step( - function(){ - that.getDatabase(req, this) - }, - function(err, data){ - if (err) throw err; - var redisKey = _.template(that.table_key, {database_name: data, table_name: req.params.table}); - - that.retrieve(that.table_metadata_db, redisKey, 'the_geom_type', this); - }, - function(err, data){ - callback(err, data); - } - ); - }; - - - me.getInfowindow = function(req, callback){ - var that = this; - - Step( - function(){ - that.getDatabase(req, this); - }, - function(err, data) { - if (err) throw err; - var redisKey = _.template(that.table_key, {database_name: data, table_name: req.params.table}); - that.retrieve(that.table_metadata_db, redisKey, 'infowindow', this); - }, - function(err, data){ - callback(err, data); - } - ); - }; - - - me.getMapMetadata = function(req, callback){ - var that = this; - - Step( - function(){ - that.getDatabase(req, this); - }, - function(err, data) { - if (err) throw err; - var redisKey = _.template(that.table_key, {database_name: data, table_name: req.params.table}); - - that.retrieve(that.table_metadata_db, redisKey, 'map_metadata', this); - }, - function(err, data){ - callback(err, data); - } - ); - }; - - // Redis Hash lookup - // @param callback will be invoked with args (err, reply) - // note that reply is null when the key is missing - me.retrieve = function(db, redisKey, hashKey, callback) { - this.redisCmd(db,'HGET',[redisKey, hashKey], callback); - }; - - // Redis Set member check - me.inSet = function(db, setKey, member, callback) { - this.redisCmd(db,'SISMEMBER',[setKey, member], callback); - }; - - // Redis INCREMENT - me.increment = function(db, key, callback) { - this.redisCmd(db,'INCR', key, callback); - }; - - /** - * Use Redis - * - * @param db - redis database number - * @param redisFunc - the redis function to execute - * @param redisArgs - the arguments for the redis function in an array - * @param callback - function to pass results too. - */ - me.redisCmd = function(db, redisFunc, redisArgs, callback) { - var redisClient; - - Step( - function getRedisClient() { - redis_pool.acquire(db, this); - }, - function executeQuery(err, data) { - if ( err ) throw err; - redisClient = data; - redisArgs.push(this); - redisClient[redisFunc.toUpperCase()].apply(redisClient, redisArgs); - }, - function releaseRedisClient(err, data) { - if ( ! _.isUndefined(redisClient) ) redis_pool.release(db, redisClient); - callback(err, data); - } - ); - }; - - return me; -}(); diff --git a/lib/cartodb/redis_pool.js b/lib/cartodb/redis_pool.js deleted file mode 100644 index 2ef86461..00000000 --- a/lib/cartodb/redis_pool.js +++ /dev/null @@ -1,80 +0,0 @@ -/** - * RedisPool. A database specific redis pooling lib - * - */ - -var redis = require('redis') - , _ = require('underscore') - , Pool = require('generic-pool').Pool; - -// constructor. -// -// - `opts` {Object} optional config for redis and pooling -var RedisPool = function(opts){ - var opts = opts || {}; - var defaults = { - host: '127.0.0.1', - port: '6379', - max: 50, - idleTimeoutMillis: 10000, - reapIntervalMillis: 1000, - log: false - }; - var options = _.defaults(opts, defaults) - - var me = { - pools: {} // cached pools by DB name - }; - - // Acquire resource. - // - // - `database` {String} redis database name - // - `callback` {Function} callback to call once acquired. Takes the form - // `callback(err, resource)` - me.acquire = function(database, callback) { - if (!this.pools[database]) { - this.pools[database] = this.makePool(database); - } - this.pools[database].acquire(function(err,resource) { - callback(err, resource); - }); - }; - - // Release resource. - // - // - `database` {String} redis database name - // - `resource` {Object} resource object to release - me.release = function(database, resource) { - this.pools[database] && this.pools[database].release(resource); - }; - - // Factory for pool objects. - me.makePool = function(database) { - return Pool({ - name: database, - create: function(callback){ - var client = redis.createClient(options.port, options.host); - client.on('connect', function () { - client.send_anyway = true; - client.select(database); - client.send_anyway = false; - callback(null, client); - }); - client.on('error', function (err) { - callback(err, null); - }); - }, - destroy: function(client) { - return client.quit(); - }, - max: options.max, - idleTimeoutMillis: options.idleTimeoutMillis, - reapIntervalMillis: options.reapIntervalMillis, - log: options.log - }); - }; - - return me; -}; - -module.exports = RedisPool; diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index 607bd1aa..ba040b58 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -1,6 +1,6 @@ var _ = require('underscore') , Step = require('step') - , cartoData = require('./carto_data') + , cartoData = require('cartodb-redis') , Cache = require('./cache_validator') , mapnik = require('mapnik') , crypto = require('crypto') diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 8636b9bd..12ccc61d 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -45,6 +45,14 @@ "mapnik-reference": { "version": "5.0.7" }, + "hiredis": { + "version": "0.1.15", + "dependencies": { + "bindings": { + "version": "1.1.1" + } + } + }, "millstone": { "version": "0.6.8", "dependencies": { @@ -197,6 +205,9 @@ } } }, + "generic-pool": { + "version": "2.0.4" + }, "express": { "version": "2.5.11", "dependencies": { @@ -258,35 +269,32 @@ "step": { "version": "0.0.5" }, - "generic-pool": { - "version": "2.0.4" - }, - "redis": { - "version": "0.8.6" - }, - "hiredis": { - "version": "0.1.15", - "dependencies": { - "bindings": { - "version": "1.1.1" - } - } - }, "request": { "version": "2.9.202" }, + "cartodb-redis": { + "version": "0.0.1", + "dependencies": { + "generic-pool": { + "version": "2.0.4" + } + } + }, "mapnik": { "version": "0.7.25" }, - "strftime": { - "version": "0.6.2" - }, "lzma": { "version": "1.2.3" }, "semver": { "version": "1.1.4" }, + "strftime": { + "version": "0.6.2" + }, + "redis": { + "version": "0.8.6" + }, "mocha": { "version": "1.2.1", "dependencies": { diff --git a/package.json b/package.json index 7306c3ee..4b6abde9 100644 --- a/package.json +++ b/package.json @@ -23,16 +23,15 @@ "underscore" : "~1.3.3", "windshaft" : "~0.14.3", "step": "0.0.x", - "generic-pool": "~2.0.3", - "redis": "~0.8.3", - "hiredis": "~0.1.14", "request": "2.9.202", + "cartodb-redis": "~0.0.1", "mapnik": "~0.7.22", - "strftime": "~0.6.0", "lzma": "~1.2.3" }, "devDependencies": { "mocha": "1.2.1", + "redis": "~0.8.3", + "strftime": "~0.6.0", "semver": "~1.1.0" }, "scripts": { diff --git a/test/unit/cartodb/redis_pool.test.js b/test/unit/cartodb/redis_pool.test.js deleted file mode 100644 index 7fa09e6a..00000000 --- a/test/unit/cartodb/redis_pool.test.js +++ /dev/null @@ -1,63 +0,0 @@ -var assert = require('../../support/assert') - , _ = require('underscore') - , RedisPool = require('../../../lib/cartodb/redis_pool') - , tests = module.exports = {}; - -suite('redis_pool', function() { - - // configure redis pool instance to use in tests - var test_opts = require('../../support/config').redis_pool; - - var redis_pool = new RedisPool(test_opts); - - test('RedisPool object exists', function(done){ - assert.ok(RedisPool); - done(); - }); - - test('RedisPool can create new redis_pool objects with default settings', function(done){ - var redis_pool = new RedisPool(); - done(); - }); - - test('RedisPool can create new redis_pool objects with specific settings', function(done){ - var redis_pool = new RedisPool(_.extend({host:'127.0.0.1', port: '6379'}, test_opts)); - done(); - }); - - - test('pool object has an acquire function', function(done){ - var found=false; - var functions = _.functions(redis_pool); - for (var i=0; i