From da12d6628dae7880e4f1e41f1ae7ae198b9e33dc Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 13 Nov 2015 16:44:26 +0100 Subject: [PATCH 1/6] Extracted redis connection and quota check to be reusable --- .../extension/sql/0.0.1/11_redis_helper.sql | 11 ++++++ server/extension/sql/0.0.1/30_admin0.sql | 19 +++++++--- .../cartodb_geocoder/quota_service.py | 6 ++-- .../cartodb_geocoder/redis_helper.py | 19 ++++++++++ .../cartodb_geocoder/user_service.py | 35 ++----------------- 5 files changed, 50 insertions(+), 40 deletions(-) create mode 100644 server/extension/sql/0.0.1/11_redis_helper.sql create mode 100644 server/lib/python/cartodb_geocoder/cartodb_geocoder/redis_helper.py diff --git a/server/extension/sql/0.0.1/11_redis_helper.sql b/server/extension/sql/0.0.1/11_redis_helper.sql new file mode 100644 index 0000000..593d9fb --- /dev/null +++ b/server/extension/sql/0.0.1/11_redis_helper.sql @@ -0,0 +1,11 @@ +-- Get the connection to redis from cache or create a new one +CREATE OR REPLACE FUNCTION cdb_geocoder_server._connect_to_redis(user_id name) +RETURNS boolean AS $$ + if user_id in GD and 'redis_connection' in GD[user_id]: + return False + else: + from cartodb_geocoder import redis_helper + redis_connection = redis_helper.RedisHelper('localhost', 6379, 5).redis_connection() + GD[user_id] = {'redis_connection': redis_connection} + return True +$$ LANGUAGE plpythonu; \ No newline at end of file diff --git a/server/extension/sql/0.0.1/30_admin0.sql b/server/extension/sql/0.0.1/30_admin0.sql index bb1fc2f..e5a5eeb 100644 --- a/server/extension/sql/0.0.1/30_admin0.sql +++ b/server/extension/sql/0.0.1/30_admin0.sql @@ -11,14 +11,23 @@ RETURNS Geometry AS $$ plpy.error('The api_key must be provided') #--TODO: rate limiting check - #--TODO: quota check + #--This will create and cache a redis connection, if needed, in the GD object for the current user + redis_conn_plan = plpy.prepare("SELECT cdb_geocoder_server._connect_to_redis($1)", ["name"]) + redis_conn_result = plpy.execute(redis_conn_plan, [user_id], 1) + qs = quota_service.QuotaService(user_id, tx_id, GD[user_id]['redis_connection']) + + if not qs.check_user_quota(): + plpy.error("Not enough quota for this user") #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html plan = plpy.prepare("SELECT cdb_geocoder_server._geocode_admin0_polygon($1) AS mypolygon", ["text"]) - rv = plpy.execute(plan, [country_name], 1) - - plpy.debug('Returning from Returning from geocode_admin0_polygons') - return rv[0]["mypolygon"] + result = plpy.execute(plan, [country_name], 1) + if result.status() == 5 and result.nrows() == 1: + qs.increment_geocoder_use() + plpy.debug('Returning from geocode_admin0_polygons') + return result[0]["mypolygon"] + else: + plpy.error('Something wrong with the georefence operation') $$ LANGUAGE plpythonu; diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py b/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py index 5d61fad..eeeb97c 100644 --- a/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py +++ b/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py @@ -1,12 +1,11 @@ -import redis import user_service from datetime import date class QuotaService: """ Class to manage all the quota operation for the Geocoder SQL API Extension """ - def __init__(self, user_id, transaction_id, **kwargs): - self._user_service = user_service.UserService(user_id, **kwargs) + def __init__(self, user_id, transaction_id, redis_connection): + self._user_service = user_service.UserService(user_id, redis_connection) self.transaction_id = transaction_id def check_user_quota(self): @@ -16,6 +15,7 @@ class QuotaService: today = date.today() current_used = self.user_service.used_quota_month(today.year, today.month) soft_geocoder_limit = self.user_service.soft_geocoder_limit() + return True if soft_geocoder_limit or (current_used + 1) < user_quota else False def increment_geocoder_use(self, amount=1): diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/redis_helper.py b/server/lib/python/cartodb_geocoder/cartodb_geocoder/redis_helper.py new file mode 100644 index 0000000..ddaa5af --- /dev/null +++ b/server/lib/python/cartodb_geocoder/cartodb_geocoder/redis_helper.py @@ -0,0 +1,19 @@ +import redis + +class RedisHelper: + + REDIS_DEFAULT_USER_DB = 5 + REDIS_DEFAULT_HOST = 'localhost' + REDIS_DEFAULT_PORT = 6379 + + def __init__(self, host, port=REDIS_DEFAULT_PORT, db=REDIS_DEFAULT_USER_DB): + self.host = host + self.port = port + self.db = db + + def redis_connection(self): + return self.__create_redis_connection() + + def __create_redis_connection(self): + pool = redis.ConnectionPool(host=self.host, port=self.port, db=self.db) + return redis.Redis(connection_pool=pool) \ No newline at end of file diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py b/server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py index 3e22d48..3a0956b 100644 --- a/server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py +++ b/server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py @@ -1,4 +1,4 @@ -import redis +import redis_helper from datetime import date class UserService: @@ -12,19 +12,9 @@ class UserService: REDIS_CONNECTION_PORT = "redis_port" REDIS_CONNECTION_DB = "redis_db" - REDIS_DEFAULT_USER_DB = 5 - REDIS_DEFAULT_HOST = 'localhost' - REDIS_DEFAULT_PORT = 6379 - - def __init__(self, user_id, **kwargs): + def __init__(self, user_id, redis_connection): self.user_id = user_id - if self.REDIS_CONNECTION_KEY in kwargs: - self._redis_connection = self.__get_redis_connection(redis_connection=kwargs[self.REDIS_CONNECTION_KEY]) - else: - if self.REDIS_CONNECTION_HOST not in kwargs: - raise Exception("You have to provide redis configuration") - redis_config = self.__build_redis_config(kwargs) - self._redis_connection = self.__get_redis_connection(redis_config = redis_config) + self._redis_connection = redis_connection def user_quota(self): # Check for exceptions or redis timeout @@ -52,25 +42,6 @@ class UserService: def redis_connection(self): return self._redis_connection - def __get_redis_connection(self, redis_connection=None, redis_config=None): - if redis_connection: - conn = redis_connection - else: - conn = self.__create_redis_connection(redis_config) - - return conn - - def __create_redis_connection(self, redis_config): - pool = redis.ConnectionPool(host=redis_config['host'], port=redis_config['port'], db=redis_config['db']) - conn = redis.Redis(connection_pool=pool) - return conn - - def __build_redis_config(self, config): - redis_host = config[self.REDIS_CONNECTION_HOST] if self.REDIS_CONNECTION_HOST in config else self.REDIS_DEFAULT_HOST - redis_port = config[self.REDIS_CONNECTION_PORT] if self.REDIS_CONNECTION_PORT in config else self.REDIS_DEFAULT_PORT - redis_db = config[self.REDIS_CONNECTION_DB] if self.REDIS_CONNECTION_DB in config else self.REDIS_DEFAULT_USER_DB - return {'host': redis_host, 'port': redis_port, 'db': redis_db} - def __get_month_redis_key(self, year, month): today = date.today() return "geocoder:{0}:{1}{2}".format(self.user_id, year, month) From b38a9b2fd3157ca2847c62f1ffcca5523b9293ac Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Mon, 16 Nov 2015 14:01:51 +0100 Subject: [PATCH 2/6] Added redis config logic --- .../extension/sql/0.0.1/11_redis_helper.sql | 21 ++++++++++++++++++- server/extension/sql/0.0.1/30_admin0.sql | 1 + .../cartodb_geocoder/redis_helper.py | 1 + 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/server/extension/sql/0.0.1/11_redis_helper.sql b/server/extension/sql/0.0.1/11_redis_helper.sql index 593d9fb..7c7791b 100644 --- a/server/extension/sql/0.0.1/11_redis_helper.sql +++ b/server/extension/sql/0.0.1/11_redis_helper.sql @@ -5,7 +5,26 @@ RETURNS boolean AS $$ return False else: from cartodb_geocoder import redis_helper - redis_connection = redis_helper.RedisHelper('localhost', 6379, 5).redis_connection() + config_params = plpy.execute("select c.host, c.port, c.timeout, c.db from cdb_geocoder_server._get_redis_conf() c;")[0] + redis_connection = redis_helper.RedisHelper(config_params['host'], config_params['port'], config_params['db']).redis_connection() GD[user_id] = {'redis_connection': redis_connection} return True +$$ LANGUAGE plpythonu; + +CREATE TYPE cdb_geocoder_server._redis_conf_params AS ( + host text, + port int, + timeout float, + db text +); + +CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_redis_conf() +RETURNS cdb_geocoder_server._redis_conf_params AS $$ + conf = plpy.execute("SELECT cdb_geocoder_server._config_get('redis_conf') conf")[0]['conf'] + if conf is None: + plpy.error("There is no redis configuration defined") + else: + import json + params = json.loads(conf) + return { "host": params['host'], "port": params['port'], 'timeout': params['timeout'], 'db': params['db'] } $$ LANGUAGE plpythonu; \ No newline at end of file diff --git a/server/extension/sql/0.0.1/30_admin0.sql b/server/extension/sql/0.0.1/30_admin0.sql index e5a5eeb..c7986d4 100644 --- a/server/extension/sql/0.0.1/30_admin0.sql +++ b/server/extension/sql/0.0.1/30_admin0.sql @@ -2,6 +2,7 @@ CREATE OR REPLACE FUNCTION cdb_geocoder_server.geocode_admin0_polygon(user_id name, tx_id bigint, country_name text) RETURNS Geometry AS $$ + from cartodb_geocoder import quota_service plpy.debug('Entering geocode_admin0_polygons') plpy.debug('user_id = %s' % user_id) diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/redis_helper.py b/server/lib/python/cartodb_geocoder/cartodb_geocoder/redis_helper.py index ddaa5af..013ac9d 100644 --- a/server/lib/python/cartodb_geocoder/cartodb_geocoder/redis_helper.py +++ b/server/lib/python/cartodb_geocoder/cartodb_geocoder/redis_helper.py @@ -15,5 +15,6 @@ class RedisHelper: return self.__create_redis_connection() def __create_redis_connection(self): + #TODO Change to use Sentinel pool = redis.ConnectionPool(host=self.host, port=self.port, db=self.db) return redis.Redis(connection_pool=pool) \ No newline at end of file From e8983283da4198d84d91be53b13bcb2e50515726 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Mon, 16 Nov 2015 16:52:50 +0100 Subject: [PATCH 3/6] Redis connection get from sentinel master --- .../extension/sql/0.0.1/11_redis_helper.sql | 50 ++++++++++++------- .../cartodb_geocoder/redis_helper.py | 20 ++++---- 2 files changed, 42 insertions(+), 28 deletions(-) diff --git a/server/extension/sql/0.0.1/11_redis_helper.sql b/server/extension/sql/0.0.1/11_redis_helper.sql index 7c7791b..e6dda61 100644 --- a/server/extension/sql/0.0.1/11_redis_helper.sql +++ b/server/extension/sql/0.0.1/11_redis_helper.sql @@ -1,23 +1,12 @@ --- Get the connection to redis from cache or create a new one -CREATE OR REPLACE FUNCTION cdb_geocoder_server._connect_to_redis(user_id name) -RETURNS boolean AS $$ - if user_id in GD and 'redis_connection' in GD[user_id]: - return False - else: - from cartodb_geocoder import redis_helper - config_params = plpy.execute("select c.host, c.port, c.timeout, c.db from cdb_geocoder_server._get_redis_conf() c;")[0] - redis_connection = redis_helper.RedisHelper(config_params['host'], config_params['port'], config_params['db']).redis_connection() - GD[user_id] = {'redis_connection': redis_connection} - return True -$$ LANGUAGE plpythonu; - CREATE TYPE cdb_geocoder_server._redis_conf_params AS ( - host text, - port int, - timeout float, - db text + sentinel_host text, + sentinel_port int, + sentinel_master_id text, + redis_db text, + timeout float ); +-- Get the Redis configuration from the _conf table -- CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_redis_conf() RETURNS cdb_geocoder_server._redis_conf_params AS $$ conf = plpy.execute("SELECT cdb_geocoder_server._config_get('redis_conf') conf")[0]['conf'] @@ -26,5 +15,30 @@ RETURNS cdb_geocoder_server._redis_conf_params AS $$ else: import json params = json.loads(conf) - return { "host": params['host'], "port": params['port'], 'timeout': params['timeout'], 'db': params['db'] } + return { + "sentinel_host": params['sentinel_host'], + "sentinel_port": params['sentinel_port'], + "sentinel_master_id": params['sentinel_master_id'], + "timeout": params['timeout'], + "redis_db": params['redis_db'] + } +$$ LANGUAGE plpythonu; + +-- Get the connection to redis from cache or create a new one +CREATE OR REPLACE FUNCTION cdb_geocoder_server._connect_to_redis(user_id name) +RETURNS boolean AS $$ + if user_id in GD and 'redis_connection' in GD[user_id]: + return False + else: + from cartodb_geocoder import redis_helper + config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, + c.sentinel_master_id, c.timeout, c.redis_db + from cdb_geocoder_server._get_redis_conf() c;""")[0] + redis_connection = redis_helper.RedisHelper(config_params['sentinel_host'], + config_params['sentinel_port'], + config_params['sentinel_master_id'], + timeout=config_params['timeout'], + redis_db=config_params['redis_db']).redis_connection() + GD[user_id] = {'redis_connection': redis_connection} + return True $$ LANGUAGE plpythonu; \ No newline at end of file diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/redis_helper.py b/server/lib/python/cartodb_geocoder/cartodb_geocoder/redis_helper.py index 013ac9d..27d4800 100644 --- a/server/lib/python/cartodb_geocoder/cartodb_geocoder/redis_helper.py +++ b/server/lib/python/cartodb_geocoder/cartodb_geocoder/redis_helper.py @@ -1,20 +1,20 @@ -import redis +from redis.sentinel import Sentinel class RedisHelper: REDIS_DEFAULT_USER_DB = 5 - REDIS_DEFAULT_HOST = 'localhost' - REDIS_DEFAULT_PORT = 6379 + REDIS_TIMEOUT = 2 #seconds - def __init__(self, host, port=REDIS_DEFAULT_PORT, db=REDIS_DEFAULT_USER_DB): - self.host = host - self.port = port - self.db = db + def __init__(self, sentinel_host, sentinel_port, sentinel_master_id, redis_db=REDIS_DEFAULT_USER_DB, **kwargs): + self.sentinel_host = sentinel_host + self.sentinel_port = sentinel_port + self.sentinel_master_id = sentinel_master_id + self.timeout = kwargs['timeout'] if 'timeout' in kwargs else REDIS_DEFAULT_TIMEOUT + self.redis_db = redis_db def redis_connection(self): return self.__create_redis_connection() def __create_redis_connection(self): - #TODO Change to use Sentinel - pool = redis.ConnectionPool(host=self.host, port=self.port, db=self.db) - return redis.Redis(connection_pool=pool) \ No newline at end of file + sentinel = Sentinel([(self.sentinel_host, 26379)], socket_timeout=self.timeout) + return sentinel.master_for(self.sentinel_master_id, socket_timeout=self.timeout, db=self.redis_db) \ No newline at end of file From e28aa9a3a2a619db552b2607c9a0ba94b2405387 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 17 Nov 2015 10:39:22 +0100 Subject: [PATCH 4/6] Removed the _conf table functions and change to use cartodb.cdb_conf --- server/extension/sql/0.0.1/10_config.sql | 38 ------------------- ...1_redis_helper.sql => 10_redis_helper.sql} | 2 +- 2 files changed, 1 insertion(+), 39 deletions(-) delete mode 100644 server/extension/sql/0.0.1/10_config.sql rename server/extension/sql/0.0.1/{11_redis_helper.sql => 10_redis_helper.sql} (94%) diff --git a/server/extension/sql/0.0.1/10_config.sql b/server/extension/sql/0.0.1/10_config.sql deleted file mode 100644 index 15d271b..0000000 --- a/server/extension/sql/0.0.1/10_config.sql +++ /dev/null @@ -1,38 +0,0 @@ --- --- This extension has its own table for configurations. --- --- The table and the function are considered to be private and therefore --- no permissions are granted for any other user but the creator/geocoder. - -CREATE TABLE IF NOT EXISTS cdb_geocoder_server._config ( KEY TEXT PRIMARY KEY, VALUE JSON NOT NULL ); - --- Needed to dump config in backups --- This can only be called from an SQL script executed by CREATE EXTENSION -SELECT pg_catalog.pg_extension_config_dump('cdb_geocoder_server._config', ''); - - -CREATE OR REPLACE FUNCTION cdb_geocoder_server._config_set(key text, value JSON) -RETURNS VOID AS $$ -BEGIN - PERFORM cdb_geocoder_server._config_remove(key); - EXECUTE 'INSERT INTO cdb_geocoder_server._config (KEY, VALUE) VALUES ($1, $2);' USING key, value; -END -$$ LANGUAGE PLPGSQL VOLATILE; - - -CREATE OR REPLACE FUNCTION cdb_geocoder_server._config_remove(key text) -RETURNS VOID AS $$ -BEGIN - EXECUTE 'DELETE FROM cdb_geocoder_server._config WHERE KEY = $1;' USING key; -END -$$ LANGUAGE PLPGSQL VOLATILE; - -CREATE OR REPLACE FUNCTION cdb_geocoder_server._config_get(key text) - RETURNS JSON AS $$ -DECLARE - value JSON; -BEGIN - EXECUTE 'SELECT VALUE FROM cdb_geocoder_server._config WHERE KEY = $1;' INTO value USING key; - RETURN value; -END -$$ LANGUAGE PLPGSQL STABLE; diff --git a/server/extension/sql/0.0.1/11_redis_helper.sql b/server/extension/sql/0.0.1/10_redis_helper.sql similarity index 94% rename from server/extension/sql/0.0.1/11_redis_helper.sql rename to server/extension/sql/0.0.1/10_redis_helper.sql index e6dda61..b39e720 100644 --- a/server/extension/sql/0.0.1/11_redis_helper.sql +++ b/server/extension/sql/0.0.1/10_redis_helper.sql @@ -9,7 +9,7 @@ CREATE TYPE cdb_geocoder_server._redis_conf_params AS ( -- Get the Redis configuration from the _conf table -- CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_redis_conf() RETURNS cdb_geocoder_server._redis_conf_params AS $$ - conf = plpy.execute("SELECT cdb_geocoder_server._config_get('redis_conf') conf")[0]['conf'] + conf = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('redis_conf') conf")[0]['conf'] if conf is None: plpy.error("There is no redis configuration defined") else: From 928e33b4894d61d9f4127e3f9a7d312c5eef02ba Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 17 Nov 2015 13:57:03 +0100 Subject: [PATCH 5/6] Added config helper for user and geocoder config --- .../cartodb_geocoder/config_helper.py | 91 +++++++++++++++++++ .../test/test_config_helper.py | 47 ++++++++++ 2 files changed, 138 insertions(+) create mode 100644 server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py create mode 100644 server/lib/python/cartodb_geocoder/test/test_config_helper.py diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py b/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py new file mode 100644 index 0000000..791bdb9 --- /dev/null +++ b/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py @@ -0,0 +1,91 @@ +import json + +class ConfigException(Exception): + pass + +class UserConfig: + + USER_CONFIG_KEYS = ['is_organization', 'entity_name'] + + def __init__(self, user_config_json): + config = json.loads(user_config_json) + filtered_config = { key: config[key] for key in self.USER_CONFIG_KEYS if key in config.keys() } + self.__check_config(filtered_config) + self.__parse_config(filtered_config) + + def __check_config(self, filtered_config): + if len(filtered_config.keys()) != len(self.USER_CONFIG_KEYS): + raise ConfigException("Passed user configuration is not correct, check it please") + + return True + + def __parse_config(self, filtered_config): + self._is_organization = filtered_config['is_organization'] + self._entity_name = filtered_config['entity_name'] + + @property + def is_organization(self): + return self._is_organization + + @property + def entity_name(self): + return self._entity_name + +class GeocoderConfig: + + GEOCODER_CONFIG_KEYS = ['street_geocoder_provider', 'google_maps_private_key', + 'nokia_monthly_quota', 'nokia_soft_geocoder_limit'] + NOKIA_GEOCODER_MANDATORY_KEYS = ['nokia_monthly_quota', 'nokia_soft_geocoder_limit'] + NOKIA_GEOCODER = 'nokia' + NOKIA_QUOTA_KEY = 'nokia_monthly_quota' + NOKIA_SOFT_LIMIT_KEY = 'nokia_soft_geocoder_limit' + GOOGLE_GEOCODER = 'google' + GEOCODER_TYPE = 'street_geocoder_provider' + GOOGLE_GEOCODER_API_KEY = 'google_maps_private_key' + + def __init__(self, geocoder_config_json): + config = json.loads(geocoder_config_json) + filtered_config = { key: config[key] for key in self.GEOCODER_CONFIG_KEYS if key in config.keys() } + self.__check_config(filtered_config) + self.__parse_config(filtered_config) + + def __check_config(self, filtered_config): + if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER: + if not set(self.NOKIA_GEOCODER_MANDATORY_KEYS).issubset(set(filtered_config.keys())): + raise ConfigException("""Nokia geocoder need the mandatory parameters 'nokia_monthly_quota' and 'nokia_soft_geocoder_limit'""") + elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER: + if self.GOOGLE_GEOCODER_API_KEY not in filtered_config.keys(): + raise ConfigException("Google geocoder need the mandatory parameter 'google_maps_private_key'") + + return True + + def __parse_config(self, filtered_config): + self._geocoder_type = filtered_config[self.GEOCODER_TYPE].lower() + self._google_maps_private_key = None + self._nokia_monthly_quota = 0 + self._nokia_soft_geocoder_limit = False + if self.GOOGLE_GEOCODER == self._geocoder_type: + self._google_maps_private_key = filtered_config[self.GOOGLE_GEOCODER_API_KEY] + elif self.NOKIA_GEOCODER == self._geocoder_type: + self._nokia_monthly_quota = filtered_config[self.NOKIA_QUOTA_KEY] + self._nokia_soft_geocoder_limit = filtered_config[self.NOKIA_SOFT_LIMIT_KEY] + + @property + def nokia_geocoder(self): + return self._geocoder_type == self.NOKIA_GEOCODER + + @property + def google_geocoder(self): + return self._geocoder_type == self.GOOGLE_GEOCODER + + @property + def google_api_key(self): + return self._google_maps_private_key + + @property + def nokia_monthly_quota(self): + return self._nokia_monthly_quota + + @property + def nokia_soft_limit(self): + return self._nokia_soft_geocoder_limit \ No newline at end of file diff --git a/server/lib/python/cartodb_geocoder/test/test_config_helper.py b/server/lib/python/cartodb_geocoder/test/test_config_helper.py new file mode 100644 index 0000000..dc4d365 --- /dev/null +++ b/server/lib/python/cartodb_geocoder/test/test_config_helper.py @@ -0,0 +1,47 @@ +from cartodb_geocoder import config_helper +from unittest import TestCase +from nose.tools import assert_raises + + +class TestConfigHelper(TestCase): + + def test_should_return_list_of_user_config_if_its_ok(self): + user_config_json = '{"is_organization": false, "entity_name": "test_user"}' + user_config = config_helper.UserConfig(user_config_json) + assert user_config.is_organization == False + assert user_config.entity_name == 'test_user' + + def test_should_return_raise_config_exception_if_not_ok(self): + user_config_json = '{"is_organization": "false"}' + assert_raises(config_helper.ConfigException, config_helper.UserConfig, user_config_json) + + def test_should_return_raise_config_exception_if_empty(self): + user_config_json = '{}' + assert_raises(config_helper.ConfigException, config_helper.UserConfig, user_config_json) + + def test_should_return_list_of_nokia_geocoder_config_if_its_ok(self): + geocoder_config_json = """{"street_geocoder_provider": "Nokia", + "nokia_monthly_quota": 100, "nokia_soft_geocoder_limit": false}""" + geocoder_config = config_helper.GeocoderConfig(geocoder_config_json) + assert geocoder_config.nokia_geocoder == True + assert geocoder_config.nokia_monthly_quota == 100 + assert geocoder_config.nokia_soft_limit == False + + def test_should_raise_configuration_exception_when_missing_nokia_geocoder_parameters(self): + geocoder_config_json = '{"street_geocoder_provider": "NokiA", "nokia_monthly_quota": "100"}' + assert_raises(config_helper.ConfigException, config_helper.GeocoderConfig, geocoder_config_json) + + def test_should_raise_configuration_exception_when_missing_nokia_geocoder_parameters_2(self): + geocoder_config_json = '{"street_geocoder_provider": "NoKia", "nokia_soft_geocoder_limit": "false"}' + assert_raises(config_helper.ConfigException, config_helper.GeocoderConfig, geocoder_config_json) + + def test_should_return_list_of_google_geocoder_config_if_its_ok(self): + geocoder_config_json = """{"street_geocoder_provider": "gOOgle", + "google_maps_private_key": "sdasdasda"}""" + geocoder_config = config_helper.GeocoderConfig(geocoder_config_json) + assert geocoder_config.google_geocoder == True + assert geocoder_config.google_api_key == 'sdasdasda' + + def test_should_raise_configuration_exception_when_missing_google_api_key(self): + geocoder_config_json = '{"street_geocoder_provider": "google"}' + assert_raises(config_helper.ConfigException, config_helper.GeocoderConfig, geocoder_config_json) \ No newline at end of file From 9e30bf2223d9ad04d08165eab9f2ab6647796123 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 17 Nov 2015 18:02:21 +0100 Subject: [PATCH 6/6] New Redis structure for services --- .gitignore | 2 +- .../extension/sql/0.0.1/10_redis_helper.sql | 2 +- server/extension/sql/0.0.1/30_admin0.sql | 22 ++--- .../cartodb_geocoder/config_helper.py | 23 ++++- .../cartodb_geocoder/quota_service.py | 34 ++++--- .../cartodb_geocoder/user_service.py | 72 +++++++++------ .../test/test_config_helper.py | 3 +- .../test/test_quota_service.py | 91 +++++++++++++++---- .../test/test_user_service.py | 66 ++++++++++---- 9 files changed, 207 insertions(+), 108 deletions(-) diff --git a/.gitignore b/.gitignore index 04f1898..dde3895 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ -*.DS_Store +.DS_Store *.pyc diff --git a/server/extension/sql/0.0.1/10_redis_helper.sql b/server/extension/sql/0.0.1/10_redis_helper.sql index b39e720..a8841fa 100644 --- a/server/extension/sql/0.0.1/10_redis_helper.sql +++ b/server/extension/sql/0.0.1/10_redis_helper.sql @@ -25,7 +25,7 @@ RETURNS cdb_geocoder_server._redis_conf_params AS $$ $$ LANGUAGE plpythonu; -- Get the connection to redis from cache or create a new one -CREATE OR REPLACE FUNCTION cdb_geocoder_server._connect_to_redis(user_id name) +CREATE OR REPLACE FUNCTION cdb_geocoder_server._connect_to_redis(user_id text) RETURNS boolean AS $$ if user_id in GD and 'redis_connection' in GD[user_id]: return False diff --git a/server/extension/sql/0.0.1/30_admin0.sql b/server/extension/sql/0.0.1/30_admin0.sql index c7986d4..e8fa519 100644 --- a/server/extension/sql/0.0.1/30_admin0.sql +++ b/server/extension/sql/0.0.1/30_admin0.sql @@ -1,8 +1,7 @@ -- Interface of the server extension -CREATE OR REPLACE FUNCTION cdb_geocoder_server.geocode_admin0_polygon(user_id name, tx_id bigint, country_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server.geocode_admin0_polygon(user_id name, user_config_data JSON, geocoder_config_data JSON, country_name text) RETURNS Geometry AS $$ - from cartodb_geocoder import quota_service plpy.debug('Entering geocode_admin0_polygons') plpy.debug('user_id = %s' % user_id) @@ -12,23 +11,14 @@ RETURNS Geometry AS $$ plpy.error('The api_key must be provided') #--TODO: rate limiting check - #--This will create and cache a redis connection, if needed, in the GD object for the current user - redis_conn_plan = plpy.prepare("SELECT cdb_geocoder_server._connect_to_redis($1)", ["name"]) - redis_conn_result = plpy.execute(redis_conn_plan, [user_id], 1) - qs = quota_service.QuotaService(user_id, tx_id, GD[user_id]['redis_connection']) - - if not qs.check_user_quota(): - plpy.error("Not enough quota for this user") + #--TODO: quota check #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html plan = plpy.prepare("SELECT cdb_geocoder_server._geocode_admin0_polygon($1) AS mypolygon", ["text"]) - result = plpy.execute(plan, [country_name], 1) - if result.status() == 5 and result.nrows() == 1: - qs.increment_geocoder_use() - plpy.debug('Returning from geocode_admin0_polygons') - return result[0]["mypolygon"] - else: - plpy.error('Something wrong with the georefence operation') + rv = plpy.execute(plan, [country_name], 1) + + plpy.debug('Returning from Returning from geocode_admin0_polygons') + return rv[0]["mypolygon"] $$ LANGUAGE plpythonu; diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py b/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py index 791bdb9..a2b148f 100644 --- a/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py +++ b/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py @@ -7,11 +7,12 @@ class UserConfig: USER_CONFIG_KEYS = ['is_organization', 'entity_name'] - def __init__(self, user_config_json): + def __init__(self, user_config_json, db_user_id = None): config = json.loads(user_config_json) filtered_config = { key: config[key] for key in self.USER_CONFIG_KEYS if key in config.keys() } self.__check_config(filtered_config) self.__parse_config(filtered_config) + self._user_id = self.__extract_uuid(db_user_id) def __check_config(self, filtered_config): if len(filtered_config.keys()) != len(self.USER_CONFIG_KEYS): @@ -19,10 +20,6 @@ class UserConfig: return True - def __parse_config(self, filtered_config): - self._is_organization = filtered_config['is_organization'] - self._entity_name = filtered_config['entity_name'] - @property def is_organization(self): return self._is_organization @@ -31,6 +28,18 @@ class UserConfig: def entity_name(self): return self._entity_name + @property + def user_id(self): + return self._user_id + + def __parse_config(self, filtered_config): + self._is_organization = filtered_config['is_organization'] + self._entity_name = filtered_config['entity_name'] + + def __extract_uuid(self, db_user_id): + # Format: development_cartodb_user_ + return db_user_id.split('_')[-1] + class GeocoderConfig: GEOCODER_CONFIG_KEYS = ['street_geocoder_provider', 'google_maps_private_key', @@ -70,6 +79,10 @@ class GeocoderConfig: self._nokia_monthly_quota = filtered_config[self.NOKIA_QUOTA_KEY] self._nokia_soft_geocoder_limit = filtered_config[self.NOKIA_SOFT_LIMIT_KEY] + @property + def service_type(self): + return self._geocoder_type + @property def nokia_geocoder(self): return self._geocoder_type == self.NOKIA_GEOCODER diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py b/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py index eeeb97c..001eb34 100644 --- a/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py +++ b/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py @@ -1,27 +1,31 @@ import user_service +import config_helper from datetime import date class QuotaService: """ Class to manage all the quota operation for the Geocoder SQL API Extension """ - def __init__(self, user_id, transaction_id, redis_connection): - self._user_service = user_service.UserService(user_id, redis_connection) - self.transaction_id = transaction_id + def __init__(self, user_config, geocoder_config, redis_connection): + self._user_config = user_config + self._geocoder_config = geocoder_config + self._user_service = user_service.UserService(self._user_config, + self._geocoder_config.service_type, redis_connection) def check_user_quota(self): """ Check if the current user quota surpasses the current quota """ - # TODO We need to add the hard/soft limit flag for the geocoder - user_quota = self.user_service.user_quota() - today = date.today() - current_used = self.user_service.used_quota_month(today.year, today.month) - soft_geocoder_limit = self.user_service.soft_geocoder_limit() + # We don't have quota check for google geocoder + if self._geocoder_config.google_geocoder: + return True - return True if soft_geocoder_limit or (current_used + 1) < user_quota else False + user_quota = self._geocoder_config.nokia_monthly_quota + today = date.today() + service_type = self._geocoder_config.service_type + current_used = self._user_service.used_quota(service_type, today.year, today.month) + soft_geocoder_limit = self._geocoder_config.nokia_soft_limit + + print "User quota: {0} --- current_used: {1} --- limit: {2}".format(user_quota, current_used, soft_geocoder_limit) + + return True if soft_geocoder_limit or current_used <= user_quota else False def increment_geocoder_use(self, amount=1): - today = date.today() - self.user_service.increment_geocoder_use(today.year, today.month, self.transaction_id) - - @property - def user_service(self): - return self._user_service \ No newline at end of file + self._user_service.increment_service_use(self._geocoder_config.service_type) \ No newline at end of file diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py b/server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py index 3a0956b..142056e 100644 --- a/server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py +++ b/server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py @@ -12,39 +12,53 @@ class UserService: REDIS_CONNECTION_PORT = "redis_port" REDIS_CONNECTION_DB = "redis_db" - def __init__(self, user_id, redis_connection): - self.user_id = user_id + def __init__(self, user_config, service_type, redis_connection): + self.user_config = user_config + self.service_type = service_type self._redis_connection = redis_connection - def user_quota(self): - # Check for exceptions or redis timeout - user_quota = self._redis_connection.hget(self.__get_user_redis_key(), self.GEOCODING_QUOTA_KEY) - return int(user_quota) if user_quota and int(user_quota) >= 0 else 0 - - def soft_geocoder_limit(self): - """ Check what kind of limit the user has """ - soft_limit = self._redis_connection.hget(self.__get_user_redis_key(), self.GEOCODING_SOFT_LIMIT_KEY) - return True if soft_limit == '1' else False - - def used_quota_month(self, year, month): + def used_quota(self, service_type, year, month, day=None): """ Recover the used quota for the user in the current month """ - # Check for exceptions or redis timeout - current_used = 0 - for _, value in self._redis_connection.hscan_iter(self.__get_month_redis_key(year,month)): - current_used += int(value) - return current_used + redis_key_data = self.__get_redis_key(service_type, year, month, day) + current_use = self._redis_connection.hget(redis_key_data['redis_name'], redis_key_data['redis_key']) + return int(current_use) if current_use else 0 - def increment_geocoder_use(self, year, month, key, amount=1): - # TODO Manage exceptions or timeout - self._redis_connection.hincrby(self.__get_month_redis_key(year, month),key,amount) + def increment_service_use(self, service_type, date=date.today(), amount=1): + """ Increment the services uses in monthly and daily basis""" + self.__increment_monthly_uses(date, service_type, amount) + self.__increment_daily_uses(date, service_type, amount) - @property - def redis_connection(self): - return self._redis_connection + # Private functions - def __get_month_redis_key(self, year, month): - today = date.today() - return "geocoder:{0}:{1}{2}".format(self.user_id, year, month) + def __increment_monthly_uses(self, date, service_type, amount): + redis_key_data = self.__get_redis_key(service_type, date.year, date.month) + self._redis_connection.hincrby(redis_key_data['redis_name'],redis_key_data['redis_key'],amount) - def __get_user_redis_key(self): - return "geocoder:{0}".format(self.user_id) \ No newline at end of file + def __increment_daily_uses(self, date, service_type, amount): + redis_key_data = self.__get_redis_key(service_type, date.year, date.month, date.day) + self._redis_connection.hincrby(redis_key_data['redis_name'],redis_key_data['redis_key'],amount) + + def __get_redis_key(self, service_type, year, month, day=None): + redis_name = self.__parse_redis_name(service_type,day) + redis_key = self.__parse_redis_key(year,month,day) + + return {'redis_name': redis_name, 'redis_key': redis_key} + + def __parse_redis_name(self,service_type, day=None): + prefix = "org" if self.user_config.is_organization else "user" + dated_key = "used_quota_day" if day else "used_quota_month" + redis_name = "{0}:{1}:{2}:{3}".format( + prefix, self.user_config.entity_name, service_type, dated_key + ) + if self.user_config.is_organization and day: + redis_name = "{0}:{1}".format(redis_name, self.user_config.user_id) + + return redis_name + + def __parse_redis_key(self,year,month,day=None): + if day: + redis_key = "{0}_{1}_{2}".format(year,month,day) + else: + redis_key = "{0}_{1}".format(year,month) + + return redis_key diff --git a/server/lib/python/cartodb_geocoder/test/test_config_helper.py b/server/lib/python/cartodb_geocoder/test/test_config_helper.py index dc4d365..061aab1 100644 --- a/server/lib/python/cartodb_geocoder/test/test_config_helper.py +++ b/server/lib/python/cartodb_geocoder/test/test_config_helper.py @@ -7,9 +7,10 @@ class TestConfigHelper(TestCase): def test_should_return_list_of_user_config_if_its_ok(self): user_config_json = '{"is_organization": false, "entity_name": "test_user"}' - user_config = config_helper.UserConfig(user_config_json) + user_config = config_helper.UserConfig(user_config_json, 'development_cartodb_user_UUID') assert user_config.is_organization == False assert user_config.entity_name == 'test_user' + assert user_config.user_id == 'UUID' def test_should_return_raise_config_exception_if_not_ok(self): user_config_json = '{"is_organization": "false"}' diff --git a/server/lib/python/cartodb_geocoder/test/test_quota_service.py b/server/lib/python/cartodb_geocoder/test/test_quota_service.py index d4c673a..a618c67 100644 --- a/server/lib/python/cartodb_geocoder/test/test_quota_service.py +++ b/server/lib/python/cartodb_geocoder/test/test_quota_service.py @@ -1,34 +1,85 @@ from mockredis import MockRedis from cartodb_geocoder import quota_service +from cartodb_geocoder import config_helper from unittest import TestCase from nose.tools import assert_raises +from datetime import datetime class TestQuotaService(TestCase): + # single user + # user:::used_quota_month:year_month + # user:::used_quota_day:year_month_day + # organization user + # org:::used_quota_month:year_month + # org::::used_quota_day:year_month_day + def setUp(self): self.fake_redis_connection = MockRedis() - self.fake_redis_connection.hset('geocoder:user_id','geocoding_quota', 100) - self.fake_redis_connection.hset('geocoder:user_id','soft_geocoder_limit', 0) - self.qs = quota_service.QuotaService('user_id', 'tx_id', redis_connection = self.fake_redis_connection) - def test_should_return_true_if_quota_with_no_use(self): - assert self.qs.check_user_quota() == True + def test_should_return_true_if_user_quota_with_no_use(self): + qs = self.__build_quota_service() + assert qs.check_user_quota() == True - def test_should_return_true_if_quota_is_not_completely_used(self): - self.fake_redis_connection.hset('geocoder:user_id:201511','tx_id', 10) - self.fake_redis_connection.hset('geocoder:user_id:201511','tx_id_2', 10) - assert self.qs.check_user_quota() == True + def test_should_return_true_if_org_quota_with_no_use(self): + qs = self.__build_quota_service(organization=True) + assert qs.check_user_quota() == True - def test_should_return_false_if_quota_is_surpassed(self): - self.fake_redis_connection.hset('geocoder:user_id','geocoding_quota', 1) - self.fake_redis_connection.hset('geocoder:user_id:201511','tx_id', 10) - self.fake_redis_connection.hset('geocoder:user_id:201511','tx_id_2', 10) - assert self.qs.check_user_quota() == False + def test_should_return_true_if_user_quota_is_not_completely_used(self): + qs = self.__build_quota_service() + self.__increment_geocoder_uses('test_user', '20151111') + assert qs.check_user_quota() == True - def test_should_return_true_if_quota_is_surpassed_but_soft_limit_is_enabled(self): - self.fake_redis_connection.hset('geocoder:user_id','geocoding_quota', 1) - self.fake_redis_connection.hset('geocoder:user_id','soft_geocoder_limit', 1) - self.fake_redis_connection.hset('geocoder:user_id:201511','tx_id', 10) - self.fake_redis_connection.hset('geocoder:user_id:201511','tx_id_2', 10) - assert self.qs.check_user_quota() == True \ No newline at end of file + def test_should_return_true_if_org_quota_is_not_completely_used(self): + qs = self.__build_quota_service(organization=True) + self.__increment_geocoder_uses('test_user', '20151111', org=True) + assert qs.check_user_quota() == True + + def test_should_return_false_if_user_quota_is_surpassed(self): + qs = self.__build_quota_service(quota = 1, soft_limit=False) + self.__increment_geocoder_uses('test_user', '20151111') + assert qs.check_user_quota() == False + + def test_should_return_false_if_org_quota_is_surpassed(self): + qs = self.__build_quota_service(organization=True, quota=1) + self.__increment_geocoder_uses('test_user', '20151111', org=True) + assert qs.check_user_quota() == False + + def test_should_return_true_if_user_quota_is_surpassed_but_soft_limit_is_enabled(self): + qs = self.__build_quota_service(quota=1, soft_limit=True) + self.__increment_geocoder_uses('test_user', '20151111') + assert qs.check_user_quota() == True + + def test_should_return_true_if_org_quota_is_surpassed_but_soft_limit_is_enabled(self): + qs = self.__build_quota_service(organization=True, quota=1, soft_limit=True) + self.__increment_geocoder_uses('test_user', '20151111', org=True) + assert qs.check_user_quota() == True + + def test_should_check_user_increment_and_quota_check_correctly(self): + qs = self.__build_quota_service(quota=2, soft_limit=False) + qs.increment_geocoder_use() + assert qs.check_user_quota() == True + + def test_should_check_org_increment_and_quota_check_correctly(self): + qs = self.__build_quota_service(organization=True, quota=2, soft_limit=False) + qs.increment_geocoder_use() + assert qs.check_user_quota() == True + + def __build_quota_service(self, quota=100, service='nokia', organization=False, soft_limit=False): + is_organization = 'true' if organization else 'false' + has_soft_limit = 'true' if soft_limit else 'false' + user_config_json = '{{"is_organization": {0}, "entity_name": "test_user"}}'.format(is_organization) + geocoder_config_json = """{{"street_geocoder_provider": "{0}","nokia_monthly_quota": {1}, + "nokia_soft_geocoder_limit": {2}}}""".format(service, quota, has_soft_limit) + user_config = config_helper.UserConfig(user_config_json, 'user_id') + geocoder_config = config_helper.GeocoderConfig(geocoder_config_json) + + return quota_service.QuotaService(user_config, geocoder_config, redis_connection = self.fake_redis_connection) + + def __increment_geocoder_uses(self, entity_name, date_string, service='nokia', amount=20, org=False): + prefix = 'org' if org else 'user' + date = datetime.strptime(date_string, "%Y%m%d") + redis_name = "{0}:{1}:{2}:used_quota_month".format(prefix, entity_name, service) + redis_key_month = "{0}_{1}".format(date.year, date.month) + self.fake_redis_connection.hset(redis_name, redis_key_month, amount) \ No newline at end of file diff --git a/server/lib/python/cartodb_geocoder/test/test_user_service.py b/server/lib/python/cartodb_geocoder/test/test_user_service.py index 491b123..2e3b969 100644 --- a/server/lib/python/cartodb_geocoder/test/test_user_service.py +++ b/server/lib/python/cartodb_geocoder/test/test_user_service.py @@ -1,37 +1,63 @@ from mockredis import MockRedis from cartodb_geocoder import user_service +from cartodb_geocoder import config_helper +from datetime import datetime from unittest import TestCase from nose.tools import assert_raises class TestUserService(TestCase): + NOKIA_GEOCODER = 'nokia' + def setUp(self): self.fake_redis_connection = MockRedis() - self.us = user_service.UserService('user_id', redis_connection = self.fake_redis_connection) - - def test_user_quota_should_be_10(self): - self.fake_redis_connection.hset('geocoder:user_id','geocoding_quota', 10) - assert self.us.user_quota() == 10 - - def test_should_return_0_if_negative_quota(self): - self.fake_redis_connection.hset('geocoder:user_id','geocoding_quota', -10) - assert self.us.user_quota() == 0 - - def test_should_return_0_if_not_user(self): - assert self.us.user_quota() == 0 def test_user_used_quota_for_a_month(self): - self.fake_redis_connection.hset('geocoder:user_id:201511','tx_id', 10) - self.fake_redis_connection.hset('geocoder:user_id:201511','tx_id_2', 10) - assert self.us.used_quota_month(2015, 11) == 20 + us = self.__build_user_service() + self.__increment_month_geocoder_uses('test_user', '20151111') + assert us.used_quota(self.NOKIA_GEOCODER, 2015, 11) == 20 + + def test_org_used_quota_for_a_month(self): + us = self.__build_user_service(organization=True) + self.__increment_month_geocoder_uses('test_user', '20151111', org=True) + assert us.used_quota(self.NOKIA_GEOCODER, 2015, 11) == 20 def test_user_not_amount_in_used_quota_for_month_should_be_0(self): - assert self.us.used_quota_month(2015, 11) == 0 + us = self.__build_user_service() + assert us.used_quota(self.NOKIA_GEOCODER, 2015, 11) == 0 - def test_increment_used_quota(self): - self.us.increment_geocoder_use(2015, 11, 'tx_id', 1) - assert self.us.used_quota_month(2015, 11) == 1 + def test_org_not_amount_in_used_quota_for_month_should_be_0(self): + us = self.__build_user_service(organization=True) + assert us.used_quota(self.NOKIA_GEOCODER, 2015, 11) == 0 + + def test_should_increment_user_used_quota(self): + us = self.__build_user_service() + date = datetime.strptime("20151111", "%Y%m%d") + us.increment_service_use(self.NOKIA_GEOCODER, date=date, amount=1) + assert us.used_quota(self.NOKIA_GEOCODER, 2015, 11) == 1 + assert us.used_quota(self.NOKIA_GEOCODER, 2015, 11, 11) == 1 + + def test_should_increment_org_used_quota(self): + us = self.__build_user_service(organization=True) + date = datetime.strptime("20151111", "%Y%m%d") + us.increment_service_use(self.NOKIA_GEOCODER, date=date, amount=1) + assert us.used_quota(self.NOKIA_GEOCODER, 2015, 11) == 1 + assert us.used_quota(self.NOKIA_GEOCODER, 2015, 11, 11) == 1 def test_exception_if_not_redis_config(self): - assert_raises(Exception, user_service.UserService, 'user_id') \ No newline at end of file + assert_raises(Exception, user_service.UserService, 'user_id') + + def __build_user_service(self, organization=False, service='nokia'): + is_organization = 'true' if organization else 'false' + user_config_json = '{{"is_organization": {0}, "entity_name": "test_user"}}'.format(is_organization) + user_config = config_helper.UserConfig(user_config_json, 'user_id') + + return user_service.UserService(user_config, service, redis_connection = self.fake_redis_connection) + + def __increment_month_geocoder_uses(self, entity_name, date_string, service='nokia', amount=20, org=False): + parent_tag = 'org' if org else 'user' + date = datetime.strptime(date_string, "%Y%m%d") + redis_name = "{0}:{1}:{2}:used_quota_month".format(parent_tag, entity_name, service) + redis_key_month = "{0}_{1}".format(date.year, date.month) + self.fake_redis_connection.hset(redis_name, redis_key_month, amount) \ No newline at end of file