From b97e83841667e9fed1f2dde63574d941f0ef681a Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 8 Mar 2016 12:24:27 +0100 Subject: [PATCH] Refactor to move logic from SQL functions to Python - Moved the logic the retrieve the redis connection params to RedisDBConfig class - Moved the logic that retrieve the services configuration to ServicesDBConfig --- .../cdb_dataservices_server--0.5.0--0.5.1.sql | 65 +++++++- .../cdb_dataservices_server--0.5.1--0.5.0.sql | 140 +++++++++++++++++- .../cdb_dataservices_server--0.5.1.sql | 91 +----------- .../extension/sql/0.5.1/10_redis_helper.sql | 19 ++- .../extension/sql/0.5.1/15_config_helper.sql | 45 +++++- .../cartodb_services/metrics/config.py | 88 ++++++++--- .../cartodb_services/metrics/log.py | 2 +- .../cartodb_services/tools/__init__.py | 2 +- .../cartodb_services/tools/redis_tools.py | 87 +++++++++-- .../cartodb_services/test/test_config.py | 18 +-- .../cartodb_services/test/test_helper.py | 16 ++ .../test/test_quota_service.py | 6 +- .../test/test_user_service.py | 7 +- 13 files changed, 445 insertions(+), 141 deletions(-) mode change 120000 => 100644 server/extension/sql/0.5.1/10_redis_helper.sql mode change 120000 => 100644 server/extension/sql/0.5.1/15_config_helper.sql diff --git a/server/extension/cdb_dataservices_server--0.5.0--0.5.1.sql b/server/extension/cdb_dataservices_server--0.5.0--0.5.1.sql index d39dbdb..647eae3 100644 --- a/server/extension/cdb_dataservices_server--0.5.0--0.5.1.sql +++ b/server/extension/cdb_dataservices_server--0.5.0--0.5.1.sql @@ -1,3 +1,66 @@ +DROP FUNCTION IF EXISTS cdb_dataservices_server._get_redis_conf_v2(text); +DROP TYPE IF EXISTS cdb_dataservices_server._redis_conf_params; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._connect_to_redis(user_id text) +RETURNS boolean AS $$ + cache_key = "redis_connection_{0}".format(user_id) + if cache_key in GD: + return False + else: + from cartodb_services.tools import RedisConnection, RedisDBConfig + metadata_config = RedisDBConfig('redis_metadata_config', plpy) + metrics_config = RedisDBConfig('redis_metrics_config', plpy) + redis_metadata_connection = RedisConnection(metadata_config).redis_connection() + redis_metrics_connection = RedisConnection(metrics_config).redis_connection() + GD[cache_key] = { + 'redis_metadata_connection': redis_metadata_connection, + 'redis_metrics_connection': redis_metrics_connection, + } + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import GeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + geocoder_config = GeocoderConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_isolines_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_isolines_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import IsolinesRoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + isolines_routing_config = IsolinesRoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = isolines_routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import RoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + routing_config = RoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ from cartodb_services.here import HereMapsGeocoder @@ -505,4 +568,4 @@ RETURNS cdb_dataservices_server.simple_route AS $$ plpy.error(error_msg) finally: quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu SECURITY DEFINER; \ No newline at end of file +$$ LANGUAGE plpythonu SECURITY DEFINER; diff --git a/server/extension/cdb_dataservices_server--0.5.1--0.5.0.sql b/server/extension/cdb_dataservices_server--0.5.1--0.5.0.sql index dab17ee..a31e055 100644 --- a/server/extension/cdb_dataservices_server--0.5.1--0.5.0.sql +++ b/server/extension/cdb_dataservices_server--0.5.1--0.5.0.sql @@ -1,3 +1,141 @@ +CREATE TYPE cdb_dataservices_server._redis_conf_params AS ( + sentinel_master_id text, + redis_host text, + redis_port int, + redis_db text, + timeout float +); + +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_redis_conf_v2(config_key text) +RETURNS cdb_dataservices_server._redis_conf_params AS $$ + conf_query = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(config_key) + conf = plpy.execute(conf_query)[0]['conf'] + if conf is None: + plpy.error("There is no redis configuration defined") + else: + import json + params = json.loads(conf) + redis_conf_params = { + "redis_host": params['redis_host'], + "redis_port": params['redis_port'], + "timeout": params['timeout'], + "redis_db": params['redis_db'] + } + if "sentinel_master_id" in params: + redis_conf_params["sentinel_master_id"] = params["sentinel_master_id"] + else: + redis_conf_params["sentinel_master_id"] = None + + return redis_conf_params +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._connect_to_redis(user_id text) +RETURNS boolean AS $$ + cache_key = "redis_connection_{0}".format(user_id) + if cache_key in GD: + return False + else: + from cartodb_services.tools import RedisConnection + metadata_config_params = plpy.execute("""select c.sentinel_master_id, c.redis_host, + c.redis_port, c.timeout, c.redis_db + from cdb_dataservices_server._get_redis_conf_v2('redis_metadata_config') c;""")[0] + metrics_config_params = plpy.execute("""select c.sentinel_master_id, c.redis_host, + c.redis_port, c.timeout, c.redis_db + from cdb_dataservices_server._get_redis_conf_v2('redis_metrics_config') c;""")[0] + redis_metadata_connection = RedisConnection(metadata_config_params['sentinel_master_id'], + metadata_config_params['redis_host'], + metadata_config_params['redis_port'], + timeout=metadata_config_params['timeout'], + redis_db=metadata_config_params['redis_db']).redis_connection() + redis_metrics_connection = RedisConnection(metrics_config_params['sentinel_master_id'], + metrics_config_params['redis_host'], + metrics_config_params['redis_port'], + timeout=metrics_config_params['timeout'], + redis_db=metrics_config_params['redis_db']).redis_connection() + GD[cache_key] = { + 'redis_metadata_connection': redis_metadata_connection, + 'redis_metrics_connection': redis_metrics_connection, + } + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + import json + from cartodb_services.metrics import GeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf'] + if not heremaps_conf_json: + heremaps_app_id = None + heremaps_app_code = None + else: + heremaps_conf = json.loads(heremaps_conf_json) + heremaps_app_id = heremaps_conf['app_id'] + heremaps_app_code = heremaps_conf['app_code'] + geocoder_config = GeocoderConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code) + # --Think about the security concerns with this kind of global cache, it should be only available + # --for this user session but... + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_isolines_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_isolines_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + import json + from cartodb_services.metrics import IsolinesRoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf'] + if not heremaps_conf_json: + heremaps_app_id = None + heremaps_app_code = None + else: + heremaps_conf = json.loads(heremaps_conf_json) + heremaps_app_id = heremaps_conf['app_id'] + heremaps_app_code = heremaps_conf['app_code'] + isolines_routing_config = IsolinesRoutingConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code) + # --Think about the security concerns with this kind of global cache, it should be only available + # --for this user session but... + GD[cache_key] = isolines_routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + import json + from cartodb_services.metrics import RoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + mapzen_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('mapzen_conf') as mapzen_conf", 1)[0]['mapzen_conf'] + if not mapzen_conf_json: + mapzen_app_key = None + else: + mapzen_conf = json.loads(mapzen_conf_json) + mapzen_app_key = mapzen_conf['routing_app_key'] + routing_config = RoutingConfig(redis_conn, username, orgname, mapzen_app_key) + # --Think about the security concerns with this kind of global cache, it should be only available + # --for this user session but... + GD[cache_key] = routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ from cartodb_services.here import HereMapsGeocoder @@ -505,4 +643,4 @@ RETURNS cdb_dataservices_server.simple_route AS $$ plpy.error(error_msg) finally: quota_service.increment_total_geocoder_use() -$$ LANGUAGE plpythonu SECURITY DEFINER; \ No newline at end of file +$$ LANGUAGE plpythonu SECURITY DEFINER; diff --git a/server/extension/cdb_dataservices_server--0.5.1.sql b/server/extension/cdb_dataservices_server--0.5.1.sql index 75b7380..ad46cf6 100644 --- a/server/extension/cdb_dataservices_server--0.5.1.sql +++ b/server/extension/cdb_dataservices_server--0.5.1.sql @@ -74,38 +74,6 @@ RETURNS cdb_dataservices_server.simple_route AS $$ result = plpy.execute(mapzen_plan, [username, orgname, origin, destination, mode, options, units]) return [result[0]['shape'],result[0]['length'], result[0]['duration']] $$ LANGUAGE plpythonu; -CREATE TYPE cdb_dataservices_server._redis_conf_params AS ( - sentinel_master_id text, - redis_host text, - redis_port int, - redis_db text, - timeout float -); - --- Get the Redis configuration from the _conf table -- -CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_redis_conf_v2(config_key text) -RETURNS cdb_dataservices_server._redis_conf_params AS $$ - conf_query = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(config_key) - conf = plpy.execute(conf_query)[0]['conf'] - if conf is None: - plpy.error("There is no redis configuration defined") - else: - import json - params = json.loads(conf) - redis_conf_params = { - "redis_host": params['redis_host'], - "redis_port": params['redis_port'], - "timeout": params['timeout'], - "redis_db": params['redis_db'] - } - if "sentinel_master_id" in params: - redis_conf_params["sentinel_master_id"] = params["sentinel_master_id"] - else: - redis_conf_params["sentinel_master_id"] = None - - return redis_conf_params -$$ LANGUAGE plpythonu; - -- Get the connection to redis from cache or create a new one CREATE OR REPLACE FUNCTION cdb_dataservices_server._connect_to_redis(user_id text) RETURNS boolean AS $$ @@ -113,23 +81,11 @@ RETURNS boolean AS $$ if cache_key in GD: return False else: - from cartodb_services.tools import RedisConnection - metadata_config_params = plpy.execute("""select c.sentinel_master_id, c.redis_host, - c.redis_port, c.timeout, c.redis_db - from cdb_dataservices_server._get_redis_conf_v2('redis_metadata_config') c;""")[0] - metrics_config_params = plpy.execute("""select c.sentinel_master_id, c.redis_host, - c.redis_port, c.timeout, c.redis_db - from cdb_dataservices_server._get_redis_conf_v2('redis_metrics_config') c;""")[0] - redis_metadata_connection = RedisConnection(metadata_config_params['sentinel_master_id'], - metadata_config_params['redis_host'], - metadata_config_params['redis_port'], - timeout=metadata_config_params['timeout'], - redis_db=metadata_config_params['redis_db']).redis_connection() - redis_metrics_connection = RedisConnection(metrics_config_params['sentinel_master_id'], - metrics_config_params['redis_host'], - metrics_config_params['redis_port'], - timeout=metrics_config_params['timeout'], - redis_db=metrics_config_params['redis_db']).redis_connection() + from cartodb_services.tools import RedisConnection, RedisDBConfig + metadata_config = RedisDBConfig('redis_metadata_config', plpy) + metrics_config = RedisDBConfig('redis_metrics_config', plpy) + redis_metadata_connection = RedisConnection(metadata_config).redis_connection() + redis_metrics_connection = RedisConnection(metrics_config).redis_connection() GD[cache_key] = { 'redis_metadata_connection': redis_metadata_connection, 'redis_metrics_connection': redis_metrics_connection, @@ -143,21 +99,10 @@ RETURNS boolean AS $$ if cache_key in GD: return False else: - import json from cartodb_services.metrics import GeocoderConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] - heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf'] - if not heremaps_conf_json: - heremaps_app_id = None - heremaps_app_code = None - else: - heremaps_conf = json.loads(heremaps_conf_json) - heremaps_app_id = heremaps_conf['app_id'] - heremaps_app_code = heremaps_conf['app_code'] - geocoder_config = GeocoderConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code) - # --Think about the security concerns with this kind of global cache, it should be only available - # --for this user session but... + geocoder_config = GeocoderConfig(redis_conn, plpy, username, orgname) GD[cache_key] = geocoder_config return True $$ LANGUAGE plpythonu SECURITY DEFINER; @@ -169,21 +114,10 @@ RETURNS boolean AS $$ if cache_key in GD: return False else: - import json from cartodb_services.metrics import IsolinesRoutingConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] - heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf'] - if not heremaps_conf_json: - heremaps_app_id = None - heremaps_app_code = None - else: - heremaps_conf = json.loads(heremaps_conf_json) - heremaps_app_id = heremaps_conf['app_id'] - heremaps_app_code = heremaps_conf['app_code'] - isolines_routing_config = IsolinesRoutingConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code) - # --Think about the security concerns with this kind of global cache, it should be only available - # --for this user session but... + isolines_routing_config = IsolinesRoutingConfig(redis_conn, plpy, username, orgname) GD[cache_key] = isolines_routing_config return True $$ LANGUAGE plpythonu SECURITY DEFINER; @@ -195,19 +129,10 @@ RETURNS boolean AS $$ if cache_key in GD: return False else: - import json from cartodb_services.metrics import RoutingConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] - mapzen_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('mapzen_conf') as mapzen_conf", 1)[0]['mapzen_conf'] - if not mapzen_conf_json: - mapzen_app_key = None - else: - mapzen_conf = json.loads(mapzen_conf_json) - mapzen_app_key = mapzen_conf['routing_app_key'] - routing_config = RoutingConfig(redis_conn, username, orgname, mapzen_app_key) - # --Think about the security concerns with this kind of global cache, it should be only available - # --for this user session but... + routing_config = RoutingConfig(redis_conn, plpy, username, orgname) GD[cache_key] = routing_config return True $$ LANGUAGE plpythonu SECURITY DEFINER; diff --git a/server/extension/sql/0.5.1/10_redis_helper.sql b/server/extension/sql/0.5.1/10_redis_helper.sql deleted file mode 120000 index 71e8ff6..0000000 --- a/server/extension/sql/0.5.1/10_redis_helper.sql +++ /dev/null @@ -1 +0,0 @@ -../0.5.0/10_redis_helper.sql \ No newline at end of file diff --git a/server/extension/sql/0.5.1/10_redis_helper.sql b/server/extension/sql/0.5.1/10_redis_helper.sql new file mode 100644 index 0000000..e010088 --- /dev/null +++ b/server/extension/sql/0.5.1/10_redis_helper.sql @@ -0,0 +1,18 @@ +-- Get the connection to redis from cache or create a new one +CREATE OR REPLACE FUNCTION cdb_dataservices_server._connect_to_redis(user_id text) +RETURNS boolean AS $$ + cache_key = "redis_connection_{0}".format(user_id) + if cache_key in GD: + return False + else: + from cartodb_services.tools import RedisConnection, RedisDBConfig + metadata_config = RedisDBConfig('redis_metadata_config', plpy) + metrics_config = RedisDBConfig('redis_metrics_config', plpy) + redis_metadata_connection = RedisConnection(metadata_config).redis_connection() + redis_metrics_connection = RedisConnection(metrics_config).redis_connection() + GD[cache_key] = { + 'redis_metadata_connection': redis_metadata_connection, + 'redis_metrics_connection': redis_metrics_connection, + } + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; diff --git a/server/extension/sql/0.5.1/15_config_helper.sql b/server/extension/sql/0.5.1/15_config_helper.sql deleted file mode 120000 index fdce38c..0000000 --- a/server/extension/sql/0.5.1/15_config_helper.sql +++ /dev/null @@ -1 +0,0 @@ -../0.5.0/15_config_helper.sql \ No newline at end of file diff --git a/server/extension/sql/0.5.1/15_config_helper.sql b/server/extension/sql/0.5.1/15_config_helper.sql new file mode 100644 index 0000000..d39a4fc --- /dev/null +++ b/server/extension/sql/0.5.1/15_config_helper.sql @@ -0,0 +1,44 @@ +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import GeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + geocoder_config = GeocoderConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_isolines_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_isolines_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import IsolinesRoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + isolines_routing_config = IsolinesRoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = isolines_routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + from cartodb_services.metrics import RoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + routing_config = RoutingConfig(redis_conn, plpy, username, orgname) + GD[cache_key] = routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index 0f318a0..129e27e 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -27,6 +27,7 @@ class ServiceConfig(object): def organization(self): return self._orgname + class RoutingConfig(ServiceConfig): ROUTING_CONFIG_KEYS = ['username', 'orgname', 'mapzen_app_key'] @@ -34,11 +35,11 @@ class RoutingConfig(ServiceConfig): USERNAME_KEY = 'username' ORGNAME_KEY = 'orgname' - def __init__(self, redis_connection, username, orgname=None, - mapzen_app_key=None): + def __init__(self, redis_connection, db_conn, username, orgname=None): super(RoutingConfig, self).__init__(redis_connection, username, - orgname) - self._mapzen_app_key = mapzen_app_key + orgname) + db_config = ServicesDBConfig(db_conn) + self._mapzen_app_key = db_config.mapzen_routing_app_key @property def service_type(self): @@ -53,7 +54,8 @@ class IsolinesRoutingConfig(ServiceConfig): ROUTING_CONFIG_KEYS = ['here_isolines_quota', 'soft_here_isolines_limit', 'period_end_date', 'username', 'orgname', - 'heremaps_app_id', 'heremaps_app_code', 'geocoder_type'] + 'heremaps_app_id', 'heremaps_app_code', + 'geocoder_type'] NOKIA_APP_ID_KEY = 'heremaps_app_id' NOKIA_APP_CODE_KEY = 'heremaps_app_code' QUOTA_KEY = 'here_isolines_quota' @@ -64,24 +66,22 @@ class IsolinesRoutingConfig(ServiceConfig): GEOCODER_TYPE_KEY = 'geocoder_type' GOOGLE_GEOCODER = 'google' - def __init__(self, redis_connection, username, orgname=None, - heremaps_app_id=None, heremaps_app_code=None): + def __init__(self, redis_connection, db_conn, username, orgname=None): super(IsolinesRoutingConfig, self).__init__(redis_connection, username, orgname) - config = self.__get_user_config(username, orgname, heremaps_app_id, - heremaps_app_code) + db_config = ServicesDBConfig(db_conn) + config = self.__get_user_config(username, orgname, db_config) filtered_config = {key: config[key] for key in self.ROUTING_CONFIG_KEYS if key in config.keys()} self.__parse_config(filtered_config) - def __get_user_config(self, username, orgname=None, heremaps_app_id=None, - heremaps_app_code=None): + def __get_user_config(self, username, orgname, db_config): user_config = self._redis_connection.hgetall( "rails:users:{0}".format(username)) if not user_config: raise ConfigException("""There is no user config available. Please check your configuration.'""") else: - user_config[self.NOKIA_APP_ID_KEY] = heremaps_app_id - user_config[self.NOKIA_APP_CODE_KEY] = heremaps_app_code + user_config[self.NOKIA_APP_ID_KEY] = db_config.heremaps_app_id + user_config[self.NOKIA_APP_CODE_KEY] = db_config.heremaps_app_code if orgname: self.__get_organization_config(orgname, user_config) @@ -182,25 +182,23 @@ class GeocoderConfig(ServiceConfig): PERIOD_END_DATE = 'period_end_date' LOG_PATH = '/var/log/postgresql/geocodings.log' - def __init__(self, redis_connection, username, orgname=None, - heremaps_app_id=None, heremaps_app_code=None): + def __init__(self, redis_connection, db_conn, username, orgname=None): super(GeocoderConfig, self).__init__(redis_connection, username, orgname) - config = self.__get_user_config(username, orgname, heremaps_app_id, - heremaps_app_code) + db_config = ServicesDBConfig(db_conn) + config = self.__get_user_config(username, orgname, db_config) filtered_config = {key: config[key] for key in self.GEOCODER_CONFIG_KEYS if key in config.keys()} self.__check_config(filtered_config) self.__parse_config(filtered_config) - def __get_user_config(self, username, orgname=None, heremaps_app_id=None, - heremaps_app_code=None): + def __get_user_config(self, username, orgname, db_config): user_config = self._redis_connection.hgetall( "rails:users:{0}".format(username)) if not user_config: raise ConfigException("""There is no user config available. Please check your configuration.'""") else: - user_config[self.NOKIA_GEOCODER_APP_ID_KEY] = heremaps_app_id - user_config[self.NOKIA_GEOCODER_APP_CODE_KEY] = heremaps_app_code + user_config[self.NOKIA_GEOCODER_APP_ID_KEY] = db_config.heremaps_app_id + user_config[self.NOKIA_GEOCODER_APP_CODE_KEY] = db_config.heremaps_app_code if orgname: self.__get_organization_config(orgname, user_config) @@ -304,3 +302,51 @@ class GeocoderConfig(ServiceConfig): @property def log_path(self): return self.LOG_PATH + + +class ServicesDBConfig: + + def __init__(self, db_conn): + self._db_conn = db_conn + return self._build() + + def _build(self): + self._get_here_config() + self._get_mapzen_config() + + def _get_here_config(self): + heremaps_conf_json = self._get_conf('heremaps_conf') + if not heremaps_conf_json: + raise ConfigException('Here maps configuration missing') + else: + heremaps_conf = json.loads(heremaps_conf_json) + self._heremaps_app_id = heremaps_conf['app_id'] + self._heremaps_app_code = heremaps_conf['app_code'] + + def _get_mapzen_config(self): + mapzen_conf_json = self._get_conf('mapzen_conf') + if not mapzen_conf_json: + raise ConfigException('Mapzen configuration missing') + else: + mapzen_conf = json.loads(mapzen_conf_json) + self._mapzen_routing_app_key = mapzen_conf['routing_app_key'] + + def _get_conf(self, key): + try: + sql = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(key) + conf = self._db_conn.execute(sql, 1) + return conf[0]['conf'] + except: + raise ConfigException("Malformed config for {0}".format(key)) + + @property + def heremaps_app_id(self): + return self._heremaps_app_id + + @property + def heremaps_app_code(self): + return self._heremaps_app_code + + @property + def mapzen_routing_app_key(self): + return self._mapzen_routing_app_key diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/log.py b/server/lib/python/cartodb_services/cartodb_services/metrics/log.py index 82dacb2..b4b8d2b 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/log.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/log.py @@ -1,4 +1,4 @@ - +from datetime import datetime import abc import json import re diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py b/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py index dbf91b5..d884632 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py @@ -1,3 +1,3 @@ -from redis_tools import RedisConnection +from redis_tools import RedisConnection, RedisDBConfig from coordinates import Coordinate from polyline import PolyLine diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py b/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py index cc04f9b..637e81b 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py @@ -1,32 +1,87 @@ from redis.sentinel import Sentinel from redis import StrictRedis +import json class RedisConnection: - REDIS_DEFAULT_USER_DB = 5 - REDIS_DEFAULT_TIMEOUT = 2 #seconds - - def __init__(self, sentinel_master_id, redis_host, redis_port, - redis_db=REDIS_DEFAULT_USER_DB, **kwargs): - self.redis_host = redis_host - self.redis_port = redis_port - self.sentinel_master_id = sentinel_master_id - self.timeout = kwargs['timeout'] if 'timeout' in kwargs else self.REDIS_DEFAULT_TIMEOUT - self.redis_db = redis_db + def __init__(self, config): + self._config = config def redis_connection(self): return self.__create_redis_connection() def __create_redis_connection(self): - if self.sentinel_master_id == None: - return StrictRedis(host=self.redis_host, port=self.redis_port, db=self.redis_db) - else: - sentinel = Sentinel([(self.redis_host, - self.redis_port)], - socket_timeout=self.timeout) + if self._config.sentinel_id: + sentinel = Sentinel([(self._config.host, + self._config.port)], + socket_timeout=self._config.timeout) return sentinel.master_for( self.sentinel_master_id, socket_timeout=self.timeout, db=self.redis_db ) + else: + conn = StrictRedis(host=self._config.host, port=self._config.port, + db=self._config.db) + return conn + + +class RedisDBConfig: + + DEFAULT_USER_DB = 5 + DEFAULT_TIMEOUT = 2 # seconds + + def __init__(self, key, db_conn): + self._db_conn = db_conn + return self._build(key) + + def _build(self, key): + conf_query = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format( + key) + conf = self._db_conn.execute(conf_query)[0]['conf'] + if conf is None: + raise "There is no redis configuration defined" + else: + params = json.loads(conf) + self._host = params['redis_host'] + self._port = params['redis_port'] + + if "timeout" in params: + self._timeout = params['timeout'] + else: + self._timeout = self.DEFAULT_TIMEOUT + + if "redis_db" in params: + self._db = params['redis_db'] + else: + self._db = self.DEFAULT_USER_DB + + if "sentinel_master_id" in params: + self._sentinel_id = params["sentinel_master_id"] + else: + self._sentinel_id = None + + def __str__(self): + return "Host: {0}, Port: {1}, Sentinel id: {2}, DB: {3}".format( + self.host, self.port, self.sentinel_id, self.db) + + @property + def host(self): + return self._host + + @property + def port(self): + return self._port + + @property + def timeout(self): + return self._timeout + + @property + def db(self): + return self._db + + @property + def sentinel_id(self): + return self._sentinel_id diff --git a/server/lib/python/cartodb_services/test/test_config.py b/server/lib/python/cartodb_services/test/test_config.py index 7588427..b92dc31 100644 --- a/server/lib/python/cartodb_services/test/test_config.py +++ b/server/lib/python/cartodb_services/test/test_config.py @@ -10,12 +10,12 @@ class TestConfig(TestCase): def setUp(self): self.redis_conn = MockRedis() + self.plpy_mock = test_helper.build_plpy_mock() def test_should_return_list_of_nokia_geocoder_config_if_its_ok(self): test_helper.build_redis_user_config(self.redis_conn, 'test_user') - geocoder_config = GeocoderConfig(self.redis_conn, - 'test_user', None, - 'nokia_id', 'nokia_cod') + geocoder_config = GeocoderConfig(self.redis_conn, self.plpy_mock, + 'test_user', None) assert geocoder_config.heremaps_geocoder is True assert geocoder_config.geocoding_quota == 100 assert geocoder_config.soft_geocoding_limit is False @@ -25,17 +25,17 @@ class TestConfig(TestCase): test_helper.build_redis_user_config(self.redis_conn, 'test_user') test_helper.build_redis_org_config(self.redis_conn, 'test_org', quota=200, end_date=yesterday) - geocoder_config = GeocoderConfig(self.redis_conn, - 'test_user', 'test_org', - 'nokia_id', 'nokia_cod') + geocoder_config = GeocoderConfig(self.redis_conn, self.plpy_mock, + 'test_user', 'test_org') assert geocoder_config.heremaps_geocoder is True assert geocoder_config.geocoding_quota == 200 assert geocoder_config.soft_geocoding_limit is False assert geocoder_config.period_end_date.date() == yesterday.date() - def test_should_raise_configuration_exception_when_missing_nokia_geocoder_parameters(self): + def test_should_raise_exception_when_missing_parameters(self): + plpy_mock = test_helper.build_plpy_mock(empty=True) test_helper.build_redis_user_config(self.redis_conn, 'test_user') assert_raises(ConfigException, GeocoderConfig, - self.redis_conn, 'test_user', - None, None, None) + self.redis_conn, plpy_mock, 'test_user', + None) diff --git a/server/lib/python/cartodb_services/test/test_helper.py b/server/lib/python/cartodb_services/test/test_helper.py index 9f1f50f..e732077 100644 --- a/server/lib/python/cartodb_services/test/test_helper.py +++ b/server/lib/python/cartodb_services/test/test_helper.py @@ -1,4 +1,5 @@ from datetime import datetime, date +from mock import Mock def build_redis_user_config(redis_conn, username, quota=100, soft_limit=False, @@ -31,3 +32,18 @@ def increment_geocoder_uses(redis_conn, username, orgname=None, redis_name = "{0}:{1}:{2}:{3}:{4}".format(prefix, entity_name, service, metric, yearmonth) redis_conn.zincrby(redis_name, date.day, amount) + + +def build_plpy_mock(empty=False): + plpy_mock = Mock() + if not empty: + plpy_mock.execute.side_effect = _plpy_execute_side_effect + + return plpy_mock + + +def _plpy_execute_side_effect(*args, **kwargs): + if args[0] == 'SELECT cartodb.CDB_Conf_GetConf(heremaps_conf) as conf': + return [{'conf': '{"app_id": "app_id", "app_code": "code"}'}] + elif args[0] == 'SELECT cartodb.CDB_Conf_GetConf(mapzen_conf) as conf': + return [{'conf': '{"routing_app_key": "app_key"}'}] diff --git a/server/lib/python/cartodb_services/test/test_quota_service.py b/server/lib/python/cartodb_services/test/test_quota_service.py index fbbb4ff..9df7f88 100644 --- a/server/lib/python/cartodb_services/test/test_quota_service.py +++ b/server/lib/python/cartodb_services/test/test_quota_service.py @@ -88,9 +88,9 @@ class TestQuotaService(TestCase): if orgname: test_helper.build_redis_org_config(self.redis_conn, orgname, quota=quota, end_date=end_date) - geocoder_config = GeocoderConfig(self.redis_conn, - username, orgname, - 'nokia_id', 'nokia_cod') + plpy_mock = test_helper.build_plpy_mock() + geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock, + username, orgname) return QuotaService(geocoder_config, redis_connection = self.redis_conn) diff --git a/server/lib/python/cartodb_services/test/test_user_service.py b/server/lib/python/cartodb_services/test/test_user_service.py index 8535ec3..9066643 100644 --- a/server/lib/python/cartodb_services/test/test_user_service.py +++ b/server/lib/python/cartodb_services/test/test_user_service.py @@ -4,6 +4,7 @@ from cartodb_services.metrics import UserMetricsService from cartodb_services.metrics import GeocoderConfig from datetime import datetime, date from unittest import TestCase +from mock import Mock from nose.tools import assert_raises from datetime import timedelta @@ -84,7 +85,7 @@ class TestUserService(TestCase): if orgname: test_helper.build_redis_org_config(self.redis_conn, orgname, quota=quota, end_date=end_date) - geocoder_config = GeocoderConfig(self.redis_conn, - username, orgname, - 'nokia_id', 'nokia_cod') + plpy_mock = test_helper.build_plpy_mock() + geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock, + username, orgname,) return UserMetricsService(geocoder_config, self.redis_conn)