From 1f3a655ae590d90c9f7e1bafbd33b4c35051f8bf Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 29 Sep 2016 16:57:31 +0200 Subject: [PATCH] Add all the code to instantiate a user_config_storage (WIP) --- server/extension/sql/20_geocode_street.sql | 13 ++- .../refactor/storage/redis_config.py | 85 +++++++++++++++++++ .../refactor/storage/redis_connection.py | 22 +++++ 3 files changed, 116 insertions(+), 4 deletions(-) create mode 100644 server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py create mode 100644 server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_connection.py diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index 2c4df2f..acaff8e 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -143,17 +143,22 @@ RETURNS Geometry AS $$ from cartodb_services.mapzen.types import country_to_iso3 from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger - from cartodb_services.refactor.storage.server_config import InDbServerConfigStorage + from cartodb_services.refactor.storage.server_config import InDbServerConfigStorage, RedisConfigStorage from cartodb_services.refactor.tools.logger import LoggerConfigBuilder - - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + from cartodb_services.refactor.storage.redis_config import RedisMetadataConnectionConfigBuilder + from cartodb_services.refactor.storage.redis_connection import RedisConnectionBuilder server_config_storage = InDbServerConfigStorage() logger_config = LoggerConfigBuilder(server_config_storage).get() logger = Logger(logger_config) + redis_metadata_connection_config = RedisMetadataConnectionConfigBuilder(server_config_storage).get() + redis_metadata_connection = RedisConnectionBuilder(redis_metadata_connection_config) + user_config_storage = RedisConfigStorage(redis_metadata_connection, 'rails:users:{0}'.format(username)) + + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] quota_service = QuotaService(user_geocoder_config, redis_conn) diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py new file mode 100644 index 0000000..8c156eb --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py @@ -0,0 +1,85 @@ +""" +How to use this (just a draft, WIP): + + redis_connection_config = RedisConnectionConfigBuilder(server_config_storage).get() + connection = RedisConnectionBuilder(redis_connection_config) + + user_config_storage = RedisConfigStorage(connection, user) + user_config_storage = UserConfigStorageFactory(environment).get() +""" + +class RedisConnectionConfig(object): + """ + This represents a value object to contain configuration needed to set up + a connection to a redis server. + """ + + + def __init__(self, host, port, timeout, db, sentinel_id): + self._host = host + self._port = port + self._timeout = timeout + self._db = db + self._sentinel_id = sentinel_id + + @property + def host(self): + return self._host + + @property + def port(self): + return self._port + + @property + def timeout(self): + return self._timeout + + @property + def db(self): + return self._db + + @property + def sentinel_id(self): + return self._sentinel_id + + +# TODO make it abstract +class RedisConnectionConfigBuilder(object): + + DEFAULT_USER_DB = 5 + DEFAULT_TIMEOUT = 1.5 # seconds + + def __init__(self, server_config_storage, config_key): + self._server_config_storage = server_config_storage + self._config_key = config_key + + def get(self): + conf = self._server_config_storage.get(self._config_key) + if conf is None: + raise Exception("There is no redis configuration defined") + + host = conf['redis_host'] + port = conf['redis_port'] + timeout = conf['timeout'] or self.DEFAULT_TIMEOUT + db = conf['redis_db'] or self.DEFAULT_USER_DB + sentinel_id = conf['sentinel_master_id'] + + return RedisConnectionConfig(host, port, timeout, db, sentinel_id) + + +class RedisMetadataConnectionConfigBuilder(RedisConnectionConfigBuilder): + + def __init__(self, server_config_storage): + super(RedisMetadataConnectionConfigBuilder, self).__init__( + server_config_storage, + 'redis_metadata_config' + ) + + +class RedisMetricsConnectionConfigBuilder(RedisConnectionConfigBuilder): + + def __init__(self, server_config_storage): + super(RedisMetricsConnectionConfigBuilder, self).__init__( + server_config_storage, + 'redis_metrics_config' + ) diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_connection.py b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_connection.py new file mode 100644 index 0000000..a125af3 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_connection.py @@ -0,0 +1,22 @@ +from redis.sentinel import Sentinel +from redis import StrictRedis + +class RedisConnectionBuilder(): + + def __init__(self, connection_config): + self._config = connection_config + + def get(self): + if self._config.sentinel_id: + sentinel = Sentinel([(self._config.host, + self._config.port)], + socket_timeout=self._config.timeout) + return sentinel.master_for(self._config.sentinel_id, + socket_timeout=self._config.timeout, + db=self._config.db, + retry_on_timeout=True) + else: + conn = StrictRedis(host=self._config.host, port=self._config.port, + db=self._config.db, retry_on_timeout=True, + socket_timeout=self._config.timeout) + return conn