cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

User config now is retrieved from redis

Browse Source
This commit is contained in:
Mario de Frutos
2016-01-20 20:56:19 +01:00
parent a7371c0a48
commit b2fa8739e2
3 changed files with 129 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/extension/sql/0.1.0/10_redis_helper.sql
+4 -2
View File
@@ -1,4 +1,5 @@
-- Get the Redis configuration from the _conf table -- -- Get the Redis configuration from the _conf table --
DROP FUNCTION IF EXISTS cdb_geocoder_server._get_redis_conf();
CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_redis_conf(config_key text) CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_redis_conf(config_key text)
RETURNS cdb_geocoder_server._redis_conf_params AS $$ RETURNS cdb_geocoder_server._redis_conf_params AS $$
conf_query = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(config_key) conf_query = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(config_key)
@@ -20,7 +21,8 @@ $$ LANGUAGE plpythonu;
-- Get the connection to redis from cache or create a new one -- Get the connection to redis from cache or create a new one
CREATE OR REPLACE FUNCTION cdb_geocoder_server._connect_to_redis(user_id text) CREATE OR REPLACE FUNCTION cdb_geocoder_server._connect_to_redis(user_id text)
RETURNS boolean AS $$ RETURNS boolean AS $$
if user_id in GD and 'redis_connection' in GD[user_id]: cache_key = "redis_connection_{0}".format(user_id)
if cache_key in GD:
return False return False
else: else:
from cartodb_geocoder import redis_helper from cartodb_geocoder import redis_helper
@@ -40,7 +42,7 @@ RETURNS boolean AS $$
metrics_config_params['sentinel_master_id'], metrics_config_params['sentinel_master_id'],
timeout=metrics_config_params['timeout'], timeout=metrics_config_params['timeout'],
redis_db=metrics_config_params['redis_db']).redis_connection() redis_db=metrics_config_params['redis_db']).redis_connection()
GD[user_id] = { GD[cache_key] = {
'redis_metadata_connection': redis_metadata_connection, 'redis_metadata_connection': redis_metadata_connection,
'redis_metrics_connection': redis_metrics_connection, 'redis_metrics_connection': redis_metrics_connection,
} }
server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py
+103 -77
View File
@@ -1,104 +1,130 @@
import json import json
from dateutil.parser import parse as date_parse
class ConfigException(Exception): class ConfigException(Exception):
pass pass
class UserConfig: class UserConfig:
USER_CONFIG_KEYS = ['is_organization', 'entity_name'] USER_CONFIG_KEYS = ['is_organization', 'entity_name']
def __init__(self, user_config_json, db_user_id = None): def __init__(self, user_config_json, db_user_id=None):
config = json.loads(user_config_json) config = json.loads(user_config_json)
filtered_config = { key: config[key] for key in self.USER_CONFIG_KEYS if key in config.keys() } filtered_config = {key: config[key] for key in self.USER_CONFIG_KEYS if key in config.keys()}
self.__check_config(filtered_config) self.__check_config(filtered_config)
self.__parse_config(filtered_config) self.__parse_config(filtered_config)
self._user_id = self.__extract_uuid(db_user_id)
def __check_config(self, filtered_config): def __check_config(self, filtered_config):
if len(filtered_config.keys()) != len(self.USER_CONFIG_KEYS): if len(filtered_config.keys()) != len(self.USER_CONFIG_KEYS):
raise ConfigException("Passed user configuration is not correct, check it please") raise ConfigException(
"Passed user configuration is not correct, check it please")
return True return True
@property @property
def is_organization(self): def is_organization(self):
return self._is_organization return self._is_organization
@property @property
def entity_name(self): def entity_name(self):
return self._entity_name return self._entity_name
@property def __parse_config(self, filtered_config):
def user_id(self): self._is_organization = filtered_config['is_organization']
return self._user_id self._entity_name = filtered_config['entity_name']
def __parse_config(self, filtered_config):
self._is_organization = filtered_config['is_organization']
self._entity_name = filtered_config['entity_name']
def __extract_uuid(self, db_user_id):
# Format: development_cartodb_user_<UUID>
return db_user_id.split('_')[-1]
class GeocoderConfig: class GeocoderConfig:
GEOCODER_CONFIG_KEYS = ['street_geocoder_provider', 'google_maps_private_key', GEOCODER_CONFIG_KEYS = ['google_maps_client_id', 'google_maps_api_key',
'nokia_monthly_quota', 'nokia_soft_geocoder_limit'] 'geocoding_quota', 'soft_geocoding_limit',
NOKIA_GEOCODER_MANDATORY_KEYS = ['nokia_monthly_quota', 'nokia_soft_geocoder_limit'] 'geocoder_type', 'period_end_date']
NOKIA_GEOCODER = 'nokia' NOKIA_GEOCODER_MANDATORY_KEYS = ['geocoding_quota', 'soft_geocoding_limit']
NOKIA_QUOTA_KEY = 'nokia_monthly_quota' NOKIA_GEOCODER = 'heremaps'
NOKIA_SOFT_LIMIT_KEY = 'nokia_soft_geocoder_limit' GOOGLE_GEOCODER = 'google'
GOOGLE_GEOCODER = 'google' GOOGLE_GEOCODER_API_KEY = 'google_maps_api_key'
GEOCODER_TYPE = 'street_geocoder_provider' GOOGLE_GEOCODER_CLIENT_ID = 'google_maps_client_id'
GOOGLE_GEOCODER_API_KEY = 'google_maps_private_key' GEOCODER_TYPE = 'geocoder_type'
QUOTA_KEY = 'geocoding_quota'
SOFT_LIMIT_KEY = 'soft_geocoding_limit'
PERIOD_END_DATE = 'period_end_date'
def __init__(self, geocoder_config_json): def __init__(self, redis_connection, username, orgname=None):
config = json.loads(geocoder_config_json) self._redis_connection = redis_connection
filtered_config = { key: config[key] for key in self.GEOCODER_CONFIG_KEYS if key in config.keys() } config = self.__get_user_config(username, orgname)
self.__check_config(filtered_config) filtered_config = {key: config[key] for key in self.GEOCODER_CONFIG_KEYS if key in config.keys()}
self.__parse_config(filtered_config) self.__check_config(filtered_config)
self.__parse_config(filtered_config)
def __check_config(self, filtered_config): def __get_user_config(self, username, orgname=None):
if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER: user_config = self._redis_connection.hgetall(
if not set(self.NOKIA_GEOCODER_MANDATORY_KEYS).issubset(set(filtered_config.keys())): "rails:users:{0}".format(username))
raise ConfigException("""Nokia geocoder need the mandatory parameters 'nokia_monthly_quota' and 'nokia_soft_geocoder_limit'""") if orgname:
elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER: org_config = self._redis_connection.hgetall(
if self.GOOGLE_GEOCODER_API_KEY not in filtered_config.keys(): "rails:orgs:{0}".format(orgname))
raise ConfigException("Google geocoder need the mandatory parameter 'google_maps_private_key'") user_config[self.QUOTA_KEY] = org_config[self.QUOTA_KEY]
user_config[self.PERIOD_END_DATE] = org_config[self.PERIOD_END_DATE]
user_config[self.GOOGLE_GEOCODER_CLIENT_ID] = org_config[self.GOOGLE_GEOCODER_CLIENT_ID]
user_config[self.GOOGLE_GEOCODER_API_KEY] = org_config[self.GOOGLE_GEOCODER_API_KEY]
return True return user_config
def __parse_config(self, filtered_config): def __check_config(self, filtered_config):
self._geocoder_type = filtered_config[self.GEOCODER_TYPE].lower() if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER:
self._google_maps_private_key = None if not set(self.NOKIA_GEOCODER_MANDATORY_KEYS).issubset(set(filtered_config.keys())):
self._nokia_monthly_quota = 0 raise ConfigException("""Nokia geocoder needs the mandatory parameters '' and 'nokia_soft_geocoder_limit'""")
self._nokia_soft_geocoder_limit = False elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER:
if self.GOOGLE_GEOCODER == self._geocoder_type: if self.GOOGLE_GEOCODER_API_KEY not in filtered_config.keys():
self._google_maps_private_key = filtered_config[self.GOOGLE_GEOCODER_API_KEY] raise ConfigException("Google geocoder need the mandatory parameter 'google_maps_private_key'")
elif self.NOKIA_GEOCODER == self._geocoder_type:
self._nokia_monthly_quota = filtered_config[self.NOKIA_QUOTA_KEY]
self._nokia_soft_geocoder_limit = filtered_config[self.NOKIA_SOFT_LIMIT_KEY]
@property return True
def service_type(self):
return self._geocoder_type
@property def __parse_config(self, filtered_config):
def nokia_geocoder(self): self._geocoder_type = filtered_config[self.GEOCODER_TYPE].lower()
return self._geocoder_type == self.NOKIA_GEOCODER self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE])
self._google_maps_private_key = None
self._nokia_monthly_quota = 0
self._nokia_soft_geocoder_limit = False
if self.GOOGLE_GEOCODER == self._geocoder_type:
self._google_maps_private_key = filtered_config[self.GOOGLE_GEOCODER_API_KEY]
self._google_maps_client_id = filtered_config[self.GOOGLE_GEOCODER_CLIENT_ID]
elif self.NOKIA_GEOCODER == self._geocoder_type:
self._geocoding_quota = filtered_config[self.QUOTA_KEY]
self._soft_geocoding_limit = filtered_config[self.SOFT_LIMIT_KEY]
@property @property
def google_geocoder(self): def service_type(self):
return self._geocoder_type == self.GOOGLE_GEOCODER if self._geocoder_type == self.GOOGLE_GEOCODER:
return 'geocoder_google'
else:
return 'geocoder_here'
@property @property
def google_api_key(self): def heremaps_geocoder(self):
return self._google_maps_private_key return self._geocoder_type == self.NOKIA_GEOCODER
@property @property
def nokia_monthly_quota(self): def google_geocoder(self):
return self._nokia_monthly_quota return self._geocoder_type == self.GOOGLE_GEOCODER
@property @property
def nokia_soft_limit(self): def google_client_id(self):
return self._nokia_soft_geocoder_limit return self._google_maps_client_id
@property
def google_api_key(self):
return self._google_maps_private_key
@property
def geocoding_quota(self):
return self._geocoding_quota
@property
def soft_geocoding_limit(self):
return self._soft_geocoding_limit
@property
def period_end_date(self):
return self._period_end_date
server/lib/python/cartodb_geocoder/cartodb_geocoder/redis_helper.py
+22 -13
View File
@@ -1,20 +1,29 @@
from redis.sentinel import Sentinel from redis.sentinel import Sentinel
class RedisHelper: class RedisHelper:
REDIS_DEFAULT_USER_DB = 5 REDIS_DEFAULT_USER_DB = 5
REDIS_TIMEOUT = 2 #seconds REDIS_DEFAULT_TIMEOUT = 2 #seconds
REDIS_SENTINEL_DEFAULT_PORT = 26379
def __init__(self, sentinel_host, sentinel_port, sentinel_master_id, redis_db=REDIS_DEFAULT_USER_DB, **kwargs): def __init__(self, sentinel_host, sentinel_port, sentinel_master_id,
self.sentinel_host = sentinel_host redis_db=REDIS_DEFAULT_USER_DB, **kwargs):
self.sentinel_port = sentinel_port self.sentinel_host = sentinel_host
self.sentinel_master_id = sentinel_master_id self.sentinel_port = sentinel_port
self.timeout = kwargs['timeout'] if 'timeout' in kwargs else REDIS_DEFAULT_TIMEOUT self.sentinel_master_id = sentinel_master_id
self.redis_db = redis_db self.timeout = kwargs['timeout'] if 'timeout' in kwargs else self.REDIS_DEFAULT_TIMEOUT
self.redis_db = redis_db
def redis_connection(self): def redis_connection(self):
return self.__create_redis_connection() return self.__create_redis_connection()
def __create_redis_connection(self): def __create_redis_connection(self):
sentinel = Sentinel([(self.sentinel_host, 26379)], socket_timeout=self.timeout) sentinel = Sentinel([(self.sentinel_host,
return sentinel.master_for(self.sentinel_master_id, socket_timeout=self.timeout, db=self.redis_db) self.REDIS_SENTINEL_DEFAULT_PORT)],
socket_timeout=self.timeout)
return sentinel.master_for(
self.sentinel_master_id,
socket_timeout=self.timeout,
db=self.redis_db
)