Carla Iriberri
6d35cff9c7
Add config function in postgres explicitly to get MapzenGeocoderConfig. Bump versions for client and server APIs. New MapzenGeocoderConfig included to be able to use current QuotaServices with non-configured users.
561 lines
20 KiB
Python
561 lines
20 KiB
Python
import json
|
|
import abc
|
|
from dateutil.parser import parse as date_parse
|
|
|
|
|
|
class ConfigException(Exception):
|
|
pass
|
|
|
|
|
|
class ServiceConfig(object):
|
|
__metaclass__ = abc.ABCMeta
|
|
|
|
def __init__(self, redis_connection, db_conn, username, orgname=None):
|
|
self._redis_connection = redis_connection
|
|
self._username = username
|
|
self._orgname = orgname
|
|
self._db_config = ServicesDBConfig(db_conn, username, orgname)
|
|
if redis_connection:
|
|
self._redis_config = ServicesRedisConfig(redis_connection).build(
|
|
username, orgname)
|
|
else:
|
|
self._redis_config = None
|
|
|
|
@abc.abstractproperty
|
|
def service_type(self):
|
|
raise NotImplementedError('service_type property must be defined')
|
|
|
|
@property
|
|
def username(self):
|
|
return self._username
|
|
|
|
@property
|
|
def organization(self):
|
|
return self._orgname
|
|
|
|
class DataObservatoryConfig(ServiceConfig):
|
|
|
|
def __init__(self, redis_connection, db_conn, username, orgname=None):
|
|
super(DataObservatoryConfig, self).__init__(redis_connection, db_conn,
|
|
username, orgname)
|
|
|
|
@property
|
|
def monthly_quota(self):
|
|
return self._monthly_quota
|
|
|
|
@property
|
|
def period_end_date(self):
|
|
return self._period_end_date
|
|
|
|
@property
|
|
def soft_limit(self):
|
|
return self._soft_limit
|
|
|
|
@property
|
|
def connection_str(self):
|
|
return self._connection_str
|
|
|
|
class ObservatorySnapshotConfig(DataObservatoryConfig):
|
|
|
|
SOFT_LIMIT_KEY = 'soft_obs_snapshot_limit'
|
|
QUOTA_KEY = 'obs_snapshot_quota'
|
|
PERIOD_END_DATE = 'period_end_date'
|
|
|
|
def __init__(self, redis_connection, db_conn, username, orgname=None):
|
|
super(ObservatorySnapshotConfig, self).__init__(redis_connection, db_conn,
|
|
username, orgname)
|
|
self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE])
|
|
if self.SOFT_LIMIT_KEY in self._redis_config and self._redis_config[self.SOFT_LIMIT_KEY].lower() == 'true':
|
|
self._soft_limit = True
|
|
else:
|
|
self._soft_limit = False
|
|
self._monthly_quota = 0
|
|
if self.QUOTA_KEY in self._redis_config:
|
|
self._monthly_quota = float(self._redis_config[self.QUOTA_KEY])
|
|
self._connection_str = self._db_config.data_observatory_connection_str
|
|
|
|
@property
|
|
def service_type(self):
|
|
return 'obs_snapshot'
|
|
|
|
class ObservatoryConfig(DataObservatoryConfig):
|
|
|
|
SOFT_LIMIT_KEY = 'soft_obs_general_limit'
|
|
QUOTA_KEY = 'obs_general_quota'
|
|
PERIOD_END_DATE = 'period_end_date'
|
|
|
|
def __init__(self, redis_connection, db_conn, username, orgname=None):
|
|
super(ObservatoryConfig, self).__init__(redis_connection, db_conn,
|
|
username, orgname)
|
|
self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE])
|
|
if self.SOFT_LIMIT_KEY in self._redis_config and self._redis_config[self.SOFT_LIMIT_KEY].lower() == 'true':
|
|
self._soft_limit = True
|
|
else:
|
|
self._soft_limit = False
|
|
self._monthly_quota = 0
|
|
if self.QUOTA_KEY in self._redis_config:
|
|
self._monthly_quota = float(self._redis_config[self.QUOTA_KEY])
|
|
self._connection_str = self._db_config.data_observatory_connection_str
|
|
|
|
@property
|
|
def service_type(self):
|
|
return 'obs_general'
|
|
|
|
class RoutingConfig(ServiceConfig):
|
|
|
|
PERIOD_END_DATE = 'period_end_date'
|
|
|
|
def __init__(self, redis_connection, db_conn, username, orgname=None):
|
|
super(RoutingConfig, self).__init__(redis_connection, db_conn,
|
|
username, orgname)
|
|
self._mapzen_api_key = self._db_config.mapzen_routing_api_key
|
|
self._monthly_quota = self._db_config.mapzen_routing_monthly_quota
|
|
self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE])
|
|
|
|
@property
|
|
def service_type(self):
|
|
return 'routing_mapzen'
|
|
|
|
@property
|
|
def mapzen_api_key(self):
|
|
return self._mapzen_api_key
|
|
|
|
@property
|
|
def monthly_quota(self):
|
|
return self._monthly_quota
|
|
|
|
@property
|
|
def period_end_date(self):
|
|
return self._period_end_date
|
|
|
|
# Explicit class for the geocoder configuration for Mapzen
|
|
# which does not require users to be configured to use the service
|
|
class MapzenGeocoderConfig(ServiceConfig):
|
|
|
|
PERIOD_END_DATE = 'period_end_date'
|
|
|
|
def __init__(self, redis_connection, db_conn, username, orgname=None):
|
|
super(MapzenGeocoderConfig, self).__init__(redis_connection, db_conn,
|
|
username, orgname)
|
|
self._log_path = self._db_config.geocoder_log_path
|
|
try:
|
|
self._mapzen_api_key = self._db_config.mapzen_geocoder_api_key
|
|
self._monthly_quota = self._db_config.mapzen_geocoder_monthly_quota
|
|
self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE])
|
|
self._cost_per_hit = 0
|
|
except Exception as e:
|
|
raise ConfigException("Malformed config for Mapzen geocoder: {1}".format(key, e))
|
|
|
|
@property
|
|
def service_type(self):
|
|
return 'geocoder_mapzen'
|
|
|
|
@property
|
|
def mapzen_api_key(self):
|
|
return self._mapzen_api_key
|
|
|
|
@property
|
|
def period_end_date(self):
|
|
return self._period_end_date
|
|
|
|
@property
|
|
def cost_per_hit(self):
|
|
return self._cost_per_hit
|
|
|
|
@property
|
|
def google_geocoder(self):
|
|
return None
|
|
|
|
@property
|
|
def geocoding_quota(self):
|
|
return self._monthly_quota
|
|
|
|
@property
|
|
def soft_geocoding_limit(self):
|
|
return False
|
|
|
|
@property
|
|
def is_high_resolution(self):
|
|
return True
|
|
|
|
@property
|
|
def log_path(self):
|
|
return self._log_path
|
|
|
|
|
|
class IsolinesRoutingConfig(ServiceConfig):
|
|
|
|
ROUTING_CONFIG_KEYS = ['here_isolines_quota', 'soft_here_isolines_limit',
|
|
'period_end_date', 'username', 'orgname',
|
|
'heremaps_isolines_app_id', 'heremaps_isolines_app_code',
|
|
'geocoder_type']
|
|
QUOTA_KEY = 'here_isolines_quota'
|
|
SOFT_LIMIT_KEY = 'soft_here_isolines_limit'
|
|
PERIOD_END_DATE = 'period_end_date'
|
|
GEOCODER_TYPE_KEY = 'geocoder_type'
|
|
GOOGLE_GEOCODER = 'google'
|
|
|
|
def __init__(self, redis_connection, db_conn, username, orgname=None):
|
|
super(IsolinesRoutingConfig, self).__init__(redis_connection, db_conn,
|
|
username, orgname)
|
|
filtered_config = {key: self._redis_config[key] for key in self.ROUTING_CONFIG_KEYS if key in self._redis_config.keys()}
|
|
self.__parse_config(filtered_config, self._db_config)
|
|
|
|
def __parse_config(self, filtered_config, db_config):
|
|
self._geocoder_type = filtered_config[self.GEOCODER_TYPE_KEY].lower()
|
|
self._isolines_quota = float(filtered_config[self.QUOTA_KEY])
|
|
self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE])
|
|
if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true':
|
|
self._soft_isolines_limit = True
|
|
else:
|
|
self._soft_isolines_limit = False
|
|
self._heremaps_app_id = db_config.heremaps_isolines_app_id
|
|
self._heremaps_app_code = db_config.heremaps_isolines_app_code
|
|
|
|
@property
|
|
def service_type(self):
|
|
return 'here_isolines'
|
|
|
|
@property
|
|
def isolines_quota(self):
|
|
return self._isolines_quota
|
|
|
|
@property
|
|
def soft_isolines_limit(self):
|
|
return self._soft_isolines_limit
|
|
|
|
@property
|
|
def period_end_date(self):
|
|
return self._period_end_date
|
|
|
|
@property
|
|
def heremaps_app_id(self):
|
|
return self._heremaps_app_id
|
|
|
|
@property
|
|
def heremaps_app_code(self):
|
|
return self._heremaps_app_code
|
|
|
|
@property
|
|
def google_services_user(self):
|
|
return self._geocoder_type == self.GOOGLE_GEOCODER
|
|
|
|
|
|
class InternalGeocoderConfig(ServiceConfig):
|
|
|
|
def __init__(self, redis_connection, db_conn, username, orgname=None):
|
|
# For now, internal geocoder doesn't use the redis config
|
|
super(InternalGeocoderConfig, self).__init__(None, db_conn,
|
|
username, orgname)
|
|
self._log_path = self._db_config.geocoder_log_path
|
|
|
|
@property
|
|
def service_type(self):
|
|
return 'geocoder_internal'
|
|
|
|
@property
|
|
def is_high_resolution(self):
|
|
return False
|
|
|
|
@property
|
|
def cost_per_hit(self):
|
|
return 0
|
|
|
|
@property
|
|
def geocoding_quota(self):
|
|
return None
|
|
|
|
@property
|
|
def log_path(self):
|
|
return self._log_path
|
|
|
|
|
|
class GeocoderConfig(ServiceConfig):
|
|
|
|
GEOCODER_CONFIG_KEYS = ['google_maps_client_id', 'google_maps_api_key',
|
|
'geocoding_quota', 'soft_geocoding_limit',
|
|
'geocoder_type', 'period_end_date',
|
|
'heremaps_geocoder_app_id', 'heremaps_geocoder_app_code',
|
|
'mapzen_geocoder_api_key', 'username', 'orgname']
|
|
NOKIA_GEOCODER_REDIS_MANDATORY_KEYS = ['geocoding_quota', 'soft_geocoding_limit']
|
|
NOKIA_GEOCODER = 'heremaps'
|
|
NOKIA_GEOCODER_APP_ID_KEY = 'heremaps_geocoder_app_id'
|
|
NOKIA_GEOCODER_APP_CODE_KEY = 'heremaps_geocoder_app_code'
|
|
GOOGLE_GEOCODER = 'google'
|
|
GOOGLE_GEOCODER_API_KEY = 'google_maps_api_key'
|
|
GOOGLE_GEOCODER_CLIENT_ID = 'google_maps_client_id'
|
|
MAPZEN_GEOCODER = 'mapzen'
|
|
MAPZEN_GEOCODER_API_KEY = 'mapzen_geocoder_api_key'
|
|
GEOCODER_TYPE = 'geocoder_type'
|
|
QUOTA_KEY = 'geocoding_quota'
|
|
SOFT_LIMIT_KEY = 'soft_geocoding_limit'
|
|
USERNAME_KEY = 'username'
|
|
ORGNAME_KEY = 'orgname'
|
|
PERIOD_END_DATE = 'period_end_date'
|
|
|
|
def __init__(self, redis_connection, db_conn, username, orgname=None):
|
|
super(GeocoderConfig, self).__init__(redis_connection, db_conn,
|
|
username, orgname)
|
|
filtered_config = {key: self._redis_config[key] for key in self.GEOCODER_CONFIG_KEYS if key in self._redis_config.keys()}
|
|
self.__parse_config(filtered_config, self._db_config)
|
|
self.__check_config(filtered_config)
|
|
|
|
def __check_config(self, filtered_config):
|
|
if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER:
|
|
if not set(self.NOKIA_GEOCODER_REDIS_MANDATORY_KEYS).issubset(set(filtered_config.keys())) or \
|
|
not self.heremaps_app_id or not self.heremaps_app_code:
|
|
raise ConfigException("""Some mandatory parameter/s for Nokia geocoder are missing. Check it please""")
|
|
elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER:
|
|
if self.GOOGLE_GEOCODER_API_KEY not in filtered_config.keys():
|
|
raise ConfigException("""Google geocoder need the mandatory parameter 'google_maps_private_key'""")
|
|
elif filtered_config[self.GEOCODER_TYPE].lower() == self.MAPZEN_GEOCODER:
|
|
if not self.mapzen_api_key:
|
|
raise ConfigException("""Mapzen config is not setted up""")
|
|
|
|
return True
|
|
|
|
def __parse_config(self, filtered_config, db_config):
|
|
self._geocoder_type = filtered_config[self.GEOCODER_TYPE].lower()
|
|
self._geocoding_quota = float(filtered_config[self.QUOTA_KEY])
|
|
self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE])
|
|
self._log_path = db_config.geocoder_log_path
|
|
if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true':
|
|
self._soft_geocoding_limit = True
|
|
else:
|
|
self._soft_geocoding_limit = False
|
|
if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER:
|
|
self._heremaps_app_id = db_config.heremaps_geocoder_app_id
|
|
self._heremaps_app_code = db_config.heremaps_geocoder_app_code
|
|
self._cost_per_hit = db_config.heremaps_geocoder_cost_per_hit
|
|
elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER:
|
|
self._google_maps_api_key = filtered_config[self.GOOGLE_GEOCODER_API_KEY]
|
|
self._google_maps_client_id = filtered_config[self.GOOGLE_GEOCODER_CLIENT_ID]
|
|
self._cost_per_hit = 0
|
|
elif filtered_config[self.GEOCODER_TYPE].lower() == self.MAPZEN_GEOCODER:
|
|
self._mapzen_api_key = db_config.mapzen_geocoder_api_key
|
|
self._geocoding_quota = db_config.mapzen_geocoder_monthly_quota
|
|
self._cost_per_hit = 0
|
|
|
|
@property
|
|
def service_type(self):
|
|
if self._geocoder_type == self.GOOGLE_GEOCODER:
|
|
return 'geocoder_google'
|
|
elif self._geocoder_type == self.MAPZEN_GEOCODER:
|
|
return 'geocoder_mapzen'
|
|
else:
|
|
return 'geocoder_here'
|
|
|
|
@property
|
|
def heremaps_geocoder(self):
|
|
return self._geocoder_type == self.NOKIA_GEOCODER
|
|
|
|
@property
|
|
def google_geocoder(self):
|
|
return self._geocoder_type == self.GOOGLE_GEOCODER
|
|
|
|
@property
|
|
def mapzen_geocoder(self):
|
|
return self._geocoder_type == self.MAPZEN_GEOCODER
|
|
|
|
@property
|
|
def google_client_id(self):
|
|
return self._google_maps_client_id
|
|
|
|
@property
|
|
def google_api_key(self):
|
|
return self._google_maps_api_key
|
|
|
|
@property
|
|
def geocoding_quota(self):
|
|
if self.google_geocoder:
|
|
return None
|
|
else:
|
|
return self._geocoding_quota
|
|
|
|
@property
|
|
def soft_geocoding_limit(self):
|
|
return self._soft_geocoding_limit
|
|
|
|
@property
|
|
def period_end_date(self):
|
|
return self._period_end_date
|
|
|
|
@property
|
|
def heremaps_app_id(self):
|
|
return self._heremaps_app_id
|
|
|
|
@property
|
|
def heremaps_app_code(self):
|
|
return self._heremaps_app_code
|
|
|
|
@property
|
|
def mapzen_api_key(self):
|
|
return self._mapzen_api_key
|
|
|
|
@property
|
|
def is_high_resolution(self):
|
|
return True
|
|
|
|
@property
|
|
def cost_per_hit(self):
|
|
return self._cost_per_hit
|
|
|
|
@property
|
|
def log_path(self):
|
|
return self._log_path
|
|
|
|
|
|
class ServicesDBConfig:
|
|
|
|
def __init__(self, db_conn, username, orgname):
|
|
self._db_conn = db_conn
|
|
self._username = username
|
|
self._orgname = orgname
|
|
return self._build()
|
|
|
|
def _build(self):
|
|
self._get_here_config()
|
|
self._get_mapzen_config()
|
|
self._get_logger_config()
|
|
self._get_data_observatory_config()
|
|
|
|
def _get_here_config(self):
|
|
heremaps_conf_json = self._get_conf('heremaps_conf')
|
|
if not heremaps_conf_json:
|
|
raise ConfigException('Here maps configuration missing')
|
|
else:
|
|
heremaps_conf = json.loads(heremaps_conf_json)
|
|
self._heremaps_geocoder_app_id = heremaps_conf['geocoder']['app_id']
|
|
self._heremaps_geocoder_app_code = heremaps_conf['geocoder']['app_code']
|
|
self._heremaps_geocoder_cost_per_hit = heremaps_conf['geocoder'][
|
|
'geocoder_cost_per_hit']
|
|
self._heremaps_isolines_app_id = heremaps_conf['isolines']['app_id']
|
|
self._heremaps_isolines_app_code = heremaps_conf['isolines']['app_code']
|
|
|
|
def _get_mapzen_config(self):
|
|
mapzen_conf_json = self._get_conf('mapzen_conf')
|
|
if not mapzen_conf_json:
|
|
raise ConfigException('Mapzen configuration missing')
|
|
else:
|
|
mapzen_conf = json.loads(mapzen_conf_json)
|
|
self._mapzen_routing_api_key = mapzen_conf['routing']['api_key']
|
|
self._mapzen_routing_quota = mapzen_conf['routing']['monthly_quota']
|
|
self._mapzen_geocoder_api_key = mapzen_conf['geocoder']['api_key']
|
|
self._mapzen_geocoder_quota = mapzen_conf['geocoder']['monthly_quota']
|
|
|
|
def _get_data_observatory_config(self):
|
|
do_conf_json = self._get_conf('data_observatory_conf')
|
|
if not do_conf_json:
|
|
raise ConfigException('Data Observatory configuration missing')
|
|
else:
|
|
do_conf = json.loads(do_conf_json)
|
|
if self._orgname and self._orgname in do_conf['connection']['whitelist']:
|
|
self._data_observatory_connection_str = do_conf['connection']['staging']
|
|
elif self._username in do_conf['connection']['whitelist']:
|
|
self._data_observatory_connection_str = do_conf['connection']['staging']
|
|
else:
|
|
self._data_observatory_connection_str = do_conf['connection']['production']
|
|
|
|
def _get_logger_config(self):
|
|
logger_conf_json = self._get_conf('logger_conf')
|
|
if not logger_conf_json:
|
|
raise ConfigException('Logger configuration missing')
|
|
else:
|
|
logger_conf = json.loads(logger_conf_json)
|
|
self._geocoder_log_path = logger_conf['geocoder_log_path']
|
|
|
|
def _get_conf(self, key):
|
|
try:
|
|
sql = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(key)
|
|
conf = self._db_conn.execute(sql, 1)
|
|
return conf[0]['conf']
|
|
except Exception as e:
|
|
raise ConfigException("Malformed config for {0}: {1}".format(key, e))
|
|
|
|
@property
|
|
def heremaps_isolines_app_id(self):
|
|
return self._heremaps_isolines_app_id
|
|
|
|
@property
|
|
def heremaps_isolines_app_code(self):
|
|
return self._heremaps_isolines_app_code
|
|
|
|
@property
|
|
def heremaps_geocoder_app_id(self):
|
|
return self._heremaps_geocoder_app_id
|
|
|
|
@property
|
|
def heremaps_geocoder_app_code(self):
|
|
return self._heremaps_geocoder_app_code
|
|
|
|
@property
|
|
def heremaps_geocoder_cost_per_hit(self):
|
|
return self._heremaps_geocoder_cost_per_hit
|
|
|
|
@property
|
|
def mapzen_routing_api_key(self):
|
|
return self._mapzen_routing_api_key
|
|
|
|
@property
|
|
def mapzen_routing_monthly_quota(self):
|
|
return self._mapzen_routing_quota
|
|
|
|
@property
|
|
def mapzen_geocoder_api_key(self):
|
|
return self._mapzen_geocoder_api_key
|
|
|
|
@property
|
|
def mapzen_geocoder_monthly_quota(self):
|
|
return self._mapzen_geocoder_quota
|
|
|
|
@property
|
|
def geocoder_log_path(self):
|
|
return self._geocoder_log_path
|
|
|
|
@property
|
|
def data_observatory_connection_str(self):
|
|
return self._data_observatory_connection_str
|
|
|
|
|
|
class ServicesRedisConfig:
|
|
|
|
GOOGLE_GEOCODER_API_KEY = 'google_maps_api_key'
|
|
GOOGLE_GEOCODER_CLIENT_ID = 'google_maps_client_id'
|
|
QUOTA_KEY = 'geocoding_quota'
|
|
ISOLINES_QUOTA_KEY = 'here_isolines_quota'
|
|
OBS_SNAPSHOT_QUOTA_KEY = 'obs_snapshot_quota'
|
|
OBS_GENERAL_QUOTA_KEY = 'obs_general_quota'
|
|
PERIOD_END_DATE = 'period_end_date'
|
|
|
|
def __init__(self, redis_conn):
|
|
self._redis_connection = redis_conn
|
|
|
|
def build(self, username, orgname):
|
|
return self.__get_user_config(username, orgname)
|
|
|
|
def __get_user_config(self, username, orgname):
|
|
user_config = self._redis_connection.hgetall(
|
|
"rails:users:{0}".format(username))
|
|
if not user_config:
|
|
raise ConfigException("""There is no user config available. Please check your configuration.'""")
|
|
elif orgname:
|
|
self.__get_organization_config(orgname, user_config)
|
|
|
|
return user_config
|
|
|
|
def __get_organization_config(self, orgname, user_config):
|
|
org_config = self._redis_connection.hgetall(
|
|
"rails:orgs:{0}".format(orgname))
|
|
if not org_config:
|
|
raise ConfigException("""There is no organization config available. Please check your configuration.'""")
|
|
else:
|
|
user_config[self.QUOTA_KEY] = org_config[self.QUOTA_KEY]
|
|
user_config[self.ISOLINES_QUOTA_KEY] = org_config[self.ISOLINES_QUOTA_KEY]
|
|
if self.OBS_SNAPSHOT_QUOTA_KEY in org_config:
|
|
user_config[self.OBS_SNAPSHOT_QUOTA_KEY] = org_config[self.OBS_SNAPSHOT_QUOTA_KEY]
|
|
if self.OBS_GENERAL_QUOTA_KEY in org_config:
|
|
user_config[self.OBS_GENERAL_QUOTA_KEY] = org_config[self.OBS_GENERAL_QUOTA_KEY]
|
|
user_config[self.PERIOD_END_DATE] = org_config[self.PERIOD_END_DATE]
|
|
user_config[self.GOOGLE_GEOCODER_CLIENT_ID] = org_config[self.GOOGLE_GEOCODER_CLIENT_ID]
|
|
user_config[self.GOOGLE_GEOCODER_API_KEY] = org_config[self.GOOGLE_GEOCODER_API_KEY]
|