cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added limits/quotas and tests for Mapbox services

Browse Source
This commit is contained in:
Antonio
2018-01-02 16:21:57 +01:00
parent 15340fedad
commit bb86f080fc
9 changed files with 169 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/lib/python/cartodb_services/cartodb_services/metrics/config.py
+121 -4
View File
@@ -135,7 +135,8 @@ class RoutingConfig(ServiceConfig):
PERIOD_END_DATE = 'period_end_date'
ROUTING_PROVIDER_KEY = 'routing_provider'
MAPZEN_PROVIDER = 'mapzen'
DEFAULT_PROVIDER = 'mapzen'
MAPBOX_PROVIDER = 'mapbox'
DEFAULT_PROVIDER = MAPBOX_PROVIDER
QUOTA_KEY = 'mapzen_routing_quota'
SOFT_LIMIT_KEY = 'soft_mapzen_routing_limit'
METRICS_LOG_KEY = 'routing_log_path'
@@ -148,6 +149,8 @@ class RoutingConfig(ServiceConfig):
self._routing_provider = self.DEFAULT_PROVIDER
self._mapzen_api_key = self._db_config.mapzen_routing_api_key
self._mapzen_service_params = self._db_config.mapzen_routing_service_params
self._mapbox_api_key = self._db_config.mapbox_routing_api_key
self._mapbox_service_params = self._db_config.mapbox_routing_service_params
self._set_monthly_quota()
self._set_soft_limit()
self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE])
@@ -156,6 +159,8 @@ class RoutingConfig(ServiceConfig):
def service_type(self):
if self._routing_provider == self.MAPZEN_PROVIDER:
return 'routing_mapzen'
elif self._routing_provider == self.MAPBOX_PROVIDER:
return 'routing_mapbox'
@property
def provider(self):
@@ -169,6 +174,14 @@ class RoutingConfig(ServiceConfig):
def mapzen_service_params(self):
return self._mapzen_service_params
@property
def mapbox_api_key(self):
return self._mapbox_api_key
@property
def mapbox_service_params(self):
return self._mapbox_service_params
@property
def monthly_quota(self):
return self._monthly_quota
@@ -203,8 +216,9 @@ class IsolinesRoutingConfig(ServiceConfig):
ISOLINES_PROVIDER_KEY = 'isolines_provider'
GEOCODER_PROVIDER_KEY = 'geocoder_provider'
MAPZEN_PROVIDER = 'mapzen'
MAPBOX_PROVIDER = 'mapbox'
HEREMAPS_PROVIDER = 'heremaps'
DEFAULT_PROVIDER = 'mapzen'
DEFAULT_PROVIDER = MAPBOX_PROVIDER
METRICS_LOG_KEY = 'isolines_log_path'
def __init__(self, redis_connection, db_conn, username, orgname=None):
@@ -232,6 +246,10 @@ class IsolinesRoutingConfig(ServiceConfig):
self._mapzen_matrix_api_key = self._db_config.mapzen_matrix_api_key
self._mapzen_matrix_service_params = db_config.mapzen_matrix_service_params
self._mapzen_isochrones_service_params = db_config.mapzen_isochrones_service_params
elif self._isolines_provider == self.MAPBOX_PROVIDER:
self._mapbox_matrix_api_key = self._db_config.mapbox_matrix_api_key
self._mapbox_matrix_service_params = db_config.mapbox_matrix_service_params
self._mapbox_isochrones_service_params = db_config.mapbox_isochrones_service_params
@property
def service_type(self):
@@ -239,6 +257,8 @@ class IsolinesRoutingConfig(ServiceConfig):
return 'here_isolines'
elif self._isolines_provider == self.MAPZEN_PROVIDER:
return 'mapzen_isolines'
elif self._isolines_provider == self.MAPBOX_PROVIDER:
return 'mapbox_isolines'
@property
def google_services_user(self):
@@ -284,6 +304,22 @@ class IsolinesRoutingConfig(ServiceConfig):
def mapzen_provider(self):
return self._isolines_provider == self.MAPZEN_PROVIDER
@property
def mapbox_matrix_api_key(self):
return self._mapbox_matrix_api_key
@property
def mapbox_matrix_service_params(self):
return self._mapbox_matrix_service_params
@property
def mapbox_isochrones_service_params(self):
return self._mapbox_isochrones_service_params
@property
def mapbox_provider(self):
return self._isolines_provider == self.MAPBOX_PROVIDER
@property
def heremaps_provider(self):
return self._isolines_provider == self.HEREMAPS_PROVIDER
@@ -340,12 +376,14 @@ class GeocoderConfig(ServiceConfig):
MAPZEN_GEOCODER = 'mapzen'
MAPZEN_GEOCODER_API_KEY = 'mapzen_geocoder_api_key'
GEOCODER_PROVIDER = 'geocoder_provider'
MAPBOX_GEOCODER = 'mapbox'
MAPBOX_GEOCODER_API_KEY = 'mapbox_geocoder_api_key'
QUOTA_KEY = 'geocoding_quota'
SOFT_LIMIT_KEY = 'soft_geocoding_limit'
USERNAME_KEY = 'username'
ORGNAME_KEY = 'orgname'
PERIOD_END_DATE = 'period_end_date'
DEFAULT_PROVIDER = 'mapzen'
DEFAULT_PROVIDER = MAPBOX_GEOCODER
METRICS_LOG_KEY = 'geocoder_log_path'
def __init__(self, redis_connection, db_conn, username, orgname=None, forced_provider=None):
@@ -366,6 +404,9 @@ class GeocoderConfig(ServiceConfig):
elif self._geocoder_provider == self.MAPZEN_GEOCODER:
if not self.mapzen_api_key:
raise ConfigException("""Mapzen config is not set up""")
elif self._geocoder_provider == self.MAPBOX_GEOCODER:
if not self.mapbox_api_key:
raise ConfigException("""Mapbox config is not set up""")
return True
@@ -397,6 +438,10 @@ class GeocoderConfig(ServiceConfig):
self._mapzen_api_key = db_config.mapzen_geocoder_api_key
self._cost_per_hit = 0
self._mapzen_service_params = db_config.mapzen_geocoder_service_params
elif self._geocoder_provider == self.MAPBOX_GEOCODER:
self._mapbox_api_key = db_config.mapbox_geocoder_api_key
self._cost_per_hit = 0
self._mapbox_service_params = db_config.mapbox_geocoder_service_params
@property
def service_type(self):
@@ -404,6 +449,8 @@ class GeocoderConfig(ServiceConfig):
return 'geocoder_google'
elif self._geocoder_provider == self.MAPZEN_GEOCODER:
return 'geocoder_mapzen'
elif self._geocoder_provider == self.MAPBOX_GEOCODER:
return 'geocoder_mapbox'
elif self._geocoder_provider == self.NOKIA_GEOCODER:
return 'geocoder_here'
@@ -419,6 +466,10 @@ class GeocoderConfig(ServiceConfig):
def mapzen_geocoder(self):
return self._geocoder_provider == self.MAPZEN_GEOCODER
@property
def mapbox_geocoder(self):
return self._geocoder_provider == self.MAPBOX_GEOCODER
@property
def google_client_id(self):
return self._google_maps_client_id
@@ -462,6 +513,14 @@ class GeocoderConfig(ServiceConfig):
def mapzen_service_params(self):
return self._mapzen_service_params
@property
def mapbox_api_key(self):
return self._mapbox_api_key
@property
def mapbox_service_params(self):
return self._mapbox_service_params
@property
def is_high_resolution(self):
return True
@@ -478,6 +537,7 @@ class GeocoderConfig(ServiceConfig):
def service(self):
return self._service
class ServicesDBConfig:
def __init__(self, db_conn, username, orgname):
@@ -490,6 +550,7 @@ class ServicesDBConfig:
self._get_server_config()
self._get_here_config()
self._get_mapzen_config()
self._get_mapbox_config()
self._get_data_observatory_config()
def _get_server_config(self):
@@ -535,6 +596,23 @@ class ServicesDBConfig:
self._mapzen_geocoder_quota = mapzen_conf['geocoder']['monthly_quota']
self._mapzen_geocoder_service_params = mapzen_conf['geocoder'].get('service', {})
def _get_mapbox_config(self):
mapbox_conf_json = self._get_conf('mapbox_conf')
if not mapbox_conf_json:
raise ConfigException('Mapzen configuration missing')
else:
mapzen_conf = json.loads(mapbox_conf_json)
self._mapbox_matrix_api_key = mapzen_conf['matrix']['api_key']
self._mapbox_matrix_quota = mapzen_conf['matrix']['monthly_quota']
self._mapbox_matrix_service_params = mapzen_conf['matrix'].get('service', {})
self._mapbox_isochrones_service_params = mapzen_conf.get('isochrones', {}).get('service', {})
self._mapbox_routing_api_key = mapzen_conf['routing']['api_key']
self._mapbox_routing_quota = mapzen_conf['routing']['monthly_quota']
self._mapbox_routing_service_params = mapzen_conf['routing'].get('service', {})
self._mapbox_geocoder_api_key = mapzen_conf['geocoder']['api_key']
self._mapbox_geocoder_quota = mapzen_conf['geocoder']['monthly_quota']
self._mapbox_geocoder_service_params = mapzen_conf['geocoder'].get('service', {})
def _get_data_observatory_config(self):
do_conf_json = self._get_conf('data_observatory_conf')
if not do_conf_json:
@@ -548,7 +626,6 @@ class ServicesDBConfig:
else:
self._data_observatory_connection_str = do_conf['connection']['production']
def _get_conf(self, key):
try:
sql = "SELECT cdb_dataservices_server.CDB_Conf_GetConf('{0}') as conf".format(key)
@@ -629,6 +706,46 @@ class ServicesDBConfig:
def mapzen_geocoder_service_params(self):
return self._mapzen_geocoder_service_params
@property
def mapbox_matrix_api_key(self):
return self._mapbox_matrix_api_key
@property
def mapbox_matrix_monthly_quota(self):
return self._mapbox_matrix_quota
@property
def mapbox_matrix_service_params(self):
return self._mapbox_matrix_service_params
@property
def mapbox_isochrones_service_params(self):
return self._mapbox_isochrones_service_params
@property
def mapbox_routing_api_key(self):
return self._mapbox_routing_api_key
@property
def mapbox_routing_monthly_quota(self):
return self._mapbox_routing_quota
@property
def mapbox_routing_service_params(self):
return self._mapbox_routing_service_params
@property
def mapbox_geocoder_api_key(self):
return self._mapbox_geocoder_api_key
@property
def mapbox_geocoder_monthly_quota(self):
return self._mapbox_geocoder_quota
@property
def mapbox_geocoder_service_params(self):
return self._mapbox_geocoder_service_params
@property
def data_observatory_connection_str(self):
return self._data_observatory_connection_str