cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added limits/quotas and tests for Mapbox services

Browse Source
This commit is contained in:
Antonio
2018-01-02 16:21:57 +01:00
parent 15340fedad
commit bb86f080fc
9 changed files with 169 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

125
server/lib/python/cartodb_services/cartodb_services/metrics/config.py
View File

@@ -135,7 +135,8 @@ class RoutingConfig(ServiceConfig):
PERIOD_END_DATE = 'period_end_date'
ROUTING_PROVIDER_KEY = 'routing_provider'
MAPZEN_PROVIDER = 'mapzen'
DEFAULT_PROVIDER = 'mapzen'
MAPBOX_PROVIDER = 'mapbox'
DEFAULT_PROVIDER = MAPBOX_PROVIDER
QUOTA_KEY = 'mapzen_routing_quota'
SOFT_LIMIT_KEY = 'soft_mapzen_routing_limit'
METRICS_LOG_KEY = 'routing_log_path'
@@ -148,6 +149,8 @@ class RoutingConfig(ServiceConfig):
self._routing_provider = self.DEFAULT_PROVIDER
self._mapzen_api_key = self._db_config.mapzen_routing_api_key
self._mapzen_service_params = self._db_config.mapzen_routing_service_params
self._mapbox_api_key = self._db_config.mapbox_routing_api_key
self._mapbox_service_params = self._db_config.mapbox_routing_service_params
self._set_monthly_quota()
self._set_soft_limit()
self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE])
@@ -156,6 +159,8 @@ class RoutingConfig(ServiceConfig):
def service_type(self):
if self._routing_provider == self.MAPZEN_PROVIDER:
return 'routing_mapzen'
elif self._routing_provider == self.MAPBOX_PROVIDER:
return 'routing_mapbox'
@property
def provider(self):
@@ -169,6 +174,14 @@ class RoutingConfig(ServiceConfig):
def mapzen_service_params(self):
return self._mapzen_service_params
@property
def mapbox_api_key(self):
return self._mapbox_api_key
@property
def mapbox_service_params(self):
return self._mapbox_service_params
@property
def monthly_quota(self):
return self._monthly_quota
@@ -203,8 +216,9 @@ class IsolinesRoutingConfig(ServiceConfig):
ISOLINES_PROVIDER_KEY = 'isolines_provider'
GEOCODER_PROVIDER_KEY = 'geocoder_provider'
MAPZEN_PROVIDER = 'mapzen'
MAPBOX_PROVIDER = 'mapbox'
HEREMAPS_PROVIDER = 'heremaps'
DEFAULT_PROVIDER = 'mapzen'
DEFAULT_PROVIDER = MAPBOX_PROVIDER
METRICS_LOG_KEY = 'isolines_log_path'
def __init__(self, redis_connection, db_conn, username, orgname=None):
@@ -232,6 +246,10 @@ class IsolinesRoutingConfig(ServiceConfig):
self._mapzen_matrix_api_key = self._db_config.mapzen_matrix_api_key
self._mapzen_matrix_service_params = db_config.mapzen_matrix_service_params
self._mapzen_isochrones_service_params = db_config.mapzen_isochrones_service_params
elif self._isolines_provider == self.MAPBOX_PROVIDER:
self._mapbox_matrix_api_key = self._db_config.mapbox_matrix_api_key
self._mapbox_matrix_service_params = db_config.mapbox_matrix_service_params
self._mapbox_isochrones_service_params = db_config.mapbox_isochrones_service_params
@property
def service_type(self):
@@ -239,6 +257,8 @@ class IsolinesRoutingConfig(ServiceConfig):
return 'here_isolines'
elif self._isolines_provider == self.MAPZEN_PROVIDER:
return 'mapzen_isolines'
elif self._isolines_provider == self.MAPBOX_PROVIDER:
return 'mapbox_isolines'
@property
def google_services_user(self):
@@ -284,6 +304,22 @@ class IsolinesRoutingConfig(ServiceConfig):
def mapzen_provider(self):
return self._isolines_provider == self.MAPZEN_PROVIDER
@property
def mapbox_matrix_api_key(self):
return self._mapbox_matrix_api_key
@property
def mapbox_matrix_service_params(self):
return self._mapbox_matrix_service_params
@property
def mapbox_isochrones_service_params(self):
return self._mapbox_isochrones_service_params
@property
def mapbox_provider(self):
return self._isolines_provider == self.MAPBOX_PROVIDER
@property
def heremaps_provider(self):
return self._isolines_provider == self.HEREMAPS_PROVIDER
@@ -340,12 +376,14 @@ class GeocoderConfig(ServiceConfig):
MAPZEN_GEOCODER = 'mapzen'
MAPZEN_GEOCODER_API_KEY = 'mapzen_geocoder_api_key'
GEOCODER_PROVIDER = 'geocoder_provider'
MAPBOX_GEOCODER = 'mapbox'
MAPBOX_GEOCODER_API_KEY = 'mapbox_geocoder_api_key'
QUOTA_KEY = 'geocoding_quota'
SOFT_LIMIT_KEY = 'soft_geocoding_limit'
USERNAME_KEY = 'username'
ORGNAME_KEY = 'orgname'
PERIOD_END_DATE = 'period_end_date'
DEFAULT_PROVIDER = 'mapzen'
DEFAULT_PROVIDER = MAPBOX_GEOCODER
METRICS_LOG_KEY = 'geocoder_log_path'
def __init__(self, redis_connection, db_conn, username, orgname=None, forced_provider=None):
@@ -366,6 +404,9 @@ class GeocoderConfig(ServiceConfig):
elif self._geocoder_provider == self.MAPZEN_GEOCODER:
if not self.mapzen_api_key:
raise ConfigException("""Mapzen config is not set up""")
elif self._geocoder_provider == self.MAPBOX_GEOCODER:
if not self.mapbox_api_key:
raise ConfigException("""Mapbox config is not set up""")
return True
@@ -397,6 +438,10 @@ class GeocoderConfig(ServiceConfig):
self._mapzen_api_key = db_config.mapzen_geocoder_api_key
self._cost_per_hit = 0
self._mapzen_service_params = db_config.mapzen_geocoder_service_params
elif self._geocoder_provider == self.MAPBOX_GEOCODER:
self._mapbox_api_key = db_config.mapbox_geocoder_api_key
self._cost_per_hit = 0
self._mapbox_service_params = db_config.mapbox_geocoder_service_params
@property
def service_type(self):
@@ -404,6 +449,8 @@ class GeocoderConfig(ServiceConfig):
return 'geocoder_google'
elif self._geocoder_provider == self.MAPZEN_GEOCODER:
return 'geocoder_mapzen'
elif self._geocoder_provider == self.MAPBOX_GEOCODER:
return 'geocoder_mapbox'
elif self._geocoder_provider == self.NOKIA_GEOCODER:
return 'geocoder_here'
@@ -419,6 +466,10 @@ class GeocoderConfig(ServiceConfig):
def mapzen_geocoder(self):
return self._geocoder_provider == self.MAPZEN_GEOCODER
@property
def mapbox_geocoder(self):
return self._geocoder_provider == self.MAPBOX_GEOCODER
@property
def google_client_id(self):
return self._google_maps_client_id
@@ -462,6 +513,14 @@ class GeocoderConfig(ServiceConfig):
def mapzen_service_params(self):
return self._mapzen_service_params
@property
def mapbox_api_key(self):
return self._mapbox_api_key
@property
def mapbox_service_params(self):
return self._mapbox_service_params
@property
def is_high_resolution(self):
return True
@@ -478,6 +537,7 @@ class GeocoderConfig(ServiceConfig):
def service(self):
return self._service
class ServicesDBConfig:
def __init__(self, db_conn, username, orgname):
@@ -490,6 +550,7 @@ class ServicesDBConfig:
self._get_server_config()
self._get_here_config()
self._get_mapzen_config()
self._get_mapbox_config()
self._get_data_observatory_config()
def _get_server_config(self):
@@ -535,6 +596,23 @@ class ServicesDBConfig:
self._mapzen_geocoder_quota = mapzen_conf['geocoder']['monthly_quota']
self._mapzen_geocoder_service_params = mapzen_conf['geocoder'].get('service', {})
def _get_mapbox_config(self):
mapbox_conf_json = self._get_conf('mapbox_conf')
if not mapbox_conf_json:
raise ConfigException('Mapzen configuration missing')
else:
mapzen_conf = json.loads(mapbox_conf_json)
self._mapbox_matrix_api_key = mapzen_conf['matrix']['api_key']
self._mapbox_matrix_quota = mapzen_conf['matrix']['monthly_quota']
self._mapbox_matrix_service_params = mapzen_conf['matrix'].get('service', {})
self._mapbox_isochrones_service_params = mapzen_conf.get('isochrones', {}).get('service', {})
self._mapbox_routing_api_key = mapzen_conf['routing']['api_key']
self._mapbox_routing_quota = mapzen_conf['routing']['monthly_quota']
self._mapbox_routing_service_params = mapzen_conf['routing'].get('service', {})
self._mapbox_geocoder_api_key = mapzen_conf['geocoder']['api_key']
self._mapbox_geocoder_quota = mapzen_conf['geocoder']['monthly_quota']
self._mapbox_geocoder_service_params = mapzen_conf['geocoder'].get('service', {})
def _get_data_observatory_config(self):
do_conf_json = self._get_conf('data_observatory_conf')
if not do_conf_json:
@@ -548,7 +626,6 @@ class ServicesDBConfig:
else:
self._data_observatory_connection_str = do_conf['connection']['production']
def _get_conf(self, key):
try:
sql = "SELECT cdb_dataservices_server.CDB_Conf_GetConf('{0}') as conf".format(key)
@@ -629,6 +706,46 @@ class ServicesDBConfig:
def mapzen_geocoder_service_params(self):
return self._mapzen_geocoder_service_params
@property
def mapbox_matrix_api_key(self):
return self._mapbox_matrix_api_key
@property
def mapbox_matrix_monthly_quota(self):
return self._mapbox_matrix_quota
@property
def mapbox_matrix_service_params(self):
return self._mapbox_matrix_service_params
@property
def mapbox_isochrones_service_params(self):
return self._mapbox_isochrones_service_params
@property
def mapbox_routing_api_key(self):
return self._mapbox_routing_api_key
@property
def mapbox_routing_monthly_quota(self):
return self._mapbox_routing_quota
@property
def mapbox_routing_service_params(self):
return self._mapbox_routing_service_params
@property
def mapbox_geocoder_api_key(self):
return self._mapbox_geocoder_api_key
@property
def mapbox_geocoder_monthly_quota(self):
return self._mapbox_geocoder_quota
@property
def mapbox_geocoder_service_params(self):
return self._mapbox_geocoder_service_params
@property
def data_observatory_connection_str(self):
return self._data_observatory_connection_str

9
server/lib/python/cartodb_services/cartodb_services/metrics/quota.py
View File

@@ -72,15 +72,18 @@ class QuotaChecker:
if re.match('geocoder_*',
self._user_service_config.service_type) is not None:
return self.__check_geocoder_quota()
elif re.match('here_isolines',
self._user_service_config.service_type) is not None:
return self.__check_isolines_quota()
elif re.match('mapzen_isolines',
self._user_service_config.service_type) is not None:
return self.__check_isolines_quota()
elif re.match('mapbox_isolines',
self._user_service_config.service_type) is not None:
return self.__check_isolines_quota()
elif re.match('routing_mapzen',
self._user_service_config.service_type) is not None:
return self.__check_routing_quota()
elif re.match('routing_mapbox',
self._user_service_config.service_type) is not None:
return self.__check_routing_quota()
elif re.match('obs_*',
self._user_service_config.service_type) is not None:
return self.__check_data_observatory_quota()

11
server/lib/python/cartodb_services/cartodb_services/metrics/user.py
View File

@@ -21,7 +21,9 @@ class UserMetricsService:
SERVICE_GEOCODER_CACHE = 'geocoder_cache'
SERVICE_HERE_ISOLINES = 'here_isolines'
SERVICE_MAPZEN_ISOLINES = 'mapzen_isolines'
SERVICE_MAPBOX_ISOLINES = 'mapbox_isolines'
SERVICE_MAPZEN_ROUTING = 'routing_mapzen'
SERVICE_MAPBOX_ROUTING = 'routing_mapbox'
SERVICE_OBSERVATORY = 'obs_general'
DAY_OF_MONTH_ZERO_PADDED = '%d'
@@ -32,11 +34,12 @@ class UserMetricsService:
self._orgname = user_geocoder_config.organization
def used_quota(self, service_type, date):
if service_type == self.SERVICE_HERE_ISOLINES:
if service_type in [self.SERVICE_HERE_ISOLINES,
self.SERVICE_MAPZEN_ISOLINES,
self.SERVICE_MAPBOX_ISOLINES]:
return self.__used_isolines_quota(service_type, date)
if service_type == self.SERVICE_MAPZEN_ISOLINES:
return self.__used_isolines_quota(service_type, date)
elif service_type == self.SERVICE_MAPZEN_ROUTING:
elif service_type in [self.SERVICE_MAPZEN_ROUTING,
self.SERVICE_MAPBOX_ROUTING]:
return self.__used_routing_quota(service_type, date)
elif service_type == self.SERVICE_OBSERVATORY:
return self.__used_observatory_quota(service_type, date)

2
server/lib/python/cartodb_services/cartodb_services/tools/polyline.py
View File

@@ -48,7 +48,7 @@ class PolyLine:
def polyline_to_linestring(polyline):
"""Convert a Mapzen polyline shape to a PostGIS multipolygon"""
"""Convert a Mapzen polyline shape to a PostGIS linestring"""
coordinates = []
for point in polyline:
# Divide by 10 because mapzen uses one more decimal than the