From 822c574b5c3fb935898100ccc6b9557b03c6e4e6 Mon Sep 17 00:00:00 2001 From: antoniocarlon Date: Tue, 11 Feb 2020 10:00:27 +0100 Subject: [PATCH] Deleted old mapbox isolines implementation --- .../cartodb_services/mapbox/__init__.py | 2 - .../cartodb_services/mapbox/isolines.py | 241 ++++++++---------- .../cartodb_services/mapbox/matrix_client.py | 92 ------- .../cartodb_services/mapbox/true_isolines.py | 154 ----------- .../cartodb_services/mapbox/types.py | 1 - .../cartodb_services/metrics/config.py | 81 ++---- .../cartodb_services/metrics/user.py | 2 - .../service/mapbox_isolines_config.py | 4 +- .../service/mapbox_true_isolines_config.py | 123 --------- .../test/metrics/test_config.py | 4 +- .../cartodb_services/test/test_helper.py | 3 +- .../test/test_mapboxisoline.py | 19 +- .../test/test_mapboxmatrix.py | 58 ----- .../test/test_mapboxtrueisoline.py | 42 --- .../test/test_quota_service.py | 17 -- 15 files changed, 138 insertions(+), 705 deletions(-) delete mode 100644 server/lib/python/cartodb_services/cartodb_services/mapbox/matrix_client.py delete mode 100644 server/lib/python/cartodb_services/cartodb_services/mapbox/true_isolines.py delete mode 100644 server/lib/python/cartodb_services/cartodb_services/refactor/service/mapbox_true_isolines_config.py delete mode 100644 server/lib/python/cartodb_services/test/test_mapboxmatrix.py delete mode 100644 server/lib/python/cartodb_services/test/test_mapboxtrueisoline.py diff --git a/server/lib/python/cartodb_services/cartodb_services/mapbox/__init__.py b/server/lib/python/cartodb_services/cartodb_services/mapbox/__init__.py index 5709f66..a7afcc6 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapbox/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapbox/__init__.py @@ -2,5 +2,3 @@ from routing import MapboxRouting, MapboxRoutingResponse from geocoder import MapboxGeocoder from bulk_geocoder import MapboxBulkGeocoder from isolines import MapboxIsolines, MapboxIsochronesResponse -from true_isolines import MapboxTrueIsolines, MapboxTrueIsochronesResponse -from matrix_client import MapboxMatrixClient diff --git a/server/lib/python/cartodb_services/cartodb_services/mapbox/isolines.py b/server/lib/python/cartodb_services/cartodb_services/mapbox/isolines.py index 4d6563f..f3056e7 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapbox/isolines.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapbox/isolines.py @@ -1,171 +1,142 @@ -''' -Python implementation for Mapbox services based isolines. -Uses the Mapbox Time Matrix service. -''' - import json +import requests +from uritemplate import URITemplate + +from cartodb_services.tools.exceptions import ServiceException +from cartodb_services.tools.qps import qps_retry from cartodb_services.tools import Coordinate -from cartodb_services.tools.spherical import (get_angles, - calculate_dest_location) -from cartodb_services.mapbox.matrix_client import (validate_profile, - DEFAULT_PROFILE, - PROFILE_WALKING, - PROFILE_DRIVING, - PROFILE_CYCLING, - ENTRY_DURATIONS, - ENTRY_DESTINATIONS, - ENTRY_LOCATION) + +BASEURI = ('https://api.mapbox.com/isochrone/v1/mapbox/{profile}/{coordinates}?contours_minutes={contours_minutes}&access_token={apikey}') + +PROFILE_DRIVING = 'driving' +PROFILE_CYCLING = 'cycling' +PROFILE_WALKING = 'walking' +DEFAULT_PROFILE = PROFILE_DRIVING + +MAX_TIME_RANGE = 60 * 60 # The maximum time that can be specified is 60 minutes. + # https://docs.mapbox.com/api/navigation/#retrieve-isochrones-around-a-location MAX_SPEEDS = { PROFILE_WALKING: 3.3333333, # In m/s, assuming 12km/h walking speed PROFILE_CYCLING: 16.67, # In m/s, assuming 60km/h max speed - PROFILE_DRIVING: 41.67 # In m/s, assuming 140km/h max speed + PROFILE_DRIVING: 38.89 # In m/s, assuming 140km/h max speed } -DEFAULT_NUM_ANGLES = 24 -DEFAULT_MAX_ITERS = 5 -DEFAULT_TOLERANCE = 0.1 +VALID_PROFILES = [PROFILE_DRIVING, + PROFILE_CYCLING, + PROFILE_WALKING] -MATRIX_NUM_ANGLES = DEFAULT_NUM_ANGLES -MATRIX_MAX_ITERS = DEFAULT_MAX_ITERS -MATRIX_TOLERANCE = DEFAULT_TOLERANCE - -UNIT_FACTOR_ISOCHRONE = 1.0 -UNIT_FACTOR_ISODISTANCE = 1000.0 -DEFAULT_UNIT_FACTOR = UNIT_FACTOR_ISOCHRONE +ENTRY_FEATURES = 'features' +ENTRY_GEOMETRY = 'geometry' +ENTRY_COORDINATES = 'coordinates' class MapboxIsolines(): ''' - Python wrapper for Mapbox services based isolines. + Python wrapper for Mapbox based isolines. ''' - def __init__(self, matrix_client, logger, service_params=None): + def __init__(self, apikey, logger, service_params=None): service_params = service_params or {} - self._matrix_client = matrix_client + self._apikey = apikey self._logger = logger - def _calculate_matrix_cost(self, origin, targets, isorange, - profile=DEFAULT_PROFILE, - unit_factor=UNIT_FACTOR_ISOCHRONE, - number_of_angles=MATRIX_NUM_ANGLES): - response = self._matrix_client.matrix([origin] + targets, - profile) + def _uri(self, origin, time_range, profile=DEFAULT_PROFILE): + uri = URITemplate(BASEURI).expand(apikey=self._apikey, + coordinates=origin, + contours_minutes=time_range, + profile=profile) + return uri + + def _validate_profile(self, profile): + if profile not in VALID_PROFILES: + raise ValueError('{profile} is not a valid profile. ' + 'Valid profiles are: {valid_profiles}'.format( + profile=profile, + valid_profiles=', '.join( + [x for x in VALID_PROFILES]))) + + def _validate_time_ranges(self, time_ranges): + for time_range in time_ranges: + if time_range > MAX_TIME_RANGE: + raise ValueError('Cannot query time ranges greater than {max_time_range} seconds'.format( + max_time_range=MAX_TIME_RANGE)) + + def _parse_coordinates(self, boundary): + coordinates = boundary.get(ENTRY_COORDINATES, []) + return [Coordinate(c[0], c[1]) for c in coordinates] + + def _parse_isochrone_service(self, response): json_response = json.loads(response) - if not json_response: - return [] - costs = [None] * number_of_angles - destinations = [None] * number_of_angles + coordinates = [] + if json_response: + for feature in json_response[ENTRY_FEATURES]: + geometry = feature[ENTRY_GEOMETRY] + coordinates.append(self._parse_coordinates(geometry)) - for idx, cost in enumerate(json_response[ENTRY_DURATIONS][0][1:]): - if cost: - costs[idx] = cost * unit_factor + return coordinates + + @qps_retry(qps=5, provider='mapbox') + def _calculate_isoline(self, origin, time_ranges, + profile=DEFAULT_PROFILE): + self._validate_time_ranges(time_ranges) + + origin = '{lon},{lat}'.format(lat=origin.latitude, + lon=origin.longitude) + + time_ranges.sort() + time_ranges_seconds = ','.join([str(round(t/60)) for t in time_ranges]) + + uri = self._uri(origin, time_ranges_seconds, profile) + + try: + response = requests.get(uri) + + if response.status_code == requests.codes.ok: + isolines = [] + coordinates = self._parse_isochrone_service(response.text) + for t, c in zip(time_ranges, coordinates): + isolines.append(MapboxIsochronesResponse(c, t)) + + return isolines + elif response.status_code == requests.codes.bad_request: + return [] + elif response.status_code == requests.codes.unprocessable_entity: + return [] else: - costs[idx] = isorange - - for idx, destination in enumerate(json_response[ENTRY_DESTINATIONS][1:]): - destinations[idx] = Coordinate(destination[ENTRY_LOCATION][0], - destination[ENTRY_LOCATION][1]) - - return costs, destinations + raise ServiceException(response.status_code, response) + except requests.Timeout as te: + # In case of timeout we want to stop the job because the server + # could be down + self._logger.error('Timeout connecting to Mapbox isochrone service', + te) + raise ServiceException('Error getting isochrone data from Mapbox', + None) + except requests.ConnectionError as ce: + # Don't raise the exception to continue with the geocoding job + self._logger.error('Error connecting to Mapbox isochrone service', + exception=ce) + return [] def calculate_isochrone(self, origin, time_ranges, profile=DEFAULT_PROFILE): - validate_profile(profile) + self._validate_profile(profile) - max_speed = MAX_SPEEDS[profile] - - isochrones = [] - for time_range in time_ranges: - upper_rmax = max_speed * time_range # an upper bound for the radius - - coordinates = self.calculate_isoline(origin=origin, - isorange=time_range, - upper_rmax=upper_rmax, - cost_method=self._calculate_matrix_cost, - profile=profile, - unit_factor=UNIT_FACTOR_ISOCHRONE, - number_of_angles=MATRIX_NUM_ANGLES, - max_iterations=MATRIX_MAX_ITERS, - tolerance=MATRIX_TOLERANCE) - isochrones.append(MapboxIsochronesResponse(coordinates, - time_range)) - return isochrones + return self._calculate_isoline(origin=origin, + time_ranges=time_ranges, + profile=profile) def calculate_isodistance(self, origin, distance_range, profile=DEFAULT_PROFILE): - validate_profile(profile) + self._validate_profile(profile) max_speed = MAX_SPEEDS[profile] time_range = distance_range / max_speed - return self.calculate_isochrone(origin=origin, - time_ranges=[time_range], - profile=profile)[0].coordinates - - def calculate_isoline(self, origin, isorange, upper_rmax, - cost_method=_calculate_matrix_cost, - profile=DEFAULT_PROFILE, - unit_factor=DEFAULT_UNIT_FACTOR, - number_of_angles=DEFAULT_NUM_ANGLES, - max_iterations=DEFAULT_MAX_ITERS, - tolerance=DEFAULT_TOLERANCE): - # Formally, a solution is an array of {angle, radius, lat, lon, cost} - # with cardinality number_of_angles - # we're looking for a solution in which - # abs(cost - isorange) / isorange <= TOLERANCE - - # Initial setup - angles = get_angles(number_of_angles) - rmax = [upper_rmax] * number_of_angles - rmin = [0.0] * number_of_angles - location_estimates = [calculate_dest_location(origin, a, - upper_rmax / 2.0) - for a in angles] - - # Iterate to refine the first solution - for i in xrange(0, max_iterations): - # Calculate the "actual" cost for each location estimate. - # NOTE: sometimes it cannot calculate the cost and returns None. - # Just assume isorange and stop the calculations there - - costs, destinations = cost_method(origin=origin, - targets=location_estimates, - isorange=isorange, - profile=profile, - unit_factor=unit_factor, - number_of_angles=number_of_angles) - - if not costs: - continue - - errors = [(cost - isorange) / float(isorange) for cost in costs] - max_abs_error = max([abs(e) for e in errors]) - if max_abs_error <= tolerance: - # good enough, stop there - break - - # let's refine the solution, binary search - for j in xrange(0, number_of_angles): - - if abs(errors[j]) > tolerance: - if errors[j] > 0: - rmax[j] = (rmax[j] + rmin[j]) / 2.0 - else: - rmin[j] = (rmax[j] + rmin[j]) / 2.0 - - location_estimates[j] = calculate_dest_location(origin, - angles[j], - (rmax[j] + rmin[j]) / 2.0) - - # delete points that got None - location_estimates_filtered = [] - for i, c in enumerate(costs): - if c != isorange and c < isorange * (1 + tolerance): - location_estimates_filtered.append(destinations[i]) - - return location_estimates_filtered + return self._calculate_isoline(origin=origin, + time_ranges=[time_range], + profile=profile)[0].coordinates class MapboxIsochronesResponse: diff --git a/server/lib/python/cartodb_services/cartodb_services/mapbox/matrix_client.py b/server/lib/python/cartodb_services/cartodb_services/mapbox/matrix_client.py deleted file mode 100644 index fd4cf1f..0000000 --- a/server/lib/python/cartodb_services/cartodb_services/mapbox/matrix_client.py +++ /dev/null @@ -1,92 +0,0 @@ -''' -Python client for the Mapbox Time Matrix service. -''' - -import requests -from cartodb_services.metrics import Traceable -from cartodb_services.tools.coordinates import (validate_coordinates, - marshall_coordinates) -from cartodb_services.tools.exceptions import ServiceException -from cartodb_services.tools.qps import qps_retry - -BASEURI = ('https://api.mapbox.com/directions-matrix/v1/mapbox/{profile}/' - '{coordinates}' - '?access_token={token}' - '&sources=0' # Set the first coordinate as source... - '&destinations=all') # ...and the rest as destinations - -NUM_COORDINATES_MIN = 2 # https://www.mapbox.com/api-documentation/#matrix -NUM_COORDINATES_MAX = 25 # https://www.mapbox.com/api-documentation/#matrix - -PROFILE_DRIVING_TRAFFIC = 'driving-traffic' -PROFILE_DRIVING = 'driving' -PROFILE_CYCLING = 'cycling' -PROFILE_WALKING = 'walking' -DEFAULT_PROFILE = PROFILE_DRIVING - -VALID_PROFILES = [PROFILE_DRIVING_TRAFFIC, - PROFILE_DRIVING, - PROFILE_CYCLING, - PROFILE_WALKING] - -ENTRY_DURATIONS = 'durations' -ENTRY_DESTINATIONS = 'destinations' -ENTRY_LOCATION = 'location' - - -def validate_profile(profile): - if profile not in VALID_PROFILES: - raise ValueError('{profile} is not a valid profile. ' - 'Valid profiles are: {valid_profiles}'.format( - profile=profile, - valid_profiles=', '.join( - [x for x in VALID_PROFILES]))) - - -class MapboxMatrixClient(Traceable): - ''' - Python wrapper for the Mapbox Time Matrix service. - ''' - - def __init__(self, token, logger, service_params=None): - service_params = service_params or {} - self._token = token - self._logger = logger - - def _uri(self, coordinates, profile=DEFAULT_PROFILE): - return BASEURI.format(profile=profile, coordinates=coordinates, - token=self._token) - - @qps_retry(qps=1) - def matrix(self, coordinates, profile=DEFAULT_PROFILE): - validate_profile(profile) - validate_coordinates(coordinates, - NUM_COORDINATES_MIN, NUM_COORDINATES_MAX) - - coords = marshall_coordinates(coordinates) - - uri = self._uri(coords, profile) - - try: - response = requests.get(uri) - - if response.status_code == requests.codes.ok: - return response.text - elif response.status_code == requests.codes.bad_request: - return '{}' - elif response.status_code == requests.codes.unprocessable_entity: - return '{}' - else: - raise ServiceException(response.status_code, response) - except requests.Timeout as te: - # In case of timeout we want to stop the job because the server - # could be down - self._logger.error('Timeout connecting to Mapbox matrix service', - te) - raise ServiceException('Error getting matrix data from Mapbox', - None) - except requests.ConnectionError as ce: - # Don't raise the exception to continue with the geocoding job - self._logger.error('Error connecting to Mapbox matrix service', - exception=ce) - return '{}' diff --git a/server/lib/python/cartodb_services/cartodb_services/mapbox/true_isolines.py b/server/lib/python/cartodb_services/cartodb_services/mapbox/true_isolines.py deleted file mode 100644 index 89208f1..0000000 --- a/server/lib/python/cartodb_services/cartodb_services/mapbox/true_isolines.py +++ /dev/null @@ -1,154 +0,0 @@ -import json -import requests -from uritemplate import URITemplate - -from cartodb_services.tools.exceptions import ServiceException -from cartodb_services.tools.qps import qps_retry -from cartodb_services.tools import Coordinate - -BASEURI = ('https://api.mapbox.com/isochrone/v1/mapbox/{profile}/{coordinates}?contours_minutes={contours_minutes}&access_token={apikey}') - -PROFILE_DRIVING = 'driving' -PROFILE_CYCLING = 'cycling' -PROFILE_WALKING = 'walking' -DEFAULT_PROFILE = PROFILE_DRIVING - -MAX_TIME_RANGE = 60 * 60 # The maximum time that can be specified is 60 minutes. - # https://docs.mapbox.com/api/navigation/#retrieve-isochrones-around-a-location - -MAX_SPEEDS = { - PROFILE_WALKING: 3.3333333, # In m/s, assuming 12km/h walking speed - PROFILE_CYCLING: 16.67, # In m/s, assuming 60km/h max speed - PROFILE_DRIVING: 38.89 # In m/s, assuming 140km/h max speed -} - -VALID_PROFILES = [PROFILE_DRIVING, - PROFILE_CYCLING, - PROFILE_WALKING] - -ENTRY_FEATURES = 'features' -ENTRY_GEOMETRY = 'geometry' -ENTRY_COORDINATES = 'coordinates' - - -class MapboxTrueIsolines(): - ''' - Python wrapper for Mapbox based isolines. - ''' - - def __init__(self, apikey, logger, service_params=None): - service_params = service_params or {} - self._apikey = apikey - self._logger = logger - - def _uri(self, origin, time_range, profile=DEFAULT_PROFILE): - uri = URITemplate(BASEURI).expand(apikey=self._apikey, - coordinates=origin, - contours_minutes=time_range, - profile=profile) - return uri - - def _validate_profile(self, profile): - if profile not in VALID_PROFILES: - raise ValueError('{profile} is not a valid profile. ' - 'Valid profiles are: {valid_profiles}'.format( - profile=profile, - valid_profiles=', '.join( - [x for x in VALID_PROFILES]))) - - def _validate_time_ranges(self, time_ranges): - for time_range in time_ranges: - if time_range > MAX_TIME_RANGE: - raise ValueError('Cannot query time ranges greater than {max_time_range} seconds'.format( - max_time_range=MAX_TIME_RANGE)) - - def _parse_coordinates(self, boundary): - coordinates = boundary.get(ENTRY_COORDINATES, []) - return [Coordinate(c[0], c[1]) for c in coordinates] - - def _parse_isochrone_service(self, response): - json_response = json.loads(response) - - coordinates = [] - if json_response: - for feature in json_response[ENTRY_FEATURES]: - geometry = feature[ENTRY_GEOMETRY] - coordinates.append(self._parse_coordinates(geometry)) - - return coordinates - - @qps_retry(qps=5, provider='mapbox_iso') - def _calculate_isoline(self, origin, time_ranges, - profile=DEFAULT_PROFILE): - self._validate_time_ranges(time_ranges) - - origin = '{lon},{lat}'.format(lat=origin.latitude, - lon=origin.longitude) - - time_ranges.sort() - time_ranges_seconds = ','.join([str(round(t/60)) for t in time_ranges]) - - uri = self._uri(origin, time_ranges_seconds, profile) - - try: - response = requests.get(uri) - - if response.status_code == requests.codes.ok: - isolines = [] - coordinates = self._parse_isochrone_service(response.text) - for t, c in zip(time_ranges, coordinates): - isolines.append(MapboxTrueIsochronesResponse(c, t)) - - return isolines - elif response.status_code == requests.codes.bad_request: - return [] - elif response.status_code == requests.codes.unprocessable_entity: - return [] - else: - raise ServiceException(response.status_code, response) - except requests.Timeout as te: - # In case of timeout we want to stop the job because the server - # could be down - self._logger.error('Timeout connecting to Mapbox isochrone service', - te) - raise ServiceException('Error getting isochrone data from Mapbox', - None) - except requests.ConnectionError as ce: - # Don't raise the exception to continue with the geocoding job - self._logger.error('Error connecting to Mapbox isochrone service', - exception=ce) - return [] - - def calculate_isochrone(self, origin, time_ranges, - profile=DEFAULT_PROFILE): - self._validate_profile(profile) - - return self._calculate_isoline(origin=origin, - time_ranges=time_ranges, - profile=profile) - - def calculate_isodistance(self, origin, distance_range, - profile=DEFAULT_PROFILE): - self._validate_profile(profile) - - max_speed = MAX_SPEEDS[profile] - time_range = distance_range / max_speed - - return self._calculate_isoline(origin=origin, - time_ranges=[time_range], - profile=profile)[0].coordinates - - -class MapboxTrueIsochronesResponse: - - def __init__(self, coordinates, duration): - self._coordinates = coordinates - self._duration = duration - - @property - def coordinates(self): - return self._coordinates - - @property - def duration(self): - return self._duration diff --git a/server/lib/python/cartodb_services/cartodb_services/mapbox/types.py b/server/lib/python/cartodb_services/cartodb_services/mapbox/types.py index 57c6764..cc55bac 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapbox/types.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapbox/types.py @@ -1,7 +1,6 @@ MAPBOX_ROUTING_APIKEY_ROUNDROBIN = 'mapbox_routing_apikey_roundrobin' MAPBOX_GEOCODER_APIKEY_ROUNDROBIN = 'mapbox_geocoder_apikey_roundrobin' MAPBOX_ISOLINES_APIKEY_ROUNDROBIN = 'mapbox_isolines_apikey_roundrobin' -MAPBOX_ISO_ISOLINES_APIKEY_ROUNDROBIN = 'mapbox_iso_isolines_apikey_roundrobin' TRANSPORT_MODE_TO_MAPBOX = { 'car': 'driving', diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index f767b5d..dc7625d 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -224,7 +224,6 @@ class IsolinesRoutingConfig(ServiceConfig): GEOCODER_PROVIDER_KEY = 'geocoder_provider' MAPZEN_PROVIDER = 'mapzen' MAPBOX_PROVIDER = 'mapbox' - MAPBOX_ISO_PROVIDER = 'mapbox_iso' TOMTOM_PROVIDER = 'tomtom' HEREMAPS_PROVIDER = 'heremaps' DEFAULT_PROVIDER = MAPBOX_PROVIDER @@ -256,12 +255,8 @@ class IsolinesRoutingConfig(ServiceConfig): self._mapzen_matrix_service_params = db_config.mapzen_matrix_service_params self._mapzen_isochrones_service_params = db_config.mapzen_isochrones_service_params elif self._isolines_provider == self.MAPBOX_PROVIDER: - self._mapbox_matrix_api_keys = self._db_config.mapbox_matrix_api_keys - self._mapbox_matrix_service_params = db_config.mapbox_matrix_service_params - self._mapbox_isochrones_service_params = db_config.mapbox_isochrones_service_params - elif self._isolines_provider == self.MAPBOX_ISO_PROVIDER: - self._mapbox_iso_isolines_api_keys = self._db_config.mapbox_iso_isolines_api_keys - self._mapbox_iso_isolines_service_params = db_config.mapbox_iso_isolines_service_params + self._mapbox_isolinesx_api_keys = self._db_config.mapbox_isolines_api_keys + self._mapbox_isolines_service_params = db_config.mapbox_isolines_service_params elif self._isolines_provider == self.TOMTOM_PROVIDER: self._tomtom_isolinesx_api_keys = self._db_config.tomtom_isolines_api_keys self._tomtom_isolines_service_params = db_config.tomtom_isolines_service_params @@ -274,8 +269,6 @@ class IsolinesRoutingConfig(ServiceConfig): return 'mapzen_isolines' elif self._isolines_provider == self.MAPBOX_PROVIDER: return 'mapbox_isolines' - elif self._isolines_provider == self.MAPBOX_ISO_PROVIDER: - return 'mapbox_iso_isolines' elif self._isolines_provider == self.TOMTOM_PROVIDER: return 'tomtom_isolines' @@ -324,33 +317,17 @@ class IsolinesRoutingConfig(ServiceConfig): return self._isolines_provider == self.MAPZEN_PROVIDER @property - def mapbox_matrix_api_keys(self): - return self._mapbox_matrix_api_keys + def mapbox_isolines_api_keys(self): + return self._mapbox_isolines_api_keys @property - def mapbox_matrix_service_params(self): - return self._mapbox_matrix_service_params - - @property - def mapbox_isochrones_service_params(self): - return self._mapbox_isochrones_service_params + def mapbox_isolines_service_params(self): + return self._mapbox_isolines_service_params @property def mapbox_provider(self): return self._isolines_provider == self.MAPBOX_PROVIDER - @property - def mapbox_iso_isolines_api_keys(self): - return self._mapbox_iso_isolines_api_keys - - @property - def mapbox_iso_isolines_service_params(self): - return self._mapbox_iso_isolines_service_params - - @property - def mapbox_iso_provider(self): - return self._isolines_provider == self.MAPBOX_ISO_PROVIDER - @property def tomtom_isolines_api_keys(self): return self._tomtom_isolines_api_keys @@ -640,7 +617,6 @@ class ServicesDBConfig: self._get_here_config() self._get_mapzen_config() self._get_mapbox_config() - self._get_mapbox_iso_config() self._get_tomtom_config() self._get_geocodio_config() self._get_data_observatory_config() @@ -695,10 +671,9 @@ class ServicesDBConfig: raise ConfigException('Mapbox configuration missing') mapbox_conf = json.loads(mapbox_conf_json) - self._mapbox_matrix_api_keys = mapbox_conf['matrix']['api_keys'] - self._mapbox_matrix_quota = mapbox_conf['matrix']['monthly_quota'] - self._mapbox_matrix_service_params = mapbox_conf['matrix'].get('service', {}) - self._mapbox_isochrones_service_params = mapbox_conf.get('isochrones', {}).get('service', {}) + self._mapbox_isolines_api_keys = mapbox_conf['isolines']['api_keys'] + self._mapbox_isolines_quota = mapbox_conf['isolines']['monthly_quota'] + self._mapbox_isolines_service_params = mapbox_conf.get('isolines', {}).get('service', {}) self._mapbox_routing_api_keys = mapbox_conf['routing']['api_keys'] self._mapbox_routing_quota = mapbox_conf['routing']['monthly_quota'] self._mapbox_routing_service_params = mapbox_conf['routing'].get('service', {}) @@ -706,16 +681,6 @@ class ServicesDBConfig: self._mapbox_geocoder_quota = mapbox_conf['geocoder']['monthly_quota'] self._mapbox_geocoder_service_params = mapbox_conf['geocoder'].get('service', {}) - def _get_mapbox_iso_config(self): - mapbox_iso_conf_json = self._get_conf('mapbox_iso_conf') - if not mapbox_iso_conf_json: - raise ConfigException('Mapbox True Isochrones configuration missing') - - mapbox_iso_conf = json.loads(mapbox_iso_conf_json) - self._mapbox_iso_isolines_api_keys = mapbox_iso_conf['isolines']['api_keys'] - self._mapbox_iso_isolines_quota = mapbox_iso_conf['isolines']['monthly_quota'] - self._mapbox_iso_isolines_service_params = mapbox_iso_conf.get('isolines', {}).get('service', {}) - def _get_tomtom_config(self): tomtom_conf_json = self._get_conf('tomtom_conf') if not tomtom_conf_json: @@ -836,20 +801,16 @@ class ServicesDBConfig: return self._mapzen_geocoder_service_params @property - def mapbox_matrix_api_keys(self): - return self._mapbox_matrix_api_keys + def mapbox_isolines_api_keys(self): + return self._mapbox_isolines_api_keys @property - def mapbox_matrix_monthly_quota(self): - return self._mapbox_matrix_quota + def mapbox_isolines_monthly_quota(self): + return self._mapbox_isolines_quota @property - def mapbox_matrix_service_params(self): - return self._mapbox_matrix_service_params - - @property - def mapbox_isochrones_service_params(self): - return self._mapbox_isochrones_service_params + def mapbox_isolines_service_params(self): + return self._mapbox_isolines_service_params @property def mapbox_routing_api_keys(self): @@ -875,18 +836,6 @@ class ServicesDBConfig: def mapbox_geocoder_service_params(self): return self._mapbox_geocoder_service_params - @property - def mapbox_iso_isolines_api_keys(self): - return self._mapbox_iso_isolines_api_keys - - @property - def mapbox_iso_isolines_monthly_quota(self): - return self._mapbox_iso_isolines_quota - - @property - def mapbox_iso_isolines_service_params(self): - return self._mapbox_iso_isolines_service_params - @property def tomtom_isolines_api_keys(self): return self._tomtom_isolines_api_keys diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/user.py b/server/lib/python/cartodb_services/cartodb_services/metrics/user.py index d176183..70bc9a3 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/user.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/user.py @@ -22,7 +22,6 @@ class UserMetricsService: SERVICE_HERE_ISOLINES = 'here_isolines' SERVICE_MAPZEN_ISOLINES = 'mapzen_isolines' SERVICE_MAPBOX_ISOLINES = 'mapbox_isolines' - SERVICE_MAPBOX_ISO_ISOLINES = 'mapbox_iso_isolines' SERVICE_TOMTOM_ISOLINES = 'tomtom_isolines' SERVICE_MAPZEN_ROUTING = 'routing_mapzen' SERVICE_MAPBOX_ROUTING = 'routing_mapbox' @@ -40,7 +39,6 @@ class UserMetricsService: if service_type in [self.SERVICE_HERE_ISOLINES, self.SERVICE_MAPZEN_ISOLINES, self.SERVICE_MAPBOX_ISOLINES, - self.SERVICE_MAPBOX_ISO_ISOLINES, self.SERVICE_TOMTOM_ISOLINES]: return self.__used_isolines_quota(service_type, date) elif service_type in [self.SERVICE_MAPZEN_ROUTING, diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapbox_isolines_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapbox_isolines_config.py index 79baa29..5a252e9 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapbox_isolines_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapbox_isolines_config.py @@ -92,8 +92,8 @@ class MapboxIsolinesConfigBuilder(object): def get(self): mapbox_server_conf = self._server_conf.get('mapbox_conf') - mapbox_api_keys = mapbox_server_conf['matrix']['api_keys'] - mapbox_service_params = mapbox_server_conf['matrix'].get('service', {}) + mapbox_api_keys = mapbox_server_conf['isolines']['api_keys'] + mapbox_service_params = mapbox_server_conf['isolines'].get('service', {}) isolines_quota = self._get_quota() soft_isolines_limit = self._user_conf.get('soft_here_isolines_limit').lower() == 'true' diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapbox_true_isolines_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapbox_true_isolines_config.py deleted file mode 100644 index 7dbb445..0000000 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapbox_true_isolines_config.py +++ /dev/null @@ -1,123 +0,0 @@ -from dateutil.parser import parse as date_parse -from cartodb_services.refactor.service.utils import round_robin -from cartodb_services.mapbox.types import MAPBOX_ISO_ISOLINES_APIKEY_ROUNDROBIN - - -class MapboxTrueIsolinesConfig(object): - """ - Configuration needed to operate the Mapbox directions service. - """ - - def __init__(self, - isolines_quota, - soft_isolines_limit, - period_end_date, - cost_per_hit, - log_path, - mapbox_api_keys, - username, - organization, - service_params, - GD): - self._isolines_quota = isolines_quota - self._soft_isolines_limit = soft_isolines_limit - self._period_end_date = period_end_date - self._cost_per_hit = cost_per_hit - self._log_path = log_path - self._mapbox_api_keys = mapbox_api_keys - self._username = username - self._organization = organization - self._service_params = service_params - self._GD = GD - - @property - def service_type(self): - return 'mapbox_isolines' - - @property - def provider(self): - return 'mapbox' - - @property - def is_high_resolution(self): - return True - - @property - def isolines_quota(self): - return self._isolines_quota - - @property - def soft_isolines_limit(self): - return self._soft_isolines_limit - - @property - def period_end_date(self): - return self._period_end_date - - @property - def cost_per_hit(self): - return self._cost_per_hit - - @property - def log_path(self): - return self._log_path - - @property - def mapbox_api_key(self): - return round_robin(self._mapbox_api_keys, self._GD, - MAPBOX_ISO_ISOLINES_APIKEY_ROUNDROBIN) - - @property - def username(self): - return self._username - - @property - def organization(self): - return self._organization - - @property - def service_params(self): - return self._service_params - - -class MapboxTrueIsolinesConfigBuilder(object): - - def __init__(self, server_conf, user_conf, org_conf, username, orgname, GD): - self._server_conf = server_conf - self._user_conf = user_conf - self._org_conf = org_conf - self._username = username - self._orgname = orgname - self._GD = GD - - def get(self): - mapbox_server_conf = self._server_conf.get('mapbox_iso_conf') - mapbox_api_keys = mapbox_server_conf['isolines']['api_keys'] - mapbox_service_params = mapbox_server_conf['isolines'].get('service', {}) - - isolines_quota = self._get_quota() - soft_isolines_limit = self._user_conf.get('soft_here_isolines_limit').lower() == 'true' - cost_per_hit = 0 - period_end_date_str = self._org_conf.get('period_end_date') or self._user_conf.get('period_end_date') - period_end_date = date_parse(period_end_date_str) - - logger_conf = self._server_conf.get('logger_conf') - log_path = logger_conf.get('isolines_log_path', None) - - return MapboxTrueIsolinesConfig(isolines_quota, - soft_isolines_limit, - period_end_date, - cost_per_hit, - log_path, - mapbox_api_keys, - self._username, - self._orgname, - mapbox_service_params, - self._GD) - - def _get_quota(self): - isolines_quota = self._org_conf.get('here_isolines_quota') or self._user_conf.get('here_isolines_quota') - if isolines_quota is '': - return 0 - - return int(isolines_quota) diff --git a/server/lib/python/cartodb_services/test/metrics/test_config.py b/server/lib/python/cartodb_services/test/metrics/test_config.py index 0fdd6f8..a933a66 100644 --- a/server/lib/python/cartodb_services/test/metrics/test_config.py +++ b/server/lib/python/cartodb_services/test/metrics/test_config.py @@ -180,7 +180,7 @@ class TestGeocoderOrgConfig(TestCase): class TestIsolinesUserConfig(TestCase): # Don't test mapbox. See CartoDB/cartodb-management/issues/5199" - ISOLINES_PROVIDERS = ['heremaps', 'mapzen', 'tomtom', 'mapbox_iso'] + ISOLINES_PROVIDERS = ['heremaps', 'mapzen', 'tomtom'] def setUp(self): self.redis_conn = MockRedis() @@ -196,8 +196,6 @@ class TestIsolinesUserConfig(TestCase): assert isolines_config.service_type is 'mapzen_isolines' elif isolines_provider is 'mapbox': assert isolines_config.service_type is 'mapbox_isolines' - elif isolines_provider is 'mapbox_iso': - assert isolines_config.service_type is 'mapbox_iso_isolines' elif isolines_provider is 'tomtom': assert isolines_config.service_type is 'tomtom_isolines' else: diff --git a/server/lib/python/cartodb_services/test/test_helper.py b/server/lib/python/cartodb_services/test/test_helper.py index fed0839..82bb84b 100644 --- a/server/lib/python/cartodb_services/test/test_helper.py +++ b/server/lib/python/cartodb_services/test/test_helper.py @@ -76,8 +76,7 @@ def increment_service_uses(redis_conn, username, orgname=None, def plpy_mock_config(): plpy_mock._define_result("CDB_Conf_GetConf\('heremaps_conf'\)", [{'conf': '{"geocoder": {"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "app_id", "app_code": "code"}}'}]) plpy_mock._define_result("CDB_Conf_GetConf\('mapzen_conf'\)", [{'conf': '{"routing": {"api_key": "api_key_rou", "monthly_quota": 1500000}, "geocoder": {"api_key": "api_key_geo", "monthly_quota": 1500000}, "matrix": {"api_key": "api_key_mat", "monthly_quota": 1500000}}'}]) - plpy_mock._define_result("CDB_Conf_GetConf\('mapbox_conf'\)", [{'conf': '{"routing": {"api_keys": ["api_key_rou"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["api_key_geo"], "monthly_quota": 1500000}, "matrix": {"api_keys": ["api_key_mat"], "monthly_quota": 1500000}}'}]) - plpy_mock._define_result("CDB_Conf_GetConf\('mapbox_iso_conf'\)", [{'conf': '{"isolines": {"api_keys": ["api_key_mat"], "monthly_quota": 1500000}}'}]) + plpy_mock._define_result("CDB_Conf_GetConf\('mapbox_conf'\)", [{'conf': '{"routing": {"api_keys": ["api_key_rou"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["api_key_geo"], "monthly_quota": 1500000}, "isolines": {"api_keys": ["api_key_mat"], "monthly_quota": 1500000}}'}]) plpy_mock._define_result("CDB_Conf_GetConf\('tomtom_conf'\)", [{'conf': '{"routing": {"api_keys": ["api_key_rou"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["api_key_geo"], "monthly_quota": 1500000}, "isolines": {"api_keys": ["api_key_mat"], "monthly_quota": 1500000}}'}]) plpy_mock._define_result("CDB_Conf_GetConf\('geocodio_conf'\)", [{'conf': '{"geocoder": {"api_keys": ["api_key_geo"], "monthly_quota": 1500000}}'}]) plpy_mock._define_result("CDB_Conf_GetConf\('logger_conf'\)", [{'conf': '{"geocoder_log_path": "/dev/null"}'}]) diff --git a/server/lib/python/cartodb_services/test/test_mapboxisoline.py b/server/lib/python/cartodb_services/test/test_mapboxisoline.py index 091ac7c..4ba38a0 100644 --- a/server/lib/python/cartodb_services/test/test_mapboxisoline.py +++ b/server/lib/python/cartodb_services/test/test_mapboxisoline.py @@ -1,20 +1,27 @@ import unittest from mock import Mock -from cartodb_services.mapbox.isolines import MapboxIsolines -from cartodb_services.mapbox.matrix_client import DEFAULT_PROFILE -from cartodb_services.mapbox.matrix_client import MapboxMatrixClient +from cartodb_services.mapbox.isolines import MapboxIsolines, DEFAULT_PROFILE from cartodb_services.tools import Coordinate + from credentials import mapbox_api_key VALID_ORIGIN = Coordinate(-73.989, 40.733) -@unittest.skip("Stop using Matrix API. CartoDB/cartodb-management/issues/5199") class MapboxIsolinesTestCase(unittest.TestCase): def setUp(self): - matrix_client = MapboxMatrixClient(token=mapbox_api_key(), logger=Mock()) - self.mapbox_isolines = MapboxIsolines(matrix_client, logger=Mock()) + self.mapbox_isolines = MapboxIsolines(apikey=mapbox_api_key(), + logger=Mock()) + + def test_invalid_time_range(self): + time_ranges = [4000] + + with self.assertRaises(ValueError): + solution = self.mapbox_isolines.calculate_isochrone( + origin=VALID_ORIGIN, + profile=DEFAULT_PROFILE, + time_ranges=time_ranges) def test_calculate_isochrone(self): time_ranges = [300, 900] diff --git a/server/lib/python/cartodb_services/test/test_mapboxmatrix.py b/server/lib/python/cartodb_services/test/test_mapboxmatrix.py deleted file mode 100644 index b1ce42e..0000000 --- a/server/lib/python/cartodb_services/test/test_mapboxmatrix.py +++ /dev/null @@ -1,58 +0,0 @@ -import unittest -from mock import Mock -from cartodb_services.mapbox import MapboxMatrixClient -from cartodb_services.mapbox.matrix_client import DEFAULT_PROFILE -from cartodb_services.tools.exceptions import ServiceException -from cartodb_services.tools import Coordinate -from credentials import mapbox_api_key - -INVALID_TOKEN = 'invalid_token' -VALID_ORIGIN = Coordinate(-73.989, 40.733) -VALID_TARGET = Coordinate(-74, 40.733) -VALID_COORDINATES = [VALID_ORIGIN] + [VALID_TARGET] -NUM_COORDINATES_MAX = 25 -INVALID_COORDINATES_EMPTY = [] -INVALID_COORDINATES_MIN = [VALID_ORIGIN] -INVALID_COORDINATES_MAX = [VALID_ORIGIN] + \ - [VALID_TARGET - for x in range(0, NUM_COORDINATES_MAX + 1)] -VALID_PROFILE = DEFAULT_PROFILE -INVALID_PROFILE = 'invalid_profile' - - -@unittest.skip("Stop using Matrix API. CartoDB/cartodb-management/issues/5199") -class MapboxMatrixTestCase(unittest.TestCase): - def setUp(self): - self.matrix_client = MapboxMatrixClient(token=mapbox_api_key(), - logger=Mock()) - - def test_invalid_profile(self): - with self.assertRaises(ValueError): - self.matrix_client.matrix(VALID_COORDINATES, - INVALID_PROFILE) - - def test_invalid_coordinates_empty(self): - with self.assertRaises(ValueError): - self.matrix_client.matrix(INVALID_COORDINATES_EMPTY, - VALID_PROFILE) - - def test_invalid_coordinates_max(self): - with self.assertRaises(ValueError): - self.matrix_client.matrix(INVALID_COORDINATES_MAX, - VALID_PROFILE) - - def test_invalid_coordinates_min(self): - with self.assertRaises(ValueError): - self.matrix_client.matrix(INVALID_COORDINATES_MIN, - VALID_PROFILE) - - def test_invalid_token(self): - invalid_matrix = MapboxMatrixClient(token=INVALID_TOKEN, logger=Mock()) - with self.assertRaises(ServiceException): - invalid_matrix.matrix(VALID_COORDINATES, - VALID_PROFILE) - - def test_valid_request(self): - distance_matrix = self.matrix_client.matrix(VALID_COORDINATES, - VALID_PROFILE) - assert distance_matrix diff --git a/server/lib/python/cartodb_services/test/test_mapboxtrueisoline.py b/server/lib/python/cartodb_services/test/test_mapboxtrueisoline.py deleted file mode 100644 index 229496f..0000000 --- a/server/lib/python/cartodb_services/test/test_mapboxtrueisoline.py +++ /dev/null @@ -1,42 +0,0 @@ -import unittest -from mock import Mock -from cartodb_services.mapbox.true_isolines import MapboxTrueIsolines, DEFAULT_PROFILE -from cartodb_services.tools import Coordinate - -from credentials import mapbox_api_key - -VALID_ORIGIN = Coordinate(-73.989, 40.733) - - -class MapboxTrueIsolinesTestCase(unittest.TestCase): - - def setUp(self): - self.mapbox_isolines = MapboxTrueIsolines(apikey=mapbox_api_key(), - logger=Mock()) - - def test_invalid_time_range(self): - time_ranges = [4000] - - with self.assertRaises(ValueError): - solution = self.mapbox_isolines.calculate_isochrone( - origin=VALID_ORIGIN, - profile=DEFAULT_PROFILE, - time_ranges=time_ranges) - - def test_calculate_isochrone(self): - time_ranges = [300, 900] - solution = self.mapbox_isolines.calculate_isochrone( - origin=VALID_ORIGIN, - profile=DEFAULT_PROFILE, - time_ranges=time_ranges) - - assert solution - - def test_calculate_isodistance(self): - distance_range = 10000 - solution = self.mapbox_isolines.calculate_isodistance( - origin=VALID_ORIGIN, - profile=DEFAULT_PROFILE, - distance_range=distance_range) - - assert solution diff --git a/server/lib/python/cartodb_services/test/test_quota_service.py b/server/lib/python/cartodb_services/test/test_quota_service.py index 9c95dc1..ce1ca21 100644 --- a/server/lib/python/cartodb_services/test/test_quota_service.py +++ b/server/lib/python/cartodb_services/test/test_quota_service.py @@ -178,23 +178,6 @@ class TestQuotaService(TestCase): qs.increment_isolines_service_use(amount=1500000) assert qs.check_user_quota() is False - def test_should_check_user_mapbox_iso_isolines_quota_correctly(self): - qs = self.__build_isolines_quota_service('test_user', - provider='mapbox_iso') - qs.increment_isolines_service_use() - assert qs.check_user_quota() is True - qs.increment_isolines_service_use(amount=1500000) - assert qs.check_user_quota() is False - - def test_should_check_org_mapbox_iso_isolines_quota_correctly(self): - qs = self.__build_isolines_quota_service('test_user', - provider='mapbox_iso', - orgname='testorg') - qs.increment_isolines_service_use() - assert qs.check_user_quota() is True - qs.increment_isolines_service_use(amount=1500000) - assert qs.check_user_quota() is False - # Quick workaround so we don't take into account numer of credits # spent for users that have defined the quota. # See https://github.com/CartoDB/bigmetadata/issues/215