From 19d6cacdb37c2394d81d7a71456c9548c3973b34 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 26 Oct 2016 10:23:41 +0200 Subject: [PATCH 1/2] Fix empty response from matrix https://github.com/CartoDB/dataservices-api/issues/293 --- .../cartodb_services/mapzen/isolines.py | 6 ++++-- .../cartodb_services/mapzen/matrix_client.py | 16 +++++++++++++--- server/lib/python/cartodb_services/setup.py | 2 +- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py index daba485..c9e16fa 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py @@ -105,14 +105,16 @@ class MapzenIsolines: response = self._matrix_client.one_to_many([origin] + location_estimates, costing_model) costs = [None] * self.NUMBER_OF_ANGLES + if not response: + # In case the matrix client doesn't return any data + break + for idx, c in enumerate(response['one_to_many'][0][1:]): if c[cost_variable]: costs[idx] = c[cost_variable]*unit_factor else: costs[idx] = isorange - # self._logger.debug('i = %d, costs = %s' % (i, costs)) - errors = [(cost - isorange) / float(isorange) for cost in costs] max_abs_error = max([abs(e) for e in errors]) if max_abs_error <= self.TOLERANCE: diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py index 2b38155..96d11d0 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py @@ -43,7 +43,7 @@ class MatrixClient: } response = requests.get(self.ONE_TO_MANY_URL, params=request_params) - if not requests.codes.ok: + if response.status_code != requests.codes.ok: self._logger.error('Error trying to get matrix distance from mapzen', data={"response_status": response.status_code, "response_reason": response.reason, @@ -52,6 +52,16 @@ class MatrixClient: "response_headers": response.headers, "locations": locations, "costing": costing}) - raise ServiceException("Error trying to get matrix distance from mapzen", response) + # In case 4xx error we return empty because the error comes from + # the provided info by the user and we don't want to top the + # isolines generation + if response.status_code == requests.codes.bad_request: + return {} + else: + raise ServiceException("Error trying to get matrix distance from mapzen", response) - return response.json() + # response could return with empty json + try: + return response.json() + except: + return {} diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index cff4ecd..b3726f5 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.9.2', + version='0.9.3', description='CartoDB Services API Python Library', From 80963e25898542eb05eec286f7cfd9b79b0fc96c Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 26 Oct 2016 16:17:41 +0200 Subject: [PATCH 2/2] 504 errors return empty data instead of raise exception Due to some problems in Mapzen, we're receiving 504 errors from their servers. To mitigate this problem, instead of raise an exception, we're going to return empty data for that point --- .../cartodb_services/cartodb_services/mapzen/isolines.py | 2 +- .../cartodb_services/mapzen/matrix_client.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py index c9e16fa..e6f51be 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py @@ -86,7 +86,7 @@ class MapzenIsolines: def calculate_isoline(self, origin, costing_model, isorange, upper_rmax, cost_variable, unit_factor=1.0): # NOTE: not for production - self._logger.debug('Calculate isoline', data={"origin": origin, "costing_model": costing_model, "isorange": isorange}) + # self._logger.debug('Calculate isoline', data={"origin": origin, "costing_model": costing_model, "isorange": isorange}) # Formally, a solution is an array of {angle, radius, lat, lon, cost} with cardinality NUMBER_OF_ANGLES # we're looking for a solution in which abs(cost - isorange) / isorange <= TOLERANCE diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py index 96d11d0..e6bf3e3 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py @@ -57,6 +57,12 @@ class MatrixClient: # isolines generation if response.status_code == requests.codes.bad_request: return {} + elif response.status_coe == 504: + # Due to some unsolved problems in the Mapzen Matrix API we're + # getting randomly 504, probably timeouts. To avoid raise an + # exception in all the jobs, for now we're going to return + # empty in that case + return {} else: raise ServiceException("Error trying to get matrix distance from mapzen", response)