From cd8173c7e0bd35223e2bd9be311e1383e67fab7e Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 23 Oct 2018 16:36:05 +0200 Subject: [PATCH 1/5] Include note to explain why some isodistances could not be precise --- doc/isoline_functions.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/isoline_functions.md b/doc/isoline_functions.md index 97949b9..743c589 100644 --- a/doc/isoline_functions.md +++ b/doc/isoline_functions.md @@ -16,6 +16,8 @@ The following functions provide an isoline generator service, based on time or d Displays a contoured line on a map, connecting geometries to a defined area, measured by an equal range of distance (in meters). +Note that not all the providers, for example TomTom, provide us a way to define the isoline limit in distance so we need to make some estimations. Due that estimations the produced isolines could not be 100% precise. + #### Arguments Name | Type | Description | Accepted values From 66409097803c16a200281c2e44dca42f0cc2db82 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 24 Oct 2018 12:00:08 +0200 Subject: [PATCH 2/5] Add provider for QPS manger in tomtom services --- .../cartodb_services/cartodb_services/tomtom/geocoder.py | 4 ++-- .../cartodb_services/cartodb_services/tomtom/routing.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py index b5ec592..f387bba 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py @@ -70,7 +70,7 @@ class TomTomGeocoder(Traceable): return False - @qps_retry(qps=5) + @qps_retry(qps=5, provider='tomtom') def geocode(self, searchtext, city=None, state_province=None, country=None): response = self.geocode_meta(searchtext, city, state_province, country) @@ -80,7 +80,7 @@ class TomTomGeocoder(Traceable): else: return response[0] - @qps_retry(qps=5) + @qps_retry(qps=5, provider='tomtom') def geocode_meta(self, searchtext, city=None, state_province=None, country=None): if searchtext: diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/routing.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/routing.py index dfc30fb..71dfa22 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tomtom/routing.py +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/routing.py @@ -89,7 +89,7 @@ class TomTomRouting(Traceable): point[ENTRY_LONGITUDE])) return geometry - @qps_retry(qps=5) + @qps_retry(qps=5, provider='tomtom') def directions(self, waypoints, profile=DEFAULT_PROFILE, date_time=DEFAULT_DEPARTAT): self._validate_profile(profile) From 17c993f6ef16daa6497070aa14df781a5213324c Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 24 Oct 2018 12:00:44 +0200 Subject: [PATCH 3/5] Bump version --- server/lib/python/cartodb_services/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index 2693d0e..02323b8 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.20.1', + version='0.20.2', description='CartoDB Services API Python Library', From 33f40bc945c08c2790e3a385e44568f31bacaed4 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 24 Oct 2018 17:38:23 +0200 Subject: [PATCH 4/5] TOMTOM uses 403 instead of 429 for rate limiting That has a great problem when we're dealing with legit 403 status for example deactivated user, forbidden access, etc. I've added a check for the HTTP header `X-Error-Detail-Header` in order to distinguish between legit 403 and 429 error messages Possible values for `X-Error-Detail-Header` in a 403 error: o Service Requires SSL : http is used instead of https (secure) o Invalid Referer : invalid 'Referer' header value is send to https://api.tomtom.com and allowed referer values are configured on specific API key o Account Over Queries Per Second Limit : rate limit exceeded o Account Inactive : incorrect API key/API key no longer valid --- .../python/cartodb_services/cartodb_services/tools/qps.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/qps.py b/server/lib/python/cartodb_services/cartodb_services/tools/qps.py index 24d110e..9ae3abd 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/qps.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/qps.py @@ -6,6 +6,9 @@ from exceptions import TimeoutException DEFAULT_RETRY_TIMEOUT = 60 DEFAULT_QUERIES_PER_SECOND = 10 +TOMTOM_403_RATE_LIMIT_HEADER = 'Account Over Queries Per Second Limit' +TOMTOM_DETAIL_HEADER = 'X-Error-Detail-Header' + def qps_retry(original_function=None, **options): """ Query Per Second retry decorator @@ -46,6 +49,8 @@ class QPSService: response = getattr(e, 'response', None) if response is not None: if self._provider is not None and self._provider == 'tomtom' and (response.status_code == 403): + if response.headers.get(TOMTOM_DETAIL_HEADER) != TOMTOM_403_RATE_LIMIT_HEADER: + raise e self.retry(start_time, attempt_number) elif response.status_code == 429: self.retry(start_time, attempt_number) From 36c42096e420d2bab82f8d6ca38cede2ec77abdb Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 31 Oct 2018 13:58:29 +0100 Subject: [PATCH 5/5] Updated NEWS.md --- NEWS.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/NEWS.md b/NEWS.md index ca4e3d5..1cc1742 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,15 @@ + +Oct 31th, 2018 +============== +* Version `0.20.2` of the python library + * Added missing provider property to the QPS decorator in other TomTom services + * Now we only retry with the properly header coming from TomTom + +Oct 3rd, 2018 +============== +* Version `0.20.1` of the python library + * Fix QPS manager to retry with 403 status codes coming from TomTom + Sep 13th, 2018 ============== * Version `0.34.0` of the server, and `0.26.0` of the client.