From 66409097803c16a200281c2e44dca42f0cc2db82 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 24 Oct 2018 12:00:08 +0200 Subject: [PATCH 1/3] Add provider for QPS manger in tomtom services --- .../cartodb_services/cartodb_services/tomtom/geocoder.py | 4 ++-- .../cartodb_services/cartodb_services/tomtom/routing.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py index b5ec592..f387bba 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py @@ -70,7 +70,7 @@ class TomTomGeocoder(Traceable): return False - @qps_retry(qps=5) + @qps_retry(qps=5, provider='tomtom') def geocode(self, searchtext, city=None, state_province=None, country=None): response = self.geocode_meta(searchtext, city, state_province, country) @@ -80,7 +80,7 @@ class TomTomGeocoder(Traceable): else: return response[0] - @qps_retry(qps=5) + @qps_retry(qps=5, provider='tomtom') def geocode_meta(self, searchtext, city=None, state_province=None, country=None): if searchtext: diff --git a/server/lib/python/cartodb_services/cartodb_services/tomtom/routing.py b/server/lib/python/cartodb_services/cartodb_services/tomtom/routing.py index dfc30fb..71dfa22 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tomtom/routing.py +++ b/server/lib/python/cartodb_services/cartodb_services/tomtom/routing.py @@ -89,7 +89,7 @@ class TomTomRouting(Traceable): point[ENTRY_LONGITUDE])) return geometry - @qps_retry(qps=5) + @qps_retry(qps=5, provider='tomtom') def directions(self, waypoints, profile=DEFAULT_PROFILE, date_time=DEFAULT_DEPARTAT): self._validate_profile(profile) From 17c993f6ef16daa6497070aa14df781a5213324c Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 24 Oct 2018 12:00:44 +0200 Subject: [PATCH 2/3] Bump version --- server/lib/python/cartodb_services/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index 2693d0e..02323b8 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.20.1', + version='0.20.2', description='CartoDB Services API Python Library', From 33f40bc945c08c2790e3a385e44568f31bacaed4 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 24 Oct 2018 17:38:23 +0200 Subject: [PATCH 3/3] TOMTOM uses 403 instead of 429 for rate limiting That has a great problem when we're dealing with legit 403 status for example deactivated user, forbidden access, etc. I've added a check for the HTTP header `X-Error-Detail-Header` in order to distinguish between legit 403 and 429 error messages Possible values for `X-Error-Detail-Header` in a 403 error: o Service Requires SSL : http is used instead of https (secure) o Invalid Referer : invalid 'Referer' header value is send to https://api.tomtom.com and allowed referer values are configured on specific API key o Account Over Queries Per Second Limit : rate limit exceeded o Account Inactive : incorrect API key/API key no longer valid --- .../python/cartodb_services/cartodb_services/tools/qps.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/qps.py b/server/lib/python/cartodb_services/cartodb_services/tools/qps.py index 24d110e..9ae3abd 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/qps.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/qps.py @@ -6,6 +6,9 @@ from exceptions import TimeoutException DEFAULT_RETRY_TIMEOUT = 60 DEFAULT_QUERIES_PER_SECOND = 10 +TOMTOM_403_RATE_LIMIT_HEADER = 'Account Over Queries Per Second Limit' +TOMTOM_DETAIL_HEADER = 'X-Error-Detail-Header' + def qps_retry(original_function=None, **options): """ Query Per Second retry decorator @@ -46,6 +49,8 @@ class QPSService: response = getattr(e, 'response', None) if response is not None: if self._provider is not None and self._provider == 'tomtom' and (response.status_code == 403): + if response.headers.get(TOMTOM_DETAIL_HEADER) != TOMTOM_403_RATE_LIMIT_HEADER: + raise e self.retry(start_time, attempt_number) elif response.status_code == 429: self.retry(start_time, attempt_number)