Merge pull request #551 from CartoDB/tomtom-bulk-fix

Fix altered public method of TomTomGeocoder

NEWS.md

@@ -1,3 +1,18 @@
+Mar 13rd, 2019
+==============
+* Version `0.21.4` of the python library
+    * Fix TomTom bulk geocoder bug (#551)
+
+Mar 5th, 2019
+==============
+* Version `0.21.3` of the python library
+    * Fixed TomTom Qps respondes part 2 (#546)
+
+Mar 4th, 2019
+==============
+* Version `0.21.2` of the python library
+    * Fixed TomTom Qps responses (#546)
+
 Feb 25th, 2019
 ==============
 * Version `0.26.2` of the client extension

server/lib/python/cartodb_services/cartodb_services/tomtom/geocoder.py

@@ -73,15 +73,19 @@ class TomTomGeocoder(Traceable):
     @qps_retry(qps=5, provider='tomtom')
     def geocode(self, searchtext, city=None, state_province=None,
                 country=None):
-        response = self.geocode_meta(searchtext, city, state_province, country)
+        geocoder_response, http_response = self._geocode_meta(searchtext, city, state_province, country)
-        error_message = response[1].get('error', None)
+        error_message = geocoder_response[1].get('error', None)
         if error_message:
-            raise ServiceException(error_message, None)
+            raise ServiceException(error_message, http_response)
         else:
-            return response[0]
+            return geocoder_response[0]
+
+    def geocode_meta(self, searchtext, city=None, state_province=None,
+                country=None):
+        return self._geocode_meta(searchtext, city, state_province, country)[0]
 
     @qps_retry(qps=5, provider='tomtom')
-    def geocode_meta(self, searchtext, city=None, state_province=None,
+    def _geocode_meta(self, searchtext, city=None, state_province=None,
                 country=None):
         if searchtext:
             searchtext = searchtext.decode('utf-8')
@@ -93,7 +97,7 @@ class TomTomGeocoder(Traceable):
             country = country.decode('utf-8')
 
         if not self._validate_input(searchtext, city, state_province, country):
-            return EMPTY_RESPONSE
+            return (EMPTY_RESPONSE, None)
 
         address = []
         if searchtext and searchtext.strip():
@@ -107,18 +111,18 @@ class TomTomGeocoder(Traceable):
 
         try:
             response = requests.get(uri)
-            return self._parse_response(response.status_code, response.text)
+            return (self._parse_response(response.status_code, response.text), response)
         except requests.Timeout as te:
             # In case of timeout we want to stop the job because the server
             # could be down
             msg = 'Timeout connecting to TomTom geocoding server'
             self._logger.error(msg, te)
-            return geocoder_error_response(msg)
+            return (geocoder_error_response(msg), None)
         except requests.ConnectionError as ce:
             # Don't raise the exception to continue with the geocoding job
             self._logger.error('Error connecting to TomTom geocoding server',
                                exception=ce)
-            return EMPTY_RESPONSE
+            return (EMPTY_RESPONSE, None)
 
     def _parse_response(self, status_code, text):
         if status_code == requests.codes.ok:

server/lib/python/cartodb_services/cartodb_services/tools/qps.py

@@ -2,13 +2,17 @@ import time
 import random
 from datetime import datetime
 from exceptions import TimeoutException
+import re
 
 DEFAULT_RETRY_TIMEOUT = 60
 DEFAULT_QUERIES_PER_SECOND = 10
 
-TOMTOM_403_RATE_LIMIT_HEADER = 'Account Over Queries Per Second Limit'
+TOMTOM_403_RATE_LIMIT_HEADERS = [
+    'Account Over Queries Per Second Limit',
+    'Developer Over Qps'
+]
 TOMTOM_DETAIL_HEADER = 'X-Error-Detail-Header'
-
+TOMTOM_403_RATE_LIMIT_HEADER_PATTERN = re.compile('|'.join(TOMTOM_403_RATE_LIMIT_HEADERS), re.IGNORECASE)
 
 def qps_retry(original_function=None, **options):
     """ Query Per Second retry decorator
@@ -49,9 +53,11 @@ class QPSService:
                 response = getattr(e, 'response', None)
                 if response is not None:
                     if self._provider is not None and self._provider == 'tomtom' and (response.status_code == 403):
-                        if response.headers.get(TOMTOM_DETAIL_HEADER) != TOMTOM_403_RATE_LIMIT_HEADER:
+                        detail_header = response.headers.get(TOMTOM_DETAIL_HEADER)
+                        if detail_header and TOMTOM_403_RATE_LIMIT_HEADER_PATTERN.search(detail_header):
+                            self.retry(start_time, attempt_number)
+                        else:
                             raise e
-                        self.retry(start_time, attempt_number)
                     elif response.status_code == 429:
                         self.retry(start_time, attempt_number)
                     else:

server/lib/python/cartodb_services/setup.py

@@ -10,7 +10,7 @@ from setuptools import setup, find_packages
 setup(
     name='cartodb_services',
 
-    version='0.21.1',
+    version='0.21.4',
 
     description='CartoDB Services API Python Library',