_bulk_geocode logic extraction

server/lib/python/cartodb_services/cartodb_services/geocoder.py

@@ -51,7 +51,12 @@ StreetGeocoderSearch = namedtuple('StreetGeocoderSearch', 'id address city state
 class StreetPointBulkGeocoder:
     """
     Classes extending StreetPointBulkGeocoder should implement:
-        * _bulk_geocode(street_geocoder_searches)
+        * _batch_geocode(street_geocoder_searches)
+        * MAX_BATCH_SIZE
+
+    If they want to provide an alternative serial (for small batches):
+        * _should_use_batch(street_geocoder_searches)
+        * _serial_geocode(street_geocoder_searches)
     """
 
     SEARCH_KEYS = ['id', 'address', 'city', 'state', 'country']
@@ -77,10 +82,22 @@ class StreetPointBulkGeocoder:
             street_geocoder_searches.append(
                 StreetGeocoderSearch(search_id, address, city, state, country))
 
-        return self._bulk_geocode(street_geocoder_searches)
+        if len(street_geocoder_searches) > self.MAX_BATCH_SIZE:
+            raise Exception("Batch size can't be larger than {}".format(self.MAX_BATCH_SIZE))
+        if self._should_use_batch(street_geocoder_searches):
+            self._logger.debug('--> Batch geocode')
+            return self._batch_geocode(street_geocoder_searches)
+        else:
+            self._logger.debug('--> Serial geocode')
+            return self._serial_geocode(street_geocoder_searches)
+
+    def _batch_geocode(self, street_geocoder_searches):
+        raise NotImplementedError('Subclasses must implement _batch_geocode')
+
+    def _serial_geocode(self, street_geocoder_searches):
+        raise NotImplementedError('Subclasses must implement _serial_geocode')
+
+    def _should_use_batch(self, street_geocoder_searches):
+        return True
+
 
-    def _bulk_geocode(self, street_geocoder_searches):
-        """
-        Subclasses must implement _bulk_geocode
-        """
-        raise NotImplementedError('Subclasses must implement _bulk_geocode')

server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py

@@ -19,16 +19,6 @@ class GoogleMapsBulkGeocoder(GoogleMapsGeocoder, StreetPointBulkGeocoder):
     def __init__(self, client_id, client_secret, logger):
         GoogleMapsGeocoder.__init__(self, client_id, client_secret, logger)
 
-    def _bulk_geocode(self, searches):
-        if len(searches) > self.MAX_BATCH_SIZE:
-            raise Exception("Batch size can't be larger than {}".format(self.MAX_BATCH_SIZE))
-        if self._should_use_batch(searches):
-            self._logger.debug('--> Batch geocode')
-            return self._batch_geocode(searches)
-        else:
-            self._logger.debug('--> Serial geocode')
-            return self._serial_geocode(searches)
-
     def _should_use_batch(self, searches):
         return len(searches) >= self.MIN_BATCHED_SEARCH
 

server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py

@@ -15,13 +15,13 @@ from cartodb_services.tools.exceptions import ServiceException
 HereJobStatus = namedtuple('HereJobStatus', 'total_count processed_count status')
 
 class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder):
-    BATCH_URL = 'https://batch.geocoder.cit.api.here.com/6.2/jobs'
     MAX_BATCH_SIZE = 1000000  # From the docs
+    MIN_BATCHED_SEARCH = 100  # Under this, serial will be used
+    BATCH_URL = 'https://batch.geocoder.cit.api.here.com/6.2/jobs'
     # https://developer.here.com/documentation/batch-geocoder/topics/read-batch-request-output.html
     META_COLS = ['relevance', 'matchType', 'matchCode', 'matchLevel', 'matchQualityStreet']
     MAX_STALLED_RETRIES = 100
     BATCH_RETRY_SLEEP_S = 5
-    MIN_BATCHED_SEARCH = 100  # Under this, serial will be used
     JOB_FINAL_STATES = ['completed', 'cancelled', 'deleted', 'failed']
 
     def __init__(self, app_id, app_code, logger, service_params=None, maxresults=HereMapsGeocoder.DEFAULT_MAXRESULTS):
@@ -34,16 +34,6 @@ class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder):
             'app_code': self.app_code,
         }
 
-    def _bulk_geocode(self, searches):
-        if len(searches) > self.MAX_BATCH_SIZE:
-            raise Exception("Batch size can't be larger than {}".format(self.MAX_BATCH_SIZE))
-        if self._should_use_batch(searches):
-            self._logger.debug('--> Batch geocode')
-            return self._batch_geocode(searches)
-        else:
-            self._logger.debug('--> Serial geocode')
-            return self._serial_geocode(searches)
-
     def _should_use_batch(self, searches):
         return len(searches) >= self.MIN_BATCHED_SEARCH
 

server/lib/python/cartodb_services/cartodb_services/mapbox/bulk_geocoder.py

@@ -21,16 +21,6 @@ class MapboxBulkGeocoder(MapboxGeocoder, StreetPointBulkGeocoder):
         self.max_retries = service_params.get('max_retries', self.MAX_RETRIES)
         self.session = requests.Session()
 
-    def _bulk_geocode(self, searches):
-        if len(searches) > self.MAX_BATCH_SIZE:
-            raise Exception("Batch size can't be larger than {}".format(self.MAX_BATCH_SIZE))
-        if self._should_use_batch(searches):
-            self._logger.debug('--> Batch geocode')
-            return self._batch_geocode(searches)
-        else:
-            self._logger.debug('--> Serial geocode')
-            return self._serial_geocode(searches)
-
     def _should_use_batch(self, searches):
         return len(searches) >= self.MIN_BATCHED_SEARCH
 

server/lib/python/cartodb_services/cartodb_services/tomtom/bulk_geocoder.py

@@ -6,12 +6,12 @@ from cartodb_services.tools.exceptions import ServiceException
 
 
 class TomTomBulkGeocoder(TomTomGeocoder, StreetPointBulkGeocoder):
+    MAX_BATCH_SIZE = 1000000  # From the docs
+    MIN_BATCHED_SEARCH = 10  # Batch API is really fast
     BASE_URL = 'https://api.tomtom.com'
     BATCH_URL = BASE_URL + '/search/2/batch.json'
-    MAX_BATCH_SIZE = 1000000  # From the docs
     MAX_STALLED_RETRIES = 100
     BATCH_RETRY_SLEEP_S = 5
-    MIN_BATCHED_SEARCH = 10  # Batch API is really fast
     READ_TIMEOUT = 60
     CONNECT_TIMEOUT = 10
     MAX_RETRIES = 1
@@ -26,16 +26,6 @@ class TomTomBulkGeocoder(TomTomGeocoder, StreetPointBulkGeocoder):
         self.session.mount(self.BATCH_URL,
                            HTTPAdapter(max_retries=self.max_retries))
 
-    def _bulk_geocode(self, searches):
-        if len(searches) > self.MAX_BATCH_SIZE:
-            raise Exception("Batch size can't be larger than {}".format(self.MAX_BATCH_SIZE))
-        if self._should_use_batch(searches):
-            self._logger.debug('--> Batch geocode')
-            return self._batch_geocode(searches)
-        else:
-            self._logger.debug('--> Serial geocode')
-            return self._serial_geocode(searches)
-
     def _should_use_batch(self, searches):
         return len(searches) >= self.MIN_BATCHED_SEARCH