Added geocodio geocoder to the Python library

2020-01-14 13:25:44 +01:00
parent 7a08cdf096
commit 1e0a8fae06
11 changed files with 596 additions and 2 deletions
@@ -2,10 +2,12 @@ from google import GoogleMapsBulkGeocoder
 from here import HereMapsBulkGeocoder
 from tomtom import TomTomBulkGeocoder
 from mapbox import MapboxBulkGeocoder
+from geocodio import GeocodioBulkGeocoder

 BATCH_GEOCODER_CLASS_BY_PROVIDER = {
    'google': GoogleMapsBulkGeocoder,
    'heremaps': HereMapsBulkGeocoder,
    'tomtom': TomTomBulkGeocoder,
-    'mapbox': MapboxBulkGeocoder
+    'mapbox': MapboxBulkGeocoder,
+    'geocodio': GeocodioBulkGeocoder,
 }
@@ -0,0 +1,2 @@
+from geocoder import GeocodioGeocoder
+from bulk_geocoder import GeocodioBulkGeocoder
@@ -0,0 +1,76 @@
+import requests
+from cartodb_services import StreetPointBulkGeocoder
+from cartodb_services.geocodio import GeocodioGeocoder
+from iso3166 import countries
+from cartodb_services.tools.country import country_to_iso3
+
+
+class GeocodioBulkGeocoder(GeocodioGeocoder, StreetPointBulkGeocoder):
+    MAX_BATCH_SIZE = 100  # Setting an upper limit (not stated in the documentation)
+    MIN_BATCHED_SEARCH = 0
+    READ_TIMEOUT = 60
+    CONNECT_TIMEOUT = 10
+    MAX_RETRIES = 1
+
+    def __init__(self, token, logger, service_params=None):
+        GeocodioGeocoder.__init__(self, token, logger, service_params)
+
+        self.connect_timeout = self.CONNECT_TIMEOUT
+        self.read_timeout = self.READ_TIMEOUT
+        self.max_retries = self.MAX_RETRIES
+
+        if service_params is not None:
+            self.connect_timeout = service_params.get('connect_timeout', self.CONNECT_TIMEOUT)
+            self.read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT)
+            self.max_retries = service_params.get('max_retries', self.MAX_RETRIES)
+
+        self.session = requests.Session()
+
+    def _should_use_batch(self, searches):
+        return len(searches) >= self.MIN_BATCHED_SEARCH
+
+    def _serial_geocode(self, searches):
+        results = []
+        for search in searches:
+            elements = self._encoded_elements(search)
+            result = self.geocode_meta(*elements)
+
+            if result:
+                results.append((search[0], result[0], result[1]))
+            else:
+                results.append((search[0], None, None))
+
+        return results
+
+    def _encoded_elements(self, search):
+        (search_id, address, city, state, country) = search
+        address = address.encode('utf-8') if address else None
+        city = city.encode('utf-8') if city else None
+        state = state.encode('utf-8') if state else None
+        country = self._country_code(country) if country else None
+        return address, city, state, country
+
+    def _batch_geocode(self, searches):
+        if len(searches) == 1:
+            return self._serial_geocode(searches)
+        else:
+            frees = []
+            for search in searches:
+                elements = self._encoded_elements(search)
+                free = ', '.join([elem for elem in elements if elem])
+                frees.append(free)
+
+            full_results = self.geocode_free_text_meta(frees)
+            results = []
+            for s, r in zip(searches, full_results):
+                results.append((s[0], r[0], r[1]))
+            return results
+
+    def _country_code(self, country):
+        country_iso3166 = None
+        country_iso3 = country_to_iso3(country)
+        if country_iso3:
+            country_iso3166 = countries.get(country_iso3).alpha2.lower()
+
+        return country_iso3166
+
@@ -0,0 +1,125 @@
+from geocodio import GeocodioClient
+from geocodio.exceptions import GeocodioAuthError, GeocodioServerError, GeocodioDataError, GeocodioError
+
+from cartodb_services.tools.qps import qps_retry
+from cartodb_services.metrics import Traceable
+from cartodb_services.geocoder import EMPTY_RESPONSE, geocoder_metadata
+from cartodb_services.tools.exceptions import ServiceException
+
+
+RELEVANCE_BY_LOCATION_TYPE = {
+    'rooftop': 1,
+    'point': 0.9,
+    'range_interpolation': 0.8,
+    'nearest_rooftop_match': 0.7,
+    'intersection': 0.6,
+    'street_center': 0.5,
+    'place': 0.4,
+    'state': 0.1,
+}
+
+
+class GeocodioGeocoder(Traceable):
+    '''
+    Python wrapper for the Geocodio Geocoder service.
+    '''
+
+    def __init__(self, token, logger, service_params=None):
+        service_params = service_params or {}
+        self._token = token
+        self._logger = logger
+
+        self._geocoder = GeocodioClient(self._token)
+
+    def _validate_input(self, searchtext, city=None, state_province=None,
+                        country=None):
+        if searchtext and searchtext.strip():
+            return True
+        elif city:
+            return True
+        elif state_province:
+            return True
+
+        return False
+
+    @qps_retry(qps=15, provider='geocodio')
+    def geocode(self, searchtext, city=None, state_province=None,
+                country=None):
+        return self._geocode_meta(searchtext, city, state_province, country)[0]
+
+    def geocode_meta(self, searchtext, city=None, state_province=None,
+                     country=None):
+        return self._geocode_meta(searchtext, city, state_province, country)[0]
+
+    @qps_retry(qps=15, provider='geocodio')
+    def _geocode_meta(self, searchtext, city=None, state_province=None,
+                      country=None):
+        if not self._validate_input(searchtext, city, state_province, country):
+            return EMPTY_RESPONSE
+
+        try:
+            free_text_components = [searchtext, city, state_province, country]
+            response = self._geocoder.geocode(';'.join([c for c in free_text_components if c is not None and c.strip()]))
+
+            return self._parse_geocoder_response(response)
+        except GeocodioDataError as gde:
+            return EMPTY_RESPONSE
+        except GeocodioAuthError as gae:
+            raise ServiceException('Geocodio authorization error: ' + str(gae), None)
+        except GeocodioServerError as gse:
+            raise ServiceException('geocodio server error: ' + str(gse), None)
+        except GeocodioError as ge:
+            raise ServiceException('Unknown Geocodio error: ' + str(ge), None)
+
+    @qps_retry(qps=15)
+    def geocode_free_text_meta(self, free_searches, country=None):
+        """
+        :param free_searches: Free text searches
+        :return: list of [x, y] on success, [] on error
+        """
+        output = []
+
+        try:
+            if country:
+                free_searches = ['{s}, {country}'.format(s, country) for s in free_searches]
+
+            responses = self._geocoder.geocode(free_searches)
+
+            for response in responses:
+                output.append(self._parse_geocoder_response(response))
+        except GeocodioDataError as gde:
+            return EMPTY_RESPONSE
+        except GeocodioAuthError as gae:
+            raise ServiceException('Geocodio authorization error: ' + str(gae), None)
+        except GeocodioServerError as gse:
+            raise ServiceException('geocodio server error: ' + str(gse), None)
+        except GeocodioError as ge:
+            raise ServiceException('Unknown Geocodio error: ' + str(ge), None)
+
+        return output
+
+    def _parse_geocoder_response(self, response):
+        if response is None or not response:
+            return EMPTY_RESPONSE
+
+        if response.get('results') is None or not response.get('results'):
+            return EMPTY_RESPONSE
+
+        if response.coords is None or not response.coords:
+            return EMPTY_RESPONSE
+
+        coords = [None, None]
+        accuracy = None
+        accuracy_type = None
+
+        accuracy = response.accuracy
+
+        if response.coords is not None and response.coords:
+            coords = [response.coords[1], response.coords[0]]
+
+        if response.get('results'):
+            accuracy_type = response.get('results')[0].get('accuracy_type')
+
+        metadata = geocoder_metadata(RELEVANCE_BY_LOCATION_TYPE.get(accuracy_type), response.accuracy, accuracy_type)
+
+        return [coords, metadata]
@@ -0,0 +1 @@
+GEOCODIO_GEOCODER_APIKEY_ROUNDROBIN = 'geocodio_geocoder_apikey_roundrobin'
@@ -405,6 +405,8 @@ class GeocoderConfig(ServiceConfig):
    MAPBOX_GEOCODER_API_KEYS = 'mapbox_geocoder_api_keys'
    TOMTOM_GEOCODER = 'tomtom'
    TOMTOM_GEOCODER_API_KEYS = 'tomtom_geocoder_api_keys'
+    GEOCODIO_GEOCODER = 'geocodio'
+    GEOCODIO_GEOCODER_API_KEYS = 'geocodio_geocoder_api_keys'
    QUOTA_KEY = 'geocoding_quota'
    SOFT_LIMIT_KEY = 'soft_geocoding_limit'
    USERNAME_KEY = 'username'
@@ -437,6 +439,9 @@ class GeocoderConfig(ServiceConfig):
        elif self._geocoder_provider == self.TOMTOM_GEOCODER:
            if not self.tomtom_api_keys:
                raise ConfigException("""TomTom config is not set up""")
+        elif self._geocoder_provider == self.GEOCODIO_GEOCODER:
+            if not self.geocodio_api_keys:
+                raise ConfigException("""Geocodio config is not set up""")

        return True

@@ -476,6 +481,10 @@ class GeocoderConfig(ServiceConfig):
            self._tomtom_api_keys = db_config.tomtom_geocoder_api_keys
            self._cost_per_hit = 0
            self._tomtom_service_params = db_config.tomtom_geocoder_service_params
+        elif self._geocoder_provider == self.GEOCODIO_GEOCODER:
+            self._geocodio_api_keys = db_config.geocodio_geocoder_api_keys
+            self._cost_per_hit = 0
+            self._geocodio_service_params = db_config.geocodio_geocoder_service_params

    @property
    def service_type(self):
@@ -489,6 +498,8 @@ class GeocoderConfig(ServiceConfig):
            return 'geocoder_tomtom'
        elif self._geocoder_provider == self.NOKIA_GEOCODER:
            return 'geocoder_here'
+        elif self._geocoder_provider == self.GEOCODIO_GEOCODER:
+            return 'geocoder_geocodio'

    @property
    def heremaps_geocoder(self):
@@ -510,6 +521,10 @@ class GeocoderConfig(ServiceConfig):
    def tomtom_geocoder(self):
        return self._geocoder_provider == self.TOMTOM_GEOCODER

+    @property
+    def geocodio_geocoder(self):
+        return self._geocoder_provider == self.GEOCODIO_GEOCODER
+
    @property
    def google_client_id(self):
        return self._google_maps_client_id
@@ -569,6 +584,14 @@ class GeocoderConfig(ServiceConfig):
    def tomtom_service_params(self):
        return self._tomtom_service_params

+    @property
+    def geocodio_api_keys(self):
+        return self._geocodio_api_keys
+
+    @property
+    def geocodio_service_params(self):
+        return self._geocodio_service_params
+
    @property
    def is_high_resolution(self):
        return True
@@ -600,6 +623,7 @@ class ServicesDBConfig:
        self._get_mapzen_config()
        self._get_mapbox_config()
        self._get_tomtom_config()
+        self._get_geocodio_config()
        self._get_data_observatory_config()

    def _get_server_config(self):
@@ -679,6 +703,16 @@ class ServicesDBConfig:
            self._tomtom_geocoder_quota = tomtom_conf['geocoder']['monthly_quota']
            self._tomtom_geocoder_service_params = tomtom_conf['geocoder'].get('service', {})

+    def _get_geocodio_config(self):
+        geocodio_conf_json = self._get_conf('geocodio_conf')
+        if not geocodio_conf_json:
+            raise ConfigException('Geocodio configuration missing')
+        else:
+            geocodio_conf = json.loads(geocodio_conf_json)
+            self._geocodio_geocoder_api_keys = geocodio_conf['geocoder']['api_keys']
+            self._geocodio_geocoder_quota = geocodio_conf['geocoder']['monthly_quota']
+            self._geocodio_geocoder_service_params = geocodio_conf['geocoder'].get('service', {})
+
    def _get_data_observatory_config(self):
        do_conf_json = self._get_conf('data_observatory_conf')
        if not do_conf_json:
@@ -848,6 +882,18 @@ class ServicesDBConfig:
    def tomtom_geocoder_service_params(self):
        return self._tomtom_geocoder_service_params

+    @property
+    def geocodio_geocoder_api_keys(self):
+        return self._geocodio_geocoder_api_keys
+
+    @property
+    def geocodio_geocoder_monthly_quota(self):
+        return self.geocodio_geocoder_quota
+
+    @property
+    def geocodio_geocoder_service_params(self):
+        return self._geocodio_geocoder_service_params
+
    @property
    def data_observatory_connection_str(self):
        return self._data_observatory_connection_str
@@ -0,0 +1,128 @@
+from dateutil.parser import parse as date_parse
+from cartodb_services.refactor.service.utils import round_robin
+from cartodb_services.geocodio.types import GEOCODIO_GEOCODER_APIKEY_ROUNDROBIN
+
+
+class GeocodioGeocoderConfig(object):
+    """
+    Configuration needed to operate the Geocodio geocoder service.
+    """
+
+    def __init__(self,
+                 geocoding_quota,
+                 soft_geocoding_limit,
+                 period_end_date,
+                 cost_per_hit,
+                 log_path,
+                 geocodio_api_keys,
+                 username,
+                 organization,
+                 service_params,
+                 GD):
+        self._geocoding_quota = geocoding_quota
+        self._soft_geocoding_limit = soft_geocoding_limit
+        self._period_end_date = period_end_date
+        self._cost_per_hit = cost_per_hit
+        self._log_path = log_path
+        self._geocodio_api_keys = geocodio_api_keys
+        self._username = username
+        self._organization = organization
+        self._service_params = service_params
+        self._GD = GD
+
+    @property
+    def service_type(self):
+        return 'geocoder_geocodio'
+
+    @property
+    def provider(self):
+        return 'geocodio'
+
+    @property
+    def is_high_resolution(self):
+        return True
+
+    @property
+    def geocoding_quota(self):
+        return self._geocoding_quota
+
+    @property
+    def soft_geocoding_limit(self):
+        return self._soft_geocoding_limit
+
+    @property
+    def period_end_date(self):
+        return self._period_end_date
+
+    @property
+    def cost_per_hit(self):
+        return self._cost_per_hit
+
+    @property
+    def log_path(self):
+        return self._log_path
+
+    @property
+    def geocodio_api_key(self):
+        return round_robin(self._geocodio_api_keys, self._GD,
+                           GEOCODIO_GEOCODER_APIKEY_ROUNDROBIN)
+
+    @property
+    def username(self):
+        return self._username
+
+    @property
+    def organization(self):
+        return self._organization
+
+    @property
+    def service_params(self):
+        return self._service_params
+
+    # TODO: for BW compat, remove
+    @property
+    def google_geocoder(self):
+        return False
+
+
+class GeocodioGeocoderConfigBuilder(object):
+
+    def __init__(self, server_conf, user_conf, org_conf, username, orgname, GD):
+        self._server_conf = server_conf
+        self._user_conf = user_conf
+        self._org_conf = org_conf
+        self._username = username
+        self._orgname = orgname
+        self._GD = GD
+
+    def get(self):
+        geocodio_server_conf = self._server_conf.get('geocodio_conf')
+        geocodio_api_keys = geocodio_server_conf['geocoder']['api_keys']
+        geocodio_service_params = geocodio_server_conf['geocoder'].get('service', {})
+
+        geocoding_quota = self._get_quota()
+        soft_geocoding_limit = self._user_conf.get('soft_geocoding_limit').lower() == 'true'
+        cost_per_hit = 0
+        period_end_date_str = self._org_conf.get('period_end_date') or self._user_conf.get('period_end_date')
+        period_end_date = date_parse(period_end_date_str)
+
+        logger_conf = self._server_conf.get('logger_conf')
+        log_path = logger_conf.get('geocoder_log_path', None)
+
+        return GeocodioGeocoderConfig(geocoding_quota,
+                                    soft_geocoding_limit,
+                                    period_end_date,
+                                    cost_per_hit,
+                                    log_path,
+                                    geocodio_api_keys,
+                                    self._username,
+                                    self._orgname,
+                                    geocodio_service_params,
+                                    self._GD)
+
+    def _get_quota(self):
+        geocoding_quota = self._org_conf.get('geocoding_quota') or self._user_conf.get('geocoding_quota')
+        if geocoding_quota is '':
+            return 0
+
+        return int(geocoding_quota)
				`@@ -0,0 +1 @@`
				`GEOCODIO_GEOCODER_APIKEY_ROUNDROBIN = 'geocodio_geocoder_apikey_roundrobin'`