diff --git a/server/extension/sql/0.1.0/20_geocode_street.sql b/server/extension/sql/0.1.0/20_geocode_street.sql index 3063c94..3466564 100644 --- a/server/extension/sql/0.1.0/20_geocode_street.sql +++ b/server/extension/sql/0.1.0/20_geocode_street.sql @@ -15,11 +15,23 @@ RETURNS Geometry AS $$ if not quota_service.check_user_quota(): plpy.error('You have reach the limit of your quota') - geocoder = heremapsgeocoder.Geocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code) - results = geocoder.geocode_address(searchtext=searchtext, city=city, state=state_province, country=country) - coordinates = geocoder.extract_lng_lat_from_result(results[0]) - plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) - point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + try: + geocoder = heremapsgeocoder.Geocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code) + results = geocoder.geocode_address(searchtext=searchtext, city=city, state=state_province, country=country) + coordinates = geocoder.extract_lng_lat_from_result(results[0]) + quota_service.increment_success_geocoder_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + except heremapsgeocoder.EmptyGeocoderResponse: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using here maps geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) - return point['st_setsrid'] $$ LANGUAGE plpythonu; diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py b/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py index 061ffb3..a2dadc8 100644 --- a/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py +++ b/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py @@ -35,17 +35,25 @@ class QuotaService: else: return False - def increment_successful_geocoder_use(self, amount=1): + def increment_success_geocoder_use(self, amount=1): self._user_service.increment_service_use( self._user_geocoder_config.service_type, "success_responses" ) + self.increment_total_geocoder_use(amount) def increment_empty_geocoder_use(self, amount=1): self._user_service.increment_service_use( self._user_geocoder_config.service_type, "empty_responses" ) + self.increment_total_geocoder_use(amount) def increment_failed_geocoder_use(self, amount=1): self._user_service.increment_service_use( - self._user_geocoder_config.service_type, "failed_responses" + self._user_geocoder_config.service_type, "fail_responses" + ) + self.increment_total_geocoder_use(amount) + + def increment_total_geocoder_use(self, amount=1): + self._user_service.increment_service_use( + self._user_geocoder_config.service_type, "total_requests" ) diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py b/server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py index cdb72dd..fea0ab3 100644 --- a/server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py +++ b/server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py @@ -53,12 +53,12 @@ class UserService: def __increment_user_uses(self, service_type, metric, date, amount): redis_prefix = self.__parse_redis_prefix("user", self._username, service_type, metric, date) - self._redis_connection.hincrby(redis_prefix, date.day, amount) + self._redis_connection.zincrby(redis_prefix, date.day, amount) def __increment_organization_uses(self, service_type, metric, date, amount): redis_prefix = self.__parse_redis_prefix("org", self._orgname, service_type, metric, date) - self._redis_connection.hincrby(redis_prefix, date.day, amount) + self._redis_connection.zincrby(redis_prefix, date.day, amount) def __parse_redis_prefix(self, prefix, entity_name, service_type, metric, date): yearmonth_key = date.strftime('%Y%m') diff --git a/server/lib/python/heremaps/heremaps/heremapsexceptions.py b/server/lib/python/heremaps/heremaps/heremapsexceptions.py index 6416872..cb10713 100644 --- a/server/lib/python/heremaps/heremaps/heremapsexceptions.py +++ b/server/lib/python/heremaps/heremaps/heremapsexceptions.py @@ -1,22 +1,26 @@ #!/usr/local/bin/python # -*- coding: utf-8 -*- - import json + class BadGeocodingParams(Exception): def __init__(self, value): self.value = value + def __str__(self): - return repr('Bad geocoding params: ' + json.dumps(value)) + return repr('Bad geocoding params: ' + json.dumps(self.value)) + class NoGeocodingParams(Exception): def __str__(self): return repr('No params for geocoding specified') + class EmptyGeocoderResponse(Exception): def __str__(self): return repr('The request could not be geocoded') + class MalformedResult(Exception): - def __str__(self): - return repr('Result structure is malformed') + def __str__(self): + return repr('Result structure is malformed') diff --git a/server/lib/python/heremaps/heremaps/heremapsgeocoder.py b/server/lib/python/heremaps/heremaps/heremapsgeocoder.py index 696c79f..dd4bdb8 100644 --- a/server/lib/python/heremaps/heremaps/heremapsgeocoder.py +++ b/server/lib/python/heremaps/heremaps/heremapsgeocoder.py @@ -2,17 +2,19 @@ # -*- coding: utf-8 -*- import json -import urllib +import requests from heremaps.heremapsexceptions import BadGeocodingParams from heremaps.heremapsexceptions import EmptyGeocoderResponse from heremaps.heremapsexceptions import NoGeocodingParams from heremaps.heremapsexceptions import MalformedResult + class Geocoder: 'A Here Maps Geocoder wrapper for python' URL_GEOCODE_JSON = 'http://geocoder.api.here.com/6.2/geocode.json' + URL_DEV_GEOCODE_JSON = 'http://localhost:6083/geocode.json' DEFAULT_MAXRESULTS = 1 DEFAULT_GEN = 9 @@ -52,7 +54,8 @@ class Geocoder: app_code = '' maxresults = '' - def __init__(self, app_id, app_code, maxresults=DEFAULT_MAXRESULTS, gen=DEFAULT_GEN): + def __init__(self, app_id, app_code, maxresults=DEFAULT_MAXRESULTS, + gen=DEFAULT_GEN): self.app_id = app_id self.app_code = app_code self.maxresults = maxresults @@ -61,9 +64,7 @@ class Geocoder: def geocode(self, params): if not set(params.keys()).issubset(set(self.ADDRESS_PARAMS)): raise BadGeocodingParams(params) - response = self.perform_request(params) - try: results = response['Response']['View'][0]['Result'] except IndexError: @@ -73,29 +74,25 @@ class Geocoder: def perform_request(self, params): request_params = { - 'app_id' : self.app_id, - 'app_code' : self.app_code, - 'maxresults' : self.maxresults, - 'gen' : self.gen - } + 'app_id': self.app_id, + 'app_code': self.app_code, + 'maxresults': self.maxresults, + 'gen': self.gen + } request_params.update(params) - - encoded_request_params = urllib.urlencode(request_params) - - response = json.load( - urllib.urlopen(self.URL_GEOCODE_JSON - + '?' - + encoded_request_params)) - - return response + response = requests.get(self.URL_DEV_GEOCODE_JSON, params=request_params) + if response.status_code == requests.codes.ok: + return json.loads(response.text) + else: + response.raise_for_status() def geocode_address(self, **kwargs): params = {} for key, value in kwargs.iteritems(): - if value: params[key] = value - - if not params: raise NoGeocodingParams() - + if value: + params[key] = value + if not params: + raise NoGeocodingParams() return self.geocode(params) def extract_lng_lat_from_result(self, result): diff --git a/server/lib/python/heremaps/requirements.txt b/server/lib/python/heremaps/requirements.txt new file mode 100644 index 0000000..ca0dee4 --- /dev/null +++ b/server/lib/python/heremaps/requirements.txt @@ -0,0 +1 @@ +requests==2.9.1