diff --git a/server/lib/python/cartodb_services/cartodb_services/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/geocoder.py index c25216c..cea8ef4 100644 --- a/server/lib/python/cartodb_services/cartodb_services/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/geocoder.py @@ -22,12 +22,18 @@ def run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, org if geocode_results: results = [] for result in geocode_results: + if len(result) > 2: + metadata = json.dumps(result[2]) + else: + logger.warning('Geocoding for {} without metadata'.format(username)) + metadata = '{}' + if result[1] and len(result[1]) == 2: plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326) as the_geom; ", ["double precision", "double precision"]) point = plpy.execute(plan, result[1], 1)[0] - results.append([result[0], point['the_geom'], None]) + results.append([result[0], point['the_geom'], metadata]) else: - results.append([result[0], None, None]) + results.append([result[0], None, metadata]) service_manager.quota_service.increment_success_service_use(len(results)) return results else: diff --git a/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py b/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py index 3724bcd..dd47fda 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py @@ -16,7 +16,7 @@ HereJobStatus = namedtuple('HereJobStatus', 'total_count processed_count status' class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder): MAX_BATCH_SIZE = 1000000 # From the docs - MIN_BATCHED_SEARCH = 1000 # Under this, serial will be used + MIN_BATCHED_SEARCH = 100 # Under this, serial will be used BATCH_URL = 'https://batch.geocoder.cit.api.here.com/6.2/jobs' # https://developer.here.com/documentation/batch-geocoder/topics/read-batch-request-output.html META_COLS = ['relevance', 'matchType', 'matchCode', 'matchLevel', 'matchQualityStreet'] @@ -41,8 +41,8 @@ class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder): results = [] for search in searches: (search_id, address, city, state, country) = search - coordinates = self.geocode(searchtext=address, city=city, state=state, country=country) - results.append((search_id, coordinates, [])) + result = self.geocode_meta(searchtext=address, city=city, state=state, country=country) + results.append((search_id, result[0], result[1])) return results def _batch_geocode(self, searches): @@ -140,7 +140,11 @@ class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder): reader = csv.DictReader(root_zip.open(name), delimiter='|') for row in reader: if row['SeqNumber'] == '1': # First per requested data - results.append((row['recId'], [row['displayLongitude'], row['displayLatitude']])) + results.append((row['recId'], + [row['displayLongitude'], row['displayLatitude']], + { + 'relevance': float(row['relevance']) + })) return results diff --git a/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py index 6c3b81c..8139e1a 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py @@ -65,12 +65,15 @@ class HereMapsGeocoder(Traceable): self.max_retries = service_params.get('max_retries', self.MAX_RETRIES) def geocode(self, **kwargs): + return self.geocode_meta(**kwargs)[0] + + def geocode_meta(self, **kwargs): params = {} for key, value in kwargs.iteritems(): if value and value.strip(): params[key] = value if not params: - return [] + return [[], {}] return self._execute_geocode(params) def _execute_geocode(self, params): @@ -78,10 +81,11 @@ class HereMapsGeocoder(Traceable): raise BadGeocodingParams(params) try: response = self._perform_request(params) - results = response['Response']['View'][0]['Result'][0] - return self._extract_lng_lat_from_result(results) + result = response['Response']['View'][0]['Result'][0] + return [self._extract_lng_lat_from_result(result), + self._extract_metadata_from_result(result)] except IndexError: - return [] + return [[], {}] except KeyError: raise MalformedResult() @@ -118,3 +122,8 @@ class HereMapsGeocoder(Traceable): latitude = location['DisplayPosition']['Latitude'] return [longitude, latitude] + + def _extract_metadata_from_result(self, result): + return { + 'relevance': result['Relevance'] + } diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index 52ffbe0..e842396 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -2,10 +2,9 @@ # -*- coding: utf-8 -*- from unittest import TestCase -from nose.tools import assert_raises -from nose.tools import assert_not_equal, assert_equal +from nose.tools import assert_not_equal, assert_equal, assert_true from ..helpers.integration_test_helper import IntegrationTestHelper -from ..helpers.integration_test_helper import assert_close_enough +from ..helpers.integration_test_helper import assert_close_enough, isclose class TestStreetFunctionsSetUp(TestCase): provider = None @@ -321,6 +320,16 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp): assert_equal(len(response['rows']), count) assert_not_equal(response['rows'][0]['st_x'], None) + def test_relevance(self): + query = "select metadata " \ + "FROM cdb_dataservices_client.cdb_bulk_geocode_street_point(" \ + "'select 1 as cartodb_id, ''Spain'' as country, " \ + "''Barcelona'' as city, " \ + "''Plaza España 1'' as street' " \ + ", 'street', 'city', NULL, 'country')" + response = self._run_authenticated(query) + + assert_true(isclose(response['rows'][0]['metadata']['relevance'], 1)) def _run_authenticated(self, query): authenticated_query = "{}&api_key={}".format(query,