Relevance metadata for HERE

server/lib/python/cartodb_services/cartodb_services/geocoder.py

@@ -22,12 +22,18 @@ def run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, org
         if geocode_results:
             results = []
             for result in geocode_results:
+                if len(result) > 2:
+                    metadata = json.dumps(result[2])
+                else:
+                    logger.warning('Geocoding for {} without metadata'.format(username))
+                    metadata = '{}'
+
                 if result[1] and len(result[1]) == 2:
                     plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326) as the_geom; ", ["double precision", "double precision"])
                     point = plpy.execute(plan, result[1], 1)[0]
-                    results.append([result[0], point['the_geom'], None])
+                    results.append([result[0], point['the_geom'], metadata])
                 else:
-                    results.append([result[0], None, None])
+                    results.append([result[0], None, metadata])
             service_manager.quota_service.increment_success_service_use(len(results))
             return results
         else:

server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py

@@ -16,7 +16,7 @@ HereJobStatus = namedtuple('HereJobStatus', 'total_count processed_count status'
 
 class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder):
     MAX_BATCH_SIZE = 1000000  # From the docs
-    MIN_BATCHED_SEARCH = 1000  # Under this, serial will be used
+    MIN_BATCHED_SEARCH = 100  # Under this, serial will be used
     BATCH_URL = 'https://batch.geocoder.cit.api.here.com/6.2/jobs'
     # https://developer.here.com/documentation/batch-geocoder/topics/read-batch-request-output.html
     META_COLS = ['relevance', 'matchType', 'matchCode', 'matchLevel', 'matchQualityStreet']
@@ -41,8 +41,8 @@ class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder):
         results = []
         for search in searches:
             (search_id, address, city, state, country) = search
-            coordinates = self.geocode(searchtext=address, city=city, state=state, country=country)
-            results.append((search_id, coordinates, []))
+            result = self.geocode_meta(searchtext=address, city=city, state=state, country=country)
+            results.append((search_id, result[0], result[1]))
         return results
 
     def _batch_geocode(self, searches):
@@ -140,7 +140,11 @@ class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder):
                 reader = csv.DictReader(root_zip.open(name), delimiter='|')
                 for row in reader:
                     if row['SeqNumber'] == '1':  # First per requested data
-                        results.append((row['recId'], [row['displayLongitude'], row['displayLatitude']]))
+                        results.append((row['recId'],
+                                        [row['displayLongitude'], row['displayLatitude']],
+                                        {
+                                            'relevance': float(row['relevance'])
+                                        }))
 
         return results
 

server/lib/python/cartodb_services/cartodb_services/here/geocoder.py

@@ -65,12 +65,15 @@ class HereMapsGeocoder(Traceable):
         self.max_retries = service_params.get('max_retries', self.MAX_RETRIES)
 
     def geocode(self, **kwargs):
+        return self.geocode_meta(**kwargs)[0]
+
+    def geocode_meta(self, **kwargs):
         params = {}
         for key, value in kwargs.iteritems():
             if value and value.strip():
                 params[key] = value
         if not params:
-            return []
+            return [[], {}]
         return self._execute_geocode(params)
 
     def _execute_geocode(self, params):
@@ -78,10 +81,11 @@ class HereMapsGeocoder(Traceable):
             raise BadGeocodingParams(params)
         try:
             response = self._perform_request(params)
-            results = response['Response']['View'][0]['Result'][0]
-            return self._extract_lng_lat_from_result(results)
+            result = response['Response']['View'][0]['Result'][0]
+            return [self._extract_lng_lat_from_result(result),
+                    self._extract_metadata_from_result(result)]
         except IndexError:
-            return []
+            return [[], {}]
         except KeyError:
             raise MalformedResult()
 
@@ -118,3 +122,8 @@ class HereMapsGeocoder(Traceable):
         latitude = location['DisplayPosition']['Latitude']
 
         return [longitude, latitude]
+
+    def _extract_metadata_from_result(self, result):
+        return {
+            'relevance': result['Relevance']
+        }