diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index 3c73087..5e3fc2b 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -12,6 +12,9 @@ RETURNS Geometry AS $$ elif user_geocoder_config.google_geocoder: google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapzen_geocoder: + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] else: plpy.error('Requested geocoder is not available') @@ -82,3 +85,34 @@ RETURNS Geometry AS $$ finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.mapzen import MapzenGeocoder + from cartodb_services.metrics import QuotaService + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + quota_service = QuotaService(user_geocoder_config, redis_conn) + + try: + geocoder = MapzenGeocoder(user_geocoder_config.mapzen_app_key) + coordinates = geocoder.geocode(searchtext=searchtext, country=country) + if coordinates: + quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using mapzen geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; diff --git a/server/extension/test/expected/00_install_test.out b/server/extension/test/expected/00_install_test.out index f4a1873..c69ea04 100644 --- a/server/extension/test/expected/00_install_test.out +++ b/server/extension/test/expected/00_install_test.out @@ -25,7 +25,7 @@ SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"app_id": "dummy_id", "app_co (1 row) -SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing_app_key": "dummy_key"}'); +SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing_app_key": "dummy_key", "geocoder_app_key": "dummy_key"}'); cdb_conf_setconf ------------------ diff --git a/server/extension/test/sql/00_install_test.sql b/server/extension/test/sql/00_install_test.sql index 142f8a3..2e3cb2e 100644 --- a/server/extension/test/sql/00_install_test.sql +++ b/server/extension/test/sql/00_install_test.sql @@ -12,7 +12,7 @@ CREATE EXTENSION cdb_dataservices_server; SELECT cartodb.cdb_conf_setconf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}'); SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}'); SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}'); -SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing_app_key": "dummy_key"}'); +SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing_app_key": "dummy_key", "geocoder_app_key": "dummy_key"}'); SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}'); -- Mock the varnish invalidation function diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/__init__.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/__init__.py index c1a8c1f..ee7552c 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/__init__.py @@ -1 +1,2 @@ from routing import MapzenRouting, MapzenRoutingResponse +from geocoder import MapzenGeocoder diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py new file mode 100644 index 0000000..be98761 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py @@ -0,0 +1,53 @@ +import requests +import json +import re + +from exceptions import WrongParams, MalformedResult +from qps import qps_retry +from cartodb_services.tools import Coordinate, PolyLine + + +class MapzenGeocoder: + 'A Mapzen Geocoder wrapper for python' + + BASE_URL = 'https://search.mapzen.com/v1/search' + + def __init__(self, app_key, base_url=BASE_URL): + self._app_key = app_key + self._url = base_url + + @qps_retry + def geocode(self, searchtext, country=None): + request_params = self._build_requests_parameters(searchtext, country) + response = requests.get(self._url, params=request_params) + if response.status_code == requests.codes.ok: + return self.__parse_response(response.text) + elif response.status_code == requests.codes.bad_request: + return [] + else: + response.raise_for_status() + + def _build_requests_parameters(self, searchtext, country=None): + request_params = {} + request_params['text'] = searchtext + request_params['layers'] = 'address' + request_params['api_key'] = self._app_key + if country: + request_params['boundary.country'] = country + return request_params + + def __parse_response(self, response): + try: + parsed_json_response = json.loads(response) + feature = parsed_json_response['features'][0] + return self._extract_lng_lat_from_result(feature) + except IndexError: + return [] + except KeyError: + raise MalformedResult() + + def _extract_lng_lat_from_result(self, result): + location = result['geometry']['coordinates'] + longitude = location[0] + latitude = location[1] + return [longitude, latitude] diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index 6be0518..a729cb4 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -168,15 +168,17 @@ class GeocoderConfig(ServiceConfig): GEOCODER_CONFIG_KEYS = ['google_maps_client_id', 'google_maps_api_key', 'geocoding_quota', 'soft_geocoding_limit', 'geocoder_type', 'period_end_date', - 'heremaps_app_id', 'heremaps_app_code', 'username', - 'orgname'] - NOKIA_GEOCODER_MANDATORY_KEYS = ['geocoding_quota', 'soft_geocoding_limit'] + 'heremaps_app_id', 'heremaps_app_code', + 'mapzen_geocoder_app_key', 'username', 'orgname'] + NOKIA_GEOCODER_REDIS_MANDATORY_KEYS = ['geocoding_quota', 'soft_geocoding_limit'] NOKIA_GEOCODER = 'heremaps' NOKIA_GEOCODER_APP_ID_KEY = 'heremaps_app_id' NOKIA_GEOCODER_APP_CODE_KEY = 'heremaps_app_code' GOOGLE_GEOCODER = 'google' GOOGLE_GEOCODER_API_KEY = 'google_maps_api_key' GOOGLE_GEOCODER_CLIENT_ID = 'google_maps_client_id' + MAPZEN_GEOCODER = 'mapzen' + MAPZEN_GEOCODER_API_KEY = 'mapzen_geocoder_app_key' GEOCODER_TYPE = 'geocoder_type' QUOTA_KEY = 'geocoding_quota' SOFT_LIMIT_KEY = 'soft_geocoding_limit' @@ -217,11 +219,15 @@ class GeocoderConfig(ServiceConfig): def __check_config(self, filtered_config): if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER: - if not set(self.NOKIA_GEOCODER_MANDATORY_KEYS).issubset(set(filtered_config.keys())): + if not set(self.NOKIA_GEOCODER_REDIS_MANDATORY_KEYS).issubset(set(filtered_config.keys())) or \ + not self.heremaps_app_id or not self.heremaps_app_code: raise ConfigException("""Some mandatory parameter/s for Nokia geocoder are missing. Check it please""") elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER: if self.GOOGLE_GEOCODER_API_KEY not in filtered_config.keys(): raise ConfigException("""Google geocoder need the mandatory parameter 'google_maps_private_key'""") + elif filtered_config[self.GEOCODER_TYPE].lower() == self.MAPZEN_GEOCODER: + if not self.mapzen_app_key: + raise ConfigException("""Mapzen config is not setted up""") return True @@ -242,11 +248,16 @@ class GeocoderConfig(ServiceConfig): self._google_maps_api_key = filtered_config[self.GOOGLE_GEOCODER_API_KEY] self._google_maps_client_id = filtered_config[self.GOOGLE_GEOCODER_CLIENT_ID] self._cost_per_hit = 0 + elif filtered_config[self.GEOCODER_TYPE].lower() == self.MAPZEN_GEOCODER: + self._mapzen_app_key = db_config.mapzen_geocoder_app_key + self._cost_per_hit = 0 @property def service_type(self): if self._geocoder_type == self.GOOGLE_GEOCODER: return 'geocoder_google' + elif self._geocoder_type == self.MAPZEN_GEOCODER: + return 'geocoder_mapzen' else: return 'geocoder_here' @@ -258,6 +269,10 @@ class GeocoderConfig(ServiceConfig): def google_geocoder(self): return self._geocoder_type == self.GOOGLE_GEOCODER + @property + def mapzen_geocoder(self): + return self._geocoder_type == self.MAPZEN_GEOCODER + @property def google_client_id(self): return self._google_maps_client_id @@ -289,6 +304,10 @@ class GeocoderConfig(ServiceConfig): def heremaps_app_code(self): return self._heremaps_app_code + @property + def mapzen_app_key(self): + return self._mapzen_app_key + @property def is_high_resolution(self): return True @@ -331,6 +350,7 @@ class ServicesDBConfig: else: mapzen_conf = json.loads(mapzen_conf_json) self._mapzen_routing_app_key = mapzen_conf['routing_app_key'] + self._mapzen_geocoder_app_key = mapzen_conf['geocoder_app_key'] def _get_logger_config(self): logger_conf_json = self._get_conf('logger_conf') @@ -364,6 +384,10 @@ class ServicesDBConfig: def mapzen_routing_app_key(self): return self._mapzen_routing_app_key + @property + def mapzen_geocoder_app_key(self): + return self._mapzen_geocoder_app_key + @property def geocoder_log_path(self): return self._geocoder_log_path diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index 53f3b95..633f24c 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.3.3', + version='0.4.0', description='CartoDB Services API Python Library', diff --git a/server/lib/python/cartodb_services/test/test_helper.py b/server/lib/python/cartodb_services/test/test_helper.py index cc530c6..9856aca 100644 --- a/server/lib/python/cartodb_services/test/test_helper.py +++ b/server/lib/python/cartodb_services/test/test_helper.py @@ -46,6 +46,6 @@ def _plpy_execute_side_effect(*args, **kwargs): if args[0] == "SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as conf": return [{'conf': '{"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}'}] elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('mapzen_conf') as conf": - return [{'conf': '{"routing_app_key": "app_key"}'}] + return [{'conf': '{"routing_app_key": "app_key", "geocoder_app_key": "app_key"}'}] elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('logger_conf') as conf": return [{'conf': '{"geocoder_log_path": "/dev/null"}'}] diff --git a/server/lib/python/cartodb_services/test/test_mapzengeocoder.py b/server/lib/python/cartodb_services/test/test_mapzengeocoder.py new file mode 100644 index 0000000..0c7be54 --- /dev/null +++ b/server/lib/python/cartodb_services/test/test_mapzengeocoder.py @@ -0,0 +1,109 @@ +#!/usr/local/bin/python +# -*- coding: utf-8 -*- + +import unittest +import requests_mock + +from cartodb_services.mapzen import MapzenGeocoder +from cartodb_services.mapzen.exceptions import MalformedResult + +requests_mock.Mocker.TEST_PREFIX = 'test_' + + +@requests_mock.Mocker() +class GoogleGeocoderTestCase(unittest.TestCase): + MAPZEN_GEOCODER_URL = 'https://search.mapzen.com/v1/search' + + EMPTY_RESPONSE = """{ + "results" : [], + "status" : "ZERO_RESULTS" + }""" + + GOOD_RESPONSE = """{ + "geocoding": { + "version": "0.1", + "attribution": "https://search.mapzen.com/v1/attribution", + "query": { + "text": "Calle siempreviva 3, Valladolid", + "parsed_text": { + "name": "Calle siempreviva 3", + "regions": [ + "Calle siempreviva 3", + "Valladolid" + ], + "admin_parts": "Valladolid" + }, + "types": { + "from_layers": [ + "osmaddress", + "openaddresses" + ] + }, + "size": 10, + "private": false, + "type": [ + "osmaddress", + "openaddresses" + ], + "querySize": 20 + }, + "engine": { + "name": "Pelias", + "author": "Mapzen", + "version": "1.0" + }, + "timestamp": 1458661873749 + }, + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": { + "id": "df7428b955ae44a39dc40d52578f61e3", + "gid": "oa:address:df7428b955ae44a39dc40d52578f61e3", + "layer": "address", + "source": "oa", + "name": "5 Close Siempreviva", + "housenumber": "5", + "street": "Close Siempreviva", + "country_a": "ESP", + "country": "Spain", + "region": "Valladolid", + "localadmin": "Valladolid", + "locality": "Valladolid", + "confidence": 0.887, + "label": "5 Close Siempreviva, Valladolid, Spain" + }, + "geometry": { + "type": "Point", + "coordinates": [ + -4.730928, + 41.669034 + ] + } + } + ] + }""" + + MALFORMED_RESPONSE = """{"manolo": "escobar"}""" + + def setUp(self): + self.geocoder = MapzenGeocoder('search-XXXXXXX') + + def test_geocode_address_with_valid_params(self, req_mock): + req_mock.register_uri('GET', self.MAPZEN_GEOCODER_URL, + text=self.GOOD_RESPONSE) + response = self.geocoder.geocode( + searchtext='Calle Siempreviva 3, Valldolid', + country='ESP') + + self.assertEqual(response[0], -4.730928) + self.assertEqual(response[1], 41.669034) + + def test_geocode_with_malformed_result(self, req_mock): + req_mock.register_uri('GET', self.MAPZEN_GEOCODER_URL, + text=self.MALFORMED_RESPONSE) + with self.assertRaises(MalformedResult): + self.geocoder.geocode( + searchtext='Calle Siempreviva 3, Valladolid', + country='ESP') diff --git a/server/lib/python/cartodb_services/test/test_user_service.py b/server/lib/python/cartodb_services/test/test_user_service.py index 9066643..7f0d1fc 100644 --- a/server/lib/python/cartodb_services/test/test_user_service.py +++ b/server/lib/python/cartodb_services/test/test_user_service.py @@ -87,5 +87,5 @@ class TestUserService(TestCase): quota=quota, end_date=end_date) plpy_mock = test_helper.build_plpy_mock() geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock, - username, orgname,) + username, orgname) return UserMetricsService(geocoder_config, self.redis_conn)