diff --git a/NEWS.md b/NEWS.md index 78f590f..3a68fab 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,14 @@ +March XX, 2017 +=================== +* Version XXXXXX of the server and version XXXXXX of the python library + * New optional configuration parameters for external services can be provided through `cdb_conf`: + - In `heremaps_conf`, under `geocoder.service`: `json_url`, `connect_timeout`, `read_timeout`, `max_retries`, `gen` + - In `heremaps_conf`, under `isolines.service`: `base_url`, `connect_timeout`, `read_timeout`, `max_retries`, `isoline_path` + - In `mapzen_conf`, under `geocoder.service`: `base_url`, `connect_timeout`, `read_timeout`, `max_retries` + - In `mapzen_conf`, under `routing.service`: `base_url`, `connect_timeout`, `read_timeout` + - In `mapzen_conf`, under `matrix.service`: `one_to_many_url`, `connect_timeout`, `read_timeout` + - In `mapzen_conf`, under `isochrones.service`: `base_url`, `connect_timeout`, `read_timeout`, `max_retries` + February 2st, 2017 =================== * Version 0.21.0 of the server and version 0.15.0 of the client diff --git a/README.md b/README.md index a9df9fe..e80f9e8 100644 --- a/README.md +++ b/README.md @@ -23,15 +23,15 @@ Steps to deploy a new Data Services API version : ### Local install instructions -- install data services geocoder extension +- install data services geocoder extension ``` git clone https://github.com/CartoDB/data-services.git cd data-services/geocoder/extension sudo make install ``` - -- install observatory extension + +- install observatory extension ``` git clone https://github.com/CartoDB/observatory-extension.git @@ -40,7 +40,7 @@ Steps to deploy a new Data Services API version : ``` - install server and client extensions - + ``` # in data-services repo root path: cd client && sudo make install @@ -51,7 +51,7 @@ Steps to deploy a new Data Services API version : ``` # in data-services repo root path: - cd server/lib/python/cartodb_services && sudo pip install . --upgrade + cd server/lib/python/cartodb_services && sudo pip install . --upgrade ``` - install extensions in user database @@ -64,39 +64,184 @@ Steps to deploy a new Data Services API version : create extension cdb_dataservices_client; ``` -- add configuration for different services in server database +### Server configuration - ``` - # If sentinel is used: - SELECT CDB_Conf_SetConf('redis_metadata_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); - SELECT CDB_Conf_SetConf('redis_metrics_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); - - # If sentinel is not used - SELECT CDB_Conf_SetConf('redis_metadata_config', '{"redis_host": "localhost", "redis_port": 6379, "sentinel_master_id": "", "timeout": 0.1, "redis_db": 5}'); - SELECT CDB_Conf_SetConf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 6379, "sentinel_master_id": "", "timeout": 0.1, "redis_db": 5}'); - - SELECT CDB_Conf_SetConf('heremaps_conf', '{"geocoder": {"app_id": "here_geocoder_app_id", "app_code": "here_geocoder_app_code", "geocoder_cost_per_hit": "1"}, "isolines" : {"app_id": "here_isolines_app_id", "app_code": "here_geocoder_app_code"}}'); - SELECT CDB_Conf_SetConf('user_config', '{"is_organization": false, "entity_name": ""}') - SELECT CDB_Conf_SetConf('mapzen_conf', '{"routing": {"api_key": "valhalla_app_key", "monthly_quota": 999999}, "geocoder": {"api_key": "search_app_key", "monthly_quota": 999999}, "matrix": {"api_key": "[your_matrix_key]", "monthly_quota": 1500000}}'); - SELECT CDB_Conf_SetConf('logger_conf', '{"geocoder_log_path": "/tmp/geocodings.log", [ "min_log_level": "[debug|info|warning|error]", "rollbar_api_key": "SERVER_SIDE_API_KEY", "log_file_path": "LOG_FILE_PATH"]}'); - SELECT CDB_Conf_SetConf('data_observatory_conf', '{"connection": {"whitelist": [], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}'); +Configuration for the different services must be stored in the server database using `CDB_Conf_SetConf()`. - # Environment to decide: rollbar message, which servers for third party use, etc. If not setted uses production by default (current behavior) - SELECT CDB_Conf_SetConf('server_conf', '{"environment": "[development|staging|production]"}') - ``` +#### Redis configuration -- configure the user DB: +If sentinel is used: - ```sql - -- Point to the dataservices server DB (you can use a specific database for the server or your same user's): - SELECT CDB_Conf_SetConf('geocoder_server_config', '{ "connection_str": "host=localhost port=5432 dbname= user=postgres"}'); +```sql +SELECT CDB_Conf_SetConf( + 'redis_metadata_config', + '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}' +); +SELECT CDB_Conf_SetConf( + 'redis_metrics_config', + '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}' +); +``` - SELECT CDB_Conf_SetConf('user_config', '{"is_organization": false, "entity_name": ""}'); - ``` +If sentinel is not used: -- configure the search path in order to be able to execute the functions without using the schema: +```sql +SELECT CDB_Conf_SetConf( + 'redis_metadata_config', + '{"redis_host": "localhost", "redis_port": 6379, "sentinel_master_id": "", "timeout": 0.1, "redis_db": 5}' +); +SELECT CDB_Conf_SetConf( + 'redis_metrics_config', + '{"redis_host": "localhost", "redis_port": 6379, "sentinel_master_id": "", "timeout": 0.1, "redis_db": 5}' +); +``` - ``` - ALTER ROLE "" SET search_path="$user", public, cartodb, cdb_dataservices_client; - ``` +#### Users/Organizations + +```sql +SELECT CDB_Conf_SetConf( + 'user_config', + '{"is_organization": false, "entity_name": ""}' +); +``` + +#### HERE configuration + +```sql +SELECT CDB_Conf_SetConf( + 'heremaps_conf', + '{"geocoder": {"app_id": "here_geocoder_app_id", "app_code": "here_geocoder_app_code", "geocoder_cost_per_hit": "1"}, "isolines" : {"app_id": "here_isolines_app_id", "app_code": "here_geocoder_app_code"}}' +); +``` + +#### Mapzen configuration + +```sql +SELECT CDB_Conf_SetConf( + 'mapzen_conf', + '{"routing": {"api_key": "valhalla_app_key", "monthly_quota": 999999}, "geocoder": {"api_key": "search_app_key", "monthly_quota": 999999}, "matrix": {"api_key": "[your_matrix_key]", "monthly_quota": 1500000}}' +); +``` + +#### Data Observatory + +```sql +SELECT CDB_Conf_SetConf( + 'data_observatory_conf', + '{"connection": {"whitelist": [], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}' +); +``` + +#### Logger + +```sql +SELECT CDB_Conf_SetConf( + 'logger_conf', + '{"geocoder_log_path": "/tmp/geocodings.log", [ "min_log_level": "[debug|info|warning|error]", "rollbar_api_key": "SERVER_SIDE_API_KEY", "log_file_path": "LOG_FILE_PATH"]}' +); +``` + +#### Environment + +The execution environment (development/staging/production) affects rollbar messages and other details. +The production environment is used by default. + +```sql +SELECT CDB_Conf_SetConf( + 'server_conf', + '{"environment": "[development|staging|production]"}' +); +``` +### Server optional configuration + +External services (Mapzen, Here) can have optional configuration, which is only needed for using non-standard services, such as on-premise installations. We can add the service parameters to an existing configuration like this: + +``` +# Here geocoder +SELECT CDB_Conf_SetConf( +'heremaps_conf', +jsonb_set( + to_jsonb(CDB_Conf_GetConf('heremaps_conf')), + '{geocoder, service}', + '{"json_url":"https://geocoder.api.here.com/6.2/geocode.json","gen":9,"read_timeout":60,"connect_timeout":10,"max_retries":1}' +)::json +); + +# Here isolines +SELECT CDB_Conf_SetConf( +'heremaps_conf', +jsonb_set( + to_jsonb(CDB_Conf_GetConf('heremaps_conf')), + '{isolines, service}', + '{"base_url":"https://isoline.route.api.here.com","isoline_path":"/routing/7.2/calculateisoline.json","read_timeout":60,"connect_timeout":10,"max_retries":1}' +)::json +); + +# Mapzen geocoder +SELECT CDB_Conf_SetConf( +'mapzen_conf', +jsonb_set( + to_jsonb(CDB_Conf_GetConf('mapzen_conf')), + '{geocoder, service}', + '{"base_url":"https://search.mapzen.com/v1/search","read_timeout":60,"connect_timeout":10,"max_retries":1}' +)::json +); + +# Mapzen isochrones +SELECT CDB_Conf_SetConf( +'mapzen_conf', +jsonb_set( + to_jsonb(CDB_Conf_GetConf('mapzen_conf')), + '{isochrones, service}', + '{"base_url":"https://matrix.mapzen.com/isochrone","read_timeout":60,"connect_timeout":10,"max_retries":1}' +)::json +); + +# Mapzen isolines (matrix service) +SELECT CDB_Conf_SetConf( +'mapzen_conf', +jsonb_set( + to_jsonb(CDB_Conf_GetConf('mapzen_conf')), + '{matrix, service}', + '{"base_url":"https://matrix.mapzen.com/one_to_many","read_timeout":60,"connect_timeout":10}' +)::json +); + +# Mapzen routing +SELECT CDB_Conf_SetConf( +'mapzen_conf', +jsonb_set( + to_jsonb(CDB_Conf_GetConf('mapzen_conf')), + '{routing, service}', + '{"base_url":"https://valhalla.mapzen.com/route","read_timeout":60,"connect_timeout":10}' +)::json +); +``` +### User database configuration + +User (client) databases need also some configuration so that the client extension can access the server: +#### Users/Organizations + +```sql +SELECT CDB_Conf_SetConf('user_config', '{"is_organization": false, "entity_name": ""}'); +``` + +#### Dataservices server + +The `geocoder_server_config` (the name is not accurate for historical reasons) entry points +to the dataservices server DB (you can use a specific database for the server or your same user's): + +```sql +SELECT CDB_Conf_SetConf( + 'geocoder_server_config', + '{ "connection_str": "host=localhost port=5432 dbname= user=postgres"}' +); +``` +#### Search path + +The search path must be configured in order to be able to execute the functions without using the schema: + +```sql +ALTER ROLE "" SET search_path="$user", public, cartodb, cdb_dataservices_client; +``` diff --git a/server/extension/sql/100_routing_helper.sql b/server/extension/sql/100_routing_helper.sql index efa79bf..681d240 100644 --- a/server/extension/sql/100_routing_helper.sql +++ b/server/extension/sql/100_routing_helper.sql @@ -31,7 +31,7 @@ RETURNS cdb_dataservices_server.simple_route AS $$ raise Exception('You have reached the limit of your quota') try: - client = MapzenRouting(user_routing_config.mapzen_api_key, logger) + client = MapzenRouting(user_routing_config.mapzen_api_key, logger, user_routing_config.mapzen_service_params) if not waypoints or len(waypoints) < 2: logger.info("Empty origin or destination") diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index b4fbcd1..a8c9067 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -88,7 +88,7 @@ RETURNS Geometry AS $$ raise Exception('You have reached the limit of your quota') try: - geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code, logger) + geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code, logger, user_geocoder_config.heremaps_service_params) coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) if coordinates: quota_service.increment_success_service_use() @@ -115,7 +115,7 @@ RETURNS Geometry AS $$ redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] - + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") logger_config = GD["logger_config"] logger = Logger(logger_config) @@ -174,7 +174,7 @@ RETURNS Geometry AS $$ raise Exception('You have reached the limit of your quota') try: - geocoder = MapzenGeocoder(mapzen_geocoder_config.mapzen_api_key, logger) + geocoder = MapzenGeocoder(mapzen_geocoder_config.mapzen_api_key, logger, mapzen_geocoder_config.service_params) country_iso3 = None if country: country_iso3 = country_to_iso3(country) diff --git a/server/extension/sql/80_isolines_helper.sql b/server/extension/sql/80_isolines_helper.sql index 029c828..2e1bfb2 100644 --- a/server/extension/sql/80_isolines_helper.sql +++ b/server/extension/sql/80_isolines_helper.sql @@ -20,7 +20,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ raise Exception('You have reached the limit of your quota') try: - client = HereMapsRoutingIsoline(user_isolines_routing_config.heremaps_app_id, + client = HereMapsRoutingIsoline(user_isolines_routing_config.heremaps_app_id, user_isolines_routing_config.heremaps_app_code, logger) if source: @@ -81,7 +81,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ raise Exception('You have reached the limit of your quota') try: - client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key, logger) + client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key, logger, user_isolines_routing_config.mapzen_matrix_service_params) mapzen_isolines = MapzenIsolines(client, logger) if source: @@ -151,7 +151,7 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ try: mapzen_isochrones = MapzenIsochrones(user_isolines_routing_config.mapzen_matrix_api_key, - logger) + logger, user_isolines_routing_config.mapzen_isochrones_service_params) if source: lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] diff --git a/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py index a3eb805..6cd0a7b 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py @@ -52,14 +52,17 @@ class HereMapsGeocoder(Traceable): 'strictlanguagemode' ] + ADDRESS_PARAMS - def __init__(self, app_id, app_code, logger, maxresults=DEFAULT_MAXRESULTS, - gen=DEFAULT_GEN, host=PRODUCTION_GEOCODE_JSON_URL): + def __init__(self, app_id, app_code, logger, service_params=None, maxresults=DEFAULT_MAXRESULTS): + service_params = service_params or {} self.app_id = app_id self.app_code = app_code self._logger = logger self.maxresults = maxresults - self.gen = gen - self.host = host + self.gen = service_params.get('gen', self.DEFAULT_GEN) + self.host = service_params.get('json_url', self.PRODUCTION_GEOCODE_JSON_URL) + self.connect_timeout = service_params.get('connect_timeout', self.CONNECT_TIMEOUT) + self.read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT) + self.max_retries = service_params.get('max_retries', self.MAX_RETRIES) def geocode(self, **kwargs): params = {} @@ -92,9 +95,9 @@ class HereMapsGeocoder(Traceable): request_params.update(params) # TODO Extract HTTP client wrapper session = requests.Session() - session.mount(self.host, HTTPAdapter(max_retries=self.MAX_RETRIES)) + session.mount(self.host, HTTPAdapter(max_retries=self.max_retries)) response = session.get(self.host, params=request_params, - timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT)) + timeout=(self.connect_timeout, self.read_timeout)) self.add_response_data(response, self._logger) if response.status_code == requests.codes.ok: return json.loads(response.text) diff --git a/server/lib/python/cartodb_services/cartodb_services/here/routing.py b/server/lib/python/cartodb_services/cartodb_services/here/routing.py index 3a04a3a..fcc2e3e 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/routing.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/routing.py @@ -30,12 +30,17 @@ class HereMapsRoutingIsoline(Traceable): 'quality' ] - def __init__(self, app_id, app_code, logger, - base_url=PRODUCTION_ROUTING_BASE_URL): + def __init__(self, app_id, app_code, logger, service_params=None): + service_params = service_params or {} self._app_id = app_id self._app_code = app_code self._logger = logger - self._url = "{0}{1}".format(base_url, self.ISOLINE_PATH) + base_url = service_params.get('base_url', self.PRODUCTION_ROUTING_BASE_URL) + isoline_path = service_params.get('isoline_path', self.ISOLINE_PATH) + self.connect_timeout = service_params.get('connect_timeout', self.CONNECT_TIMEOUT) + self.read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT) + self.max_retries = service_params.get('max_retries', self.MAX_RETRIES) + self._url = "{0}{1}".format(base_url, isoline_path) def calculate_isodistance(self, source, mode, data_range, options=[]): return self.__calculate_isolines(source, mode, data_range, 'distance', @@ -57,9 +62,9 @@ class HereMapsRoutingIsoline(Traceable): parsed_options) # TODO Extract HTTP client wrapper session = requests.Session() - session.mount(self._url, HTTPAdapter(max_retries=self.MAX_RETRIES)) + session.mount(self._url, HTTPAdapter(max_retries=self.max_retries)) response = requests.get(self._url, params=request_params, - timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT)) + timeout=(self.connect_timeout, self.read_timeout)) self.add_response_data(response, self._logger) if response.status_code == requests.codes.ok: return self.__parse_isolines_response(response.text) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py index 1209388..666cdef 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py @@ -17,9 +17,13 @@ class MapzenGeocoder(Traceable): CONNECT_TIMEOUT = 10 MAX_RETRIES = 1 - def __init__(self, app_key, logger, base_url=BASE_URL): + def __init__(self, app_key, logger, service_params=None): + service_params = service_params or {} self._app_key = app_key - self._url = base_url + self._url = service_params.get('base_url', self.BASE_URL) + self._connect_timeout = service_params.get('connect_timeout', self.CONNECT_TIMEOUT) + self._read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT) + self._max_retries = service_params.get('max_retries', self.MAX_RETRIES) self._logger = logger @qps_retry(qps=20) @@ -31,9 +35,9 @@ class MapzenGeocoder(Traceable): try: # TODO Extract HTTP client wrapper session = requests.Session() - session.mount(self._url, HTTPAdapter(max_retries=self.MAX_RETRIES)) + session.mount(self._url, HTTPAdapter(max_retries=self._max_retries)) response = session.get(self._url, params=request_params, - timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT)) + timeout=(self._connect_timeout, self._read_timeout)) self.add_response_data(response, self._logger) if response.status_code == requests.codes.ok: return self.__parse_response(response.text) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/isochrones.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/isochrones.py index f7f15fd..8b17c57 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/isochrones.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/isochrones.py @@ -20,10 +20,15 @@ class MapzenIsochrones: "car": "auto" } - def __init__(self, app_key, logger, base_url=BASE_URL): + def __init__(self, app_key, logger, service_params=None): + service_params = service_params or {} self._app_key = app_key - self._url = base_url self._logger = logger + self._url = service_params.get('base_url', self.BASE_URL) + self._connect_timeout = service_params.get('connect_timeout', self.CONNECT_TIMEOUT) + self._read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT) + self._max_retries = service_params.get('max_retries', self.MAX_RETRIES) + @qps_retry(qps=7) def isochrone(self, locations, costing, ranges): @@ -32,10 +37,10 @@ class MapzenIsochrones: try: # TODO Extract HTTP client wrapper session = requests.Session() - session.mount(self._url, HTTPAdapter(max_retries=self.MAX_RETRIES)) + session.mount(self._url, HTTPAdapter(max_retries=self._max_retries)) response = session.get(self._url, params=request_params, - timeout=(self.CONNECT_TIMEOUT, - self.READ_TIMEOUT)) + timeout=(self._connect_timeout, + self._read_timeout)) if response.status_code is requests.codes.ok: return self._parse_response(response) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py index c10a912..041b382 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py @@ -23,9 +23,14 @@ class MatrixClient(Traceable): READ_TIMEOUT = 60 CONNECT_TIMEOUT = 10 - def __init__(self, matrix_key, logger): + def __init__(self, matrix_key, logger, service_params=None): + service_params = service_params or {} self._matrix_key = matrix_key self._logger = logger + self._url = service_params.get('one_to_many_url', self.ONE_TO_MANY_URL) + self._connect_timeout = service_params.get('connect_timeout', self.CONNECT_TIMEOUT) + self._read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT) + """Get distances and times to a set of locations. See https://mapzen.com/documentation/matrix/api-reference/ @@ -44,8 +49,8 @@ class MatrixClient(Traceable): 'costing': costing, 'api_key': self._matrix_key } - response = requests.get(self.ONE_TO_MANY_URL, params=request_params, - timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT)) + response = requests.get(self._url, params=request_params, + timeout=(self._connect_timeout, self._read_timeout)) self.add_response_data(response, self._logger) if response.status_code != requests.codes.ok: diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py index 0cb0c80..ac7e909 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py @@ -33,10 +33,14 @@ class MapzenRouting(Traceable): METRICS_UNITS = 'kilometers' IMPERIAL_UNITS = 'miles' - def __init__(self, app_key, logger, base_url=PRODUCTION_ROUTING_BASE_URL): + def __init__(self, app_key, logger, service_params=None): + service_params = service_params or {} self._app_key = app_key - self._url = base_url self._logger = logger + self._url = service_params.get('base_url', self.PRODUCTION_ROUTING_BASE_URL) + self._connect_timeout = service_params.get('connect_timeout', self.CONNECT_TIMEOUT) + self._read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT) + self._max_retries = service_params.get('max_retries', self.MAX_RETRIES) @qps_retry def calculate_route_point_to_point(self, waypoints, mode, @@ -50,9 +54,9 @@ class MapzenRouting(Traceable): request_params = self.__parse_request_parameters(json_request_params) # TODO Extract HTTP client wrapper session = requests.Session() - session.mount(self._url, HTTPAdapter(max_retries=self.MAX_RETRIES)) + session.mount(self._url, HTTPAdapter(max_retries=self._max_retries)) response = session.get(self._url, params=request_params, - timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT)) + timeout=(self._connect_timeout, self._read_timeout)) self.add_response_data(response, self._logger) if response.status_code == requests.codes.ok: return self.__parse_routing_response(response.text) diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index 14638f3..f3e93d0 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -147,6 +147,7 @@ class RoutingConfig(ServiceConfig): if not self._routing_provider: self._routing_provider = self.DEFAULT_PROVIDER self._mapzen_api_key = self._db_config.mapzen_routing_api_key + self._mapzen_service_params = self._db_config.mapzen_routing_service_params self._set_monthly_quota() self._set_soft_limit() self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE]) @@ -164,6 +165,10 @@ class RoutingConfig(ServiceConfig): def mapzen_api_key(self): return self._mapzen_api_key + @property + def mapzen_service_params(self): + return self._mapzen_service_params + @property def monthly_quota(self): return self._monthly_quota @@ -222,8 +227,11 @@ class IsolinesRoutingConfig(ServiceConfig): if self._isolines_provider == self.HEREMAPS_PROVIDER: self._heremaps_app_id = db_config.heremaps_isolines_app_id self._heremaps_app_code = db_config.heremaps_isolines_app_code + self._heremaps_service_params = db_config.heremaps_isolines_service_params elif self._isolines_provider == self.MAPZEN_PROVIDER: self._mapzen_matrix_api_key = self._db_config.mapzen_matrix_api_key + self._mapzen_matrix_service_params = db_config.mapzen_matrix_service_params + self._mapzen_isochrones_service_params = db_config.mapzen_isochrones_service_params @property def service_type(self): @@ -256,10 +264,22 @@ class IsolinesRoutingConfig(ServiceConfig): def heremaps_app_code(self): return self._heremaps_app_code + @property + def heremaps_service_params(self): + return self._heremaps_service_params + @property def mapzen_matrix_api_key(self): return self._mapzen_matrix_api_key + @property + def mapzen_matrix_service_params(self): + return self._mapzen_matrix_service_params + + @property + def mapzen_isochrones_service_params(self): + return self._mapzen_isochrones_service_params + @property def mapzen_provider(self): return self._isolines_provider == self.MAPZEN_PROVIDER @@ -368,6 +388,7 @@ class GeocoderConfig(ServiceConfig): self._heremaps_app_id = db_config.heremaps_geocoder_app_id self._heremaps_app_code = db_config.heremaps_geocoder_app_code self._cost_per_hit = db_config.heremaps_geocoder_cost_per_hit + self._heremaps_service_params = db_config.heremaps_geocoder_service_params elif self._geocoder_provider == self.GOOGLE_GEOCODER: self._google_maps_api_key = filtered_config[self.GOOGLE_GEOCODER_API_KEY] self._google_maps_client_id = filtered_config[self.GOOGLE_GEOCODER_CLIENT_ID] @@ -375,6 +396,7 @@ class GeocoderConfig(ServiceConfig): elif self._geocoder_provider == self.MAPZEN_GEOCODER: self._mapzen_api_key = db_config.mapzen_geocoder_api_key self._cost_per_hit = 0 + self._mapzen_service_params = db_config.mapzen_geocoder_service_params @property def service_type(self): @@ -428,10 +450,18 @@ class GeocoderConfig(ServiceConfig): def heremaps_app_code(self): return self._heremaps_app_code + @property + def heremaps_service_params(self): + return self._heremaps_service_params + @property def mapzen_api_key(self): return self._mapzen_api_key + @property + def mapzen_service_params(self): + return self._mapzen_service_params + @property def is_high_resolution(self): return True @@ -444,6 +474,9 @@ class GeocoderConfig(ServiceConfig): def provider(self): return self._geocoder_provider + @property + def service(self): + return self._service class ServicesDBConfig: @@ -480,8 +513,10 @@ class ServicesDBConfig: self._heremaps_geocoder_app_code = heremaps_conf['geocoder']['app_code'] self._heremaps_geocoder_cost_per_hit = heremaps_conf['geocoder'][ 'geocoder_cost_per_hit'] + self._heremaps_geocoder_service_params = heremaps_conf['geocoder'].get('service', {}) self._heremaps_isolines_app_id = heremaps_conf['isolines']['app_id'] self._heremaps_isolines_app_code = heremaps_conf['isolines']['app_code'] + self._heremaps_isolines_service_params = heremaps_conf['isolines'].get('service', {}) def _get_mapzen_config(self): mapzen_conf_json = self._get_conf('mapzen_conf') @@ -491,10 +526,14 @@ class ServicesDBConfig: mapzen_conf = json.loads(mapzen_conf_json) self._mapzen_matrix_api_key = mapzen_conf['matrix']['api_key'] self._mapzen_matrix_quota = mapzen_conf['matrix']['monthly_quota'] + self._mapzen_matrix_service_params = mapzen_conf['matrix'].get('service', {}) + self._mapzen_isochrones_service_params = mapzen_conf.get('isochrones', {}).get('service', {}) self._mapzen_routing_api_key = mapzen_conf['routing']['api_key'] self._mapzen_routing_quota = mapzen_conf['routing']['monthly_quota'] + self._mapzen_routing_service_params = mapzen_conf['routing'].get('service', {}) self._mapzen_geocoder_api_key = mapzen_conf['geocoder']['api_key'] self._mapzen_geocoder_quota = mapzen_conf['geocoder']['monthly_quota'] + self._mapzen_geocoder_service_params = mapzen_conf['geocoder'].get('service', {}) def _get_data_observatory_config(self): do_conf_json = self._get_conf('data_observatory_conf') @@ -530,6 +569,10 @@ class ServicesDBConfig: def heremaps_isolines_app_code(self): return self._heremaps_isolines_app_code + @property + def heremaps_isolines_service_params(self): + return self._heremaps_isolines_service_params + @property def heremaps_geocoder_app_id(self): return self._heremaps_geocoder_app_id @@ -542,6 +585,10 @@ class ServicesDBConfig: def heremaps_geocoder_cost_per_hit(self): return self._heremaps_geocoder_cost_per_hit + @property + def heremaps_geocoder_service_params(self): + return self._heremaps_geocoder_service_params + @property def mapzen_matrix_api_key(self): return self._mapzen_matrix_api_key @@ -550,6 +597,14 @@ class ServicesDBConfig: def mapzen_matrix_monthly_quota(self): return self._mapzen_matrix_quota + @property + def mapzen_matrix_service_params(self): + return self._mapzen_matrix_service_params + + @property + def mapzen_isochrones_service_params(self): + return self._mapzen_isochrones_service_params + @property def mapzen_routing_api_key(self): return self._mapzen_routing_api_key @@ -558,6 +613,10 @@ class ServicesDBConfig: def mapzen_routing_monthly_quota(self): return self._mapzen_routing_quota + @property + def mapzen_routing_service_params(self): + return self._mapzen_routing_service_params + @property def mapzen_geocoder_api_key(self): return self._mapzen_geocoder_api_key @@ -566,6 +625,10 @@ class ServicesDBConfig: def mapzen_geocoder_monthly_quota(self): return self._mapzen_geocoder_quota + @property + def mapzen_geocoder_service_params(self): + return self._mapzen_geocoder_service_params + @property def data_observatory_connection_str(self): return self._data_observatory_connection_str diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder_config.py index 4c5884a..af74e00 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder_config.py @@ -13,7 +13,8 @@ class MapzenGeocoderConfig(object): log_path, mapzen_api_key, username, - organization): + organization, + service_params): self._geocoding_quota = geocoding_quota self._soft_geocoding_limit = soft_geocoding_limit self._period_end_date = period_end_date @@ -22,6 +23,7 @@ class MapzenGeocoderConfig(object): self._mapzen_api_key = mapzen_api_key self._username = username self._organization = organization + self._service_params = service_params # Kind of generic properties. Note which ones are for actually running the # service and which ones are needed for quota stuff. @@ -72,6 +74,10 @@ class MapzenGeocoderConfig(object): def organization(self): return self._organization + @property + def service_params(self): + return self._service_params + # TODO: for BW compat, remove @property def google_geocoder(self): @@ -90,6 +96,7 @@ class MapzenGeocoderConfigBuilder(object): def get(self): mapzen_server_conf = self._server_conf.get('mapzen_conf') mapzen_api_key = mapzen_server_conf['geocoder']['api_key'] + mapzen_service_params = mapzen_server_conf['geocoder'].get('service', {}) geocoding_quota = self._get_quota(mapzen_server_conf) soft_geocoding_limit = self._user_conf.get('soft_geocoding_limit').lower() == 'true' @@ -107,7 +114,8 @@ class MapzenGeocoderConfigBuilder(object): log_path, mapzen_api_key, self._username, - self._orgname) + self._orgname, + mapzen_service_params) def _get_quota(self, mapzen_server_conf): geocoding_quota = self._org_conf.get('geocoding_quota') or self._user_conf.get('geocoding_quota') diff --git a/server/lib/python/cartodb_services/test/refactor/service/test_mapzen_geocoder_config.py b/server/lib/python/cartodb_services/test/refactor/service/test_mapzen_geocoder_config.py index d67a2bf..b61f4e6 100644 --- a/server/lib/python/cartodb_services/test/refactor/service/test_mapzen_geocoder_config.py +++ b/server/lib/python/cartodb_services/test/refactor/service/test_mapzen_geocoder_config.py @@ -13,7 +13,7 @@ class TestMapzenGeocoderUserConfig(TestCase): self._server_config = InMemoryConfigStorage({"server_conf": {"environment": "testing"}, "mapzen_conf": {"geocoder": - {"api_key": "search-xxxxxxx", "monthly_quota": 1500000} + {"api_key": "search-xxxxxxx", "monthly_quota": 1500000, "service":{"base_url":"http://base"}} }, "logger_conf": {}}) self._username = 'test_user' self._user_key = "rails:users:{0}".format(self._username) @@ -81,6 +81,14 @@ class TestMapzenGeocoderUserConfig(TestCase): self._redis_connection.hset(self._user_key, 'soft_geocoding_limit', 'false') self._redis_connection.hset(self._user_key, 'period_end_date', '2016-12-31 00:00:00') + def test_config_service_values(self): + config = MapzenGeocoderConfigBuilder(self._server_config, + self._user_config, + self._org_config, + self._username, + None).get() + assert config.service_params == {"base_url":"http://base"} + class TestMapzenGeocoderOrgConfig(TestCase): def setUp(self): @@ -151,4 +159,12 @@ class TestMapzenGeocoderOrgConfig(TestCase): self._redis_connection.hset(self._user_key, 'soft_geocoding_limit', 'false') self._redis_connection.hset(self._user_key, 'period_end_date', '2016-12-15 00:00:00') self._redis_connection.hset(self._org_key, 'geocoding_quota', '200') - self._redis_connection.hset(self._org_key, 'period_end_date', '2016-12-31 00:00:00') \ No newline at end of file + self._redis_connection.hset(self._org_key, 'period_end_date', '2016-12-31 00:00:00') + + def test_config_default_service_values(self): + config = MapzenGeocoderConfigBuilder(self._server_config, + self._user_config, + self._org_config, + self._username, + self._organization).get() + assert config.service_params == {} diff --git a/server/lib/python/cartodb_services/test/test_heremapsgeocoder.py b/server/lib/python/cartodb_services/test/test_heremapsgeocoder.py index c09d75b..daf813d 100644 --- a/server/lib/python/cartodb_services/test/test_heremapsgeocoder.py +++ b/server/lib/python/cartodb_services/test/test_heremapsgeocoder.py @@ -145,3 +145,17 @@ class HereMapsGeocoderTestCase(unittest.TestCase): searchtext='Calle amor de dios', city='Cordoba', country='España') + + def test_geocode_with_nonstandard_url(self, req_mock): + geocoder = HereMapsGeocoder(None, None, Mock(), { 'json_url': 'http://nonstandard_here_url' }) + req_mock.register_uri('GET', 'http://nonstandard_here_url', text=self.GOOD_RESPONSE) + response = geocoder.geocode( + searchtext='Calle amor de dios', + city='Cordoba', + country='España') + + self.assertEqual(response[0], -5.2794) + self.assertEqual(response[1], 37.70246) + + + diff --git a/server/lib/python/cartodb_services/test/test_heremapsrouting.py b/server/lib/python/cartodb_services/test/test_heremapsrouting.py index 6ce247a..6226572 100644 --- a/server/lib/python/cartodb_services/test/test_heremapsrouting.py +++ b/server/lib/python/cartodb_services/test/test_heremapsrouting.py @@ -212,3 +212,20 @@ class HereMapsRoutingIsolineTestCase(unittest.TestCase): parsed_url = urlparse(req_mock.request_history[0].url) url_params = parse_qs(parsed_url.query) self.assertEqual(url_params['destination'][0], 'geo!33.0,1.0') + + def test_isodistance_with_nonstandard_url(self, req_mock): + base_url = 'http://nonstandard_base' + url = "{0}{1}".format(base_url, HereMapsRoutingIsoline.ISOLINE_PATH) + routing = HereMapsRoutingIsoline(None, None, Mock(), { 'base_url': base_url }) + req_mock.register_uri('GET', url, text=self.GOOD_RESPONSE) + response = routing.calculate_isodistance('geo!33.0,1.0', 'car', + ['1000', '2000']) + self.assertEqual(len(response), 2) + self.assertEqual(response[0]['range'], 1000) + self.assertEqual(response[1]['range'], 2000) + self.assertEqual(response[0]['geom'], [u'32.9699707,0.9462833', + u'32.9699707,0.9458542', + u'32.9699707,0.9462833']) + self.assertEqual(response[1]['geom'], [u'32.9699707,0.9462833', + u'32.9699707,0.9750366', + u'32.9699707,0.9462833']) diff --git a/server/lib/python/cartodb_services/test/test_mapzengeocoder.py b/server/lib/python/cartodb_services/test/test_mapzengeocoder.py index 19d8f42..3a3b1e6 100644 --- a/server/lib/python/cartodb_services/test/test_mapzengeocoder.py +++ b/server/lib/python/cartodb_services/test/test_mapzengeocoder.py @@ -109,3 +109,14 @@ class MapzenGeocoderTestCase(unittest.TestCase): self.geocoder.geocode( searchtext='Calle Siempreviva 3, Valladolid', country='ESP') + + def test_geocode_address_with_nonstandard_url(self, req_mock): + nonstandard_url = 'http://nonstandardmapzen' + req_mock.register_uri('GET', nonstandard_url, text=self.GOOD_RESPONSE) + geocoder = MapzenGeocoder('search-XXXXXXX', Mock(), { 'base_url': nonstandard_url }) + response = geocoder.geocode( + searchtext='Calle Siempreviva 3, Valldolid', + country='ESP') + + self.assertEqual(response[0], -4.730928) + self.assertEqual(response[1], 41.669034) diff --git a/server/lib/python/cartodb_services/test/test_mapzenisochrones.py b/server/lib/python/cartodb_services/test/test_mapzenisochrones.py index 6b566c4..184b010 100644 --- a/server/lib/python/cartodb_services/test/test_mapzenisochrones.py +++ b/server/lib/python/cartodb_services/test/test_mapzenisochrones.py @@ -52,3 +52,16 @@ class MapzenIsochronesTestCase(unittest.TestCase): with self.assertRaises(ServiceException): self.mapzen_isochrones.isochrone([-41.484375, 28.993727], 'walk', [300, 900]) + + def test_nonstandard_url(self, req_mock): + url = 'http://serviceurl.com' + req_mock.register_uri('GET', url, text=self.GOOD_RESPONSE) + mapzen_isochrones = MapzenIsochrones('matrix-xxxxx', Mock(), {'base_url': url}) + + response = mapzen_isochrones.isochrone([-41.484375, 28.993727], + 'walk', [300, 900]) + self.assertEqual(len(response), 2) + self.assertEqual(response[0].coordinates, [[-3.702579,40.430893],[-3.702193,40.430122],[-3.702579,40.430893]]) + self.assertEqual(response[0].duration, 15) + self.assertEqual(response[1].coordinates, [[-3.703050,40.424995],[-3.702546,40.424694],[-3.703050,40.424995]]) + self.assertEqual(response[1].duration, 5) diff --git a/server/lib/python/cartodb_services/test/test_mapzenrouting.py b/server/lib/python/cartodb_services/test/test_mapzenrouting.py index 2315146..916558a 100644 --- a/server/lib/python/cartodb_services/test/test_mapzenrouting.py +++ b/server/lib/python/cartodb_services/test/test_mapzenrouting.py @@ -142,3 +142,17 @@ class MapzenRoutingTestCase(unittest.TestCase): self.assertEqual(response.length, 1.261) self.assertEqual(response.duration, 913) self.assertEqual(response.shape, self.GOOD_SHAPE_MULTI) + + def test_nonstandard_url(self, req_mock): + url = 'http://serviceurl.com' + routing = MapzenRouting('api_key', Mock(), {'base_url': url}) + req_mock.register_uri('GET', url, text=self.GOOD_RESPONSE_SIMPLE) + origin = Coordinate('-120.2', '38.5') + destination = Coordinate('-126.4', '43.2') + waypoints = [origin, destination] + response = routing.calculate_route_point_to_point(waypoints, + 'car') + + self.assertEqual(response.shape, self.GOOD_SHAPE_SIMPLE) + self.assertEqual(response.length, 444.59) + self.assertEqual(response.duration, 16969)