diff --git a/README.md b/README.md index 809cd71..c53057b 100644 --- a/README.md +++ b/README.md @@ -23,11 +23,11 @@ Steps to deploy a new Data Services API version : ### Local install instructions -- install data services extension +- install data services geocoder extension ``` git clone git@github.com:CartoDB/data-services.git - data-services/geocoder/extension + cd data-services/geocoder/extension sudo make install ``` @@ -35,7 +35,7 @@ Steps to deploy a new Data Services API version : ``` cd client && sudo make install - cd server && sudo make install + cd server/extension && sudo make install ``` - install python library @@ -57,21 +57,28 @@ Steps to deploy a new Data Services API version : ``` - # select CDB_Conf_SetConf('redis_metadata_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); - # select CDB_Conf_SetConf('redis_metrics_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); - - # select CDB_Conf_SetConf('heremaps_conf', '{"app_id": "APP_ID", "app_code": "APP_CODE"}'); - # select CDB_Conf_SetConf('user_config', '{"is_organization": false, "entity_name": ""}') + # If sentinel is used: + SELECT CDB_Conf_SetConf('redis_metadata_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); + SELECT CDB_Conf_SetConf('redis_metrics_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "", "timeout": 0.1, "redis_db": 5}'); + + # If sentinel is not used + SELECT CDB_Conf_SetConf('redis_metadata_config', '{"redis_host": "localhost", "redis_port": 26379, "sentinel_master_id": "", "timeout": 0.1, "redis_db": 5}'); + SELECT CDB_Conf_SetConf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 6379, "sentinel_master_id": "", "timeout": 0.1, "redis_db": 5}'); + + SELECT CDB_Conf_SetConf('heremaps_conf', '{"app_id": "APP_ID", "app_code": "APP_CODE", "geocoder_cost_per_hit": "COST_PER_HIT"}'); + SELECT CDB_Conf_SetConf('user_config', '{"is_organization": false, "entity_name": ""}') + SELECT CDB_Conf_SetConf('mapzen_conf', '{"routing_app_key": "ROUTING_API_KEY", "geocoder_app_key": "GEOCODER_API_KEY"}'); + SELECT CDB_Conf_SetConf('logger_con', '{"geocoder_log_path": "/tmp/geocodings.log"}') ``` -- configure plproxy to point to the same user database (you could do in a different one) +- configure plproxy to point to the a database (you can use a specific database for the server or your same user) ``` - select CDB_Conf_SetConf('geocoder_server_config', '{ "connection_str": "host=localhost port=5432 dbname=cartodb_dev_user_accf0647-d942-4e37-b129-8287c117e687_db user=postgres"}'); + SELECT CDB_Conf_SetConf('geocoder_server_config', '{ "connection_str": "host=localhost port=5432 dbname= user=postgres"}'); ``` -- configure the search path in order to be able to execute the functions without use the schema: +- configure the search path in order to be able to execute the functions without using the schema: ``` - alter role "" set search_path='"$user", public, cartodb, cdb_dataservices_client'; + ALTER ROLE "" SET search_path="$user", public, cartodb, cdb_dataservices_client; ``` diff --git a/doc/geocoding_functions.md b/doc/geocoding_functions.md index 34b0dde..91b0a46 100644 --- a/doc/geocoding_functions.md +++ b/doc/geocoding_functions.md @@ -208,6 +208,8 @@ UPDATE {tablename} SET the_geom = cdb_geocode_namedplace_point({city_column}, {p This function geocodes postal codes and country names and transforms them to points or polygon geometries. The postal code polygon geocoder covers the United States, France, Australia and Canada; a request for a different country will return an empty response. +**Note:** For the USA, US Census [Zip Code Tabulation Areas](https://www.census.gov/geo/reference/zctas.html) (ZCTA) are used to reference geocodes for USPS postal codes service areas. See the [FAQs](http://docs.cartodb.com/faqs/datasets-and-data/#why-does-cartodb-use-census-bureau-zctas-and-not-usps-zip-codes-for-postal-codes) about datasets and data for details. + ### cdb_geocode_postalcode_polygon(_postal_code text, country_name text_) Geocodes the postal code, and country name, into polygon data. @@ -237,8 +239,6 @@ SELECT cdb_geocode_postalcode_polygon('11211', 'USA') UPDATE {tablename} SET the_geom = cdb_geocode_postalcode_polygon({postal_code_column}, 'USA') ``` -**Note:** For the USA, US Census ZCTAs are considered. - ### cdb_geocode_postalcode_point(_code text, country_name text_) Geocodes postal codes, and country names, into point data. diff --git a/server/extension/Makefile b/server/extension/Makefile index 6dd6db0..d764817 100644 --- a/server/extension/Makefile +++ b/server/extension/Makefile @@ -13,8 +13,8 @@ OLD_VERSIONS = $(wildcard old_versions/*.sql) # @see http://www.postgresql.org/docs/current/static/extend-pgxs.html DATA = $(NEW_EXTENSION_ARTIFACT) \ $(OLD_VERSIONS) \ - cdb_dataservices_server--0.6.0--0.5.2.sql \ - cdb_dataservices_server--0.5.2--0.6.0.sql + cdb_dataservices_server--0.6.1--0.6.0.sql \ + cdb_dataservices_server--0.6.0--0.6.1.sql REGRESS = $(notdir $(basename $(wildcard test/sql/*test.sql))) TEST_DIR = test/ diff --git a/server/extension/cdb_dataservices_server--0.6.0--0.6.1.sql b/server/extension/cdb_dataservices_server--0.6.0--0.6.1.sql new file mode 100644 index 0000000..7f3b5c2 --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.6.0--0.6.1.sql @@ -0,0 +1,39 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.6.1'" to load this file. \quit +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.mapzen import MapzenGeocoder + from cartodb_services.mapzen.types import country_to_iso3 + from cartodb_services.metrics import QuotaService + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + quota_service = QuotaService(user_geocoder_config, redis_conn) + + try: + geocoder = MapzenGeocoder(user_geocoder_config.mapzen_app_key) + country_iso3 = None + if country: + country_iso3 = country_to_iso3(country) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3) + if coordinates: + quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using mapzen geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; \ No newline at end of file diff --git a/server/extension/cdb_dataservices_server--0.6.1--0.6.0.sql b/server/extension/cdb_dataservices_server--0.6.1--0.6.0.sql new file mode 100644 index 0000000..5ac3380 --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.6.1--0.6.0.sql @@ -0,0 +1,33 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.6.0'" to load this file. \quit +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.mapzen import MapzenGeocoder + from cartodb_services.metrics import QuotaService + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + quota_service = QuotaService(user_geocoder_config, redis_conn) + + try: + geocoder = MapzenGeocoder(user_geocoder_config.mapzen_app_key) + coordinates = geocoder.geocode(searchtext=searchtext, country=country) + if coordinates: + quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using mapzen geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; \ No newline at end of file diff --git a/server/extension/cdb_dataservices_server--0.5.2.sql b/server/extension/cdb_dataservices_server--0.6.1.sql similarity index 95% rename from server/extension/cdb_dataservices_server--0.5.2.sql rename to server/extension/cdb_dataservices_server--0.6.1.sql index d2906fb..8c8989a 100644 --- a/server/extension/cdb_dataservices_server--0.5.2.sql +++ b/server/extension/cdb_dataservices_server--0.6.1.sql @@ -175,6 +175,9 @@ RETURNS Geometry AS $$ elif user_geocoder_config.google_geocoder: google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapzen_geocoder: + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] else: plpy.error('Requested geocoder is not available') @@ -245,6 +248,40 @@ RETURNS Geometry AS $$ finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.mapzen import MapzenGeocoder + from cartodb_services.metrics import QuotaService + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + quota_service = QuotaService(user_geocoder_config, redis_conn) + + try: + geocoder = MapzenGeocoder(user_geocoder_config.mapzen_app_key) + country_iso3 = None + if country: + country_iso3 = country_to_iso3(country) + coordinates = geocoder.geocode(searchtext=searchtext, country=country_iso3) + if coordinates: + quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using mapzen geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService diff --git a/server/extension/cdb_dataservices_server.control b/server/extension/cdb_dataservices_server.control index f0182bc..682ea2a 100644 --- a/server/extension/cdb_dataservices_server.control +++ b/server/extension/cdb_dataservices_server.control @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices server extension' -default_version = '0.6.0' +default_version = '0.6.1' requires = 'plpythonu, postgis, cdb_geocoder' superuser = true schema = cdb_dataservices_server diff --git a/server/extension/cdb_dataservices_server--0.5.2--0.6.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.5.2--0.6.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.5.2--0.6.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.5.2--0.6.0.sql diff --git a/server/extension/cdb_dataservices_server--0.6.0--0.5.2.sql b/server/extension/old_versions/cdb_dataservices_server--0.6.0--0.5.2.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.6.0--0.5.2.sql rename to server/extension/old_versions/cdb_dataservices_server--0.6.0--0.5.2.sql diff --git a/server/extension/cdb_dataservices_server--0.6.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.6.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.6.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.6.0.sql diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index 5e3fc2b..ae34242 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -97,7 +97,10 @@ RETURNS Geometry AS $$ try: geocoder = MapzenGeocoder(user_geocoder_config.mapzen_app_key) - coordinates = geocoder.geocode(searchtext=searchtext, country=country) + country_iso3 = None + if country: + country_iso3 = country_to_iso3(country) + coordinates = geocoder.geocode(searchtext=searchtext, country=country_iso3) if coordinates: quota_service.increment_success_service_use() plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) diff --git a/server/extension/test/expected/00_install_test.out b/server/extension/test/expected/00_install_test.out index c69ea04..d6951fd 100644 --- a/server/extension/test/expected/00_install_test.out +++ b/server/extension/test/expected/00_install_test.out @@ -19,7 +19,7 @@ SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localh (1 row) -SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}'); +SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"geocoder": {"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "dummy_id", "app_code": "dummy_code"}}'); cdb_conf_setconf ------------------ diff --git a/server/extension/test/sql/00_install_test.sql b/server/extension/test/sql/00_install_test.sql index 2e3cb2e..d86e983 100644 --- a/server/extension/test/sql/00_install_test.sql +++ b/server/extension/test/sql/00_install_test.sql @@ -11,7 +11,7 @@ CREATE EXTENSION cdb_dataservices_server; -- Mock the redis server connection to point to this very test db SELECT cartodb.cdb_conf_setconf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}'); SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}'); -SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}'); +SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"geocoder": {"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "dummy_id", "app_code": "dummy_code"}}'); SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing_app_key": "dummy_key", "geocoder_app_key": "dummy_key"}'); SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}'); diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py index be98761..9adbfea 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py @@ -17,8 +17,8 @@ class MapzenGeocoder: self._url = base_url @qps_retry - def geocode(self, searchtext, country=None): - request_params = self._build_requests_parameters(searchtext, country) + def geocode(self, searchtext, city=None, state_province=None, country=None): + request_params = self._build_requests_parameters(searchtext, city, state_province, country) response = requests.get(self._url, params=request_params) if response.status_code == requests.codes.ok: return self.__parse_response(response.text) @@ -27,15 +27,26 @@ class MapzenGeocoder: else: response.raise_for_status() - def _build_requests_parameters(self, searchtext, country=None): + def _build_requests_parameters(self, searchtext, city=None, + state_province=None, country=None): request_params = {} - request_params['text'] = searchtext + search_string = self._build_search_text(searchtext, city, state_province) + request_params['text'] = search_string request_params['layers'] = 'address' request_params['api_key'] = self._app_key if country: request_params['boundary.country'] = country return request_params + def _build_search_text(self, searchtext, city, state_province): + search_string = searchtext + if city: + search_string = "{0}, {1}".format(search_string, city) + if state_province: + search_string = "{0}, {1}".format(search_string, state_province) + + return search_string + def __parse_response(self, response): try: parsed_json_response = json.loads(response) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/qps.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/qps.py index 40ff1c7..fe8ebb2 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/qps.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/qps.py @@ -1,4 +1,5 @@ import time +import random from datetime import datetime from exceptions import TimeoutException diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/types.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/types.py index fa3aeed..df82aba 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/types.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/types.py @@ -18,3 +18,17 @@ def polyline_to_linestring(polyline): geometry = None return geometry + +def country_to_iso3(country): + """ Convert country to its iso3 code """ + try: + country_plan = plpy.prepare("SELECT adm0_a3 as iso3 FROM admin0_synonyms WHERE lower(regexp_replace($1, " \ + "'[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text = name_; ", ['text']) + country_result = plpy.execute(country_plan, [country], 1) + if country_result: + return country_result[0]['iso3'] + else: + return None + except BaseException as e: + plpy.warning("Can't get the iso3 code from {0}: {1}".format(country, e)) + return None diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index a729cb4..7731b0f 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -54,10 +54,10 @@ class IsolinesRoutingConfig(ServiceConfig): ROUTING_CONFIG_KEYS = ['here_isolines_quota', 'soft_here_isolines_limit', 'period_end_date', 'username', 'orgname', - 'heremaps_app_id', 'heremaps_app_code', + 'heremaps_isolines_app_id', 'heremaps_isolines_app_code', 'geocoder_type'] - NOKIA_APP_ID_KEY = 'heremaps_app_id' - NOKIA_APP_CODE_KEY = 'heremaps_app_code' + NOKIA_APP_ID_KEY = 'heremaps_isolines_app_id' + NOKIA_APP_CODE_KEY = 'heremaps_isolines_app_code' QUOTA_KEY = 'here_isolines_quota' SOFT_LIMIT_KEY = 'soft_here_isolines_limit' USERNAME_KEY = 'username' @@ -102,8 +102,8 @@ class IsolinesRoutingConfig(ServiceConfig): self._soft_isolines_limit = True else: self._soft_isolines_limit = False - self._heremaps_app_id = db_config.heremaps_app_id - self._heremaps_app_code = db_config.heremaps_app_code + self._heremaps_app_id = db_config.heremaps_isolines_app_id + self._heremaps_app_code = db_config.heremaps_isolines_app_code @property def service_type(self): @@ -168,12 +168,12 @@ class GeocoderConfig(ServiceConfig): GEOCODER_CONFIG_KEYS = ['google_maps_client_id', 'google_maps_api_key', 'geocoding_quota', 'soft_geocoding_limit', 'geocoder_type', 'period_end_date', - 'heremaps_app_id', 'heremaps_app_code', + 'heremaps_geocoder_app_id', 'heremaps_geocoder_app_code', 'mapzen_geocoder_app_key', 'username', 'orgname'] NOKIA_GEOCODER_REDIS_MANDATORY_KEYS = ['geocoding_quota', 'soft_geocoding_limit'] NOKIA_GEOCODER = 'heremaps' - NOKIA_GEOCODER_APP_ID_KEY = 'heremaps_app_id' - NOKIA_GEOCODER_APP_CODE_KEY = 'heremaps_app_code' + NOKIA_GEOCODER_APP_ID_KEY = 'heremaps_geocoder_app_id' + NOKIA_GEOCODER_APP_CODE_KEY = 'heremaps_geocoder_app_code' GOOGLE_GEOCODER = 'google' GOOGLE_GEOCODER_API_KEY = 'google_maps_api_key' GOOGLE_GEOCODER_CLIENT_ID = 'google_maps_client_id' @@ -241,8 +241,8 @@ class GeocoderConfig(ServiceConfig): else: self._soft_geocoding_limit = False if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER: - self._heremaps_app_id = db_config.heremaps_app_id - self._heremaps_app_code = db_config.heremaps_app_code + self._heremaps_app_id = db_config.heremaps_geocoder_app_id + self._heremaps_app_code = db_config.heremaps_geocoder_app_code self._cost_per_hit = db_config.heremaps_geocoder_cost_per_hit elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER: self._google_maps_api_key = filtered_config[self.GOOGLE_GEOCODER_API_KEY] @@ -338,10 +338,12 @@ class ServicesDBConfig: raise ConfigException('Here maps configuration missing') else: heremaps_conf = json.loads(heremaps_conf_json) - self._heremaps_app_id = heremaps_conf['app_id'] - self._heremaps_app_code = heremaps_conf['app_code'] - self._heremaps_geocoder_cost_per_hit = heremaps_conf[ + self._heremaps_geocoder_app_id = heremaps_conf['geocoder']['app_id'] + self._heremaps_geocoder_app_code = heremaps_conf['geocoder']['app_code'] + self._heremaps_geocoder_cost_per_hit = heremaps_conf['geocoder'][ 'geocoder_cost_per_hit'] + self._heremaps_isolines_app_id = heremaps_conf['isolines']['app_id'] + self._heremaps_isolines_app_code = heremaps_conf['isolines']['app_code'] def _get_mapzen_config(self): mapzen_conf_json = self._get_conf('mapzen_conf') @@ -369,12 +371,20 @@ class ServicesDBConfig: raise ConfigException("Malformed config for {0}: {1}".format(key, e)) @property - def heremaps_app_id(self): - return self._heremaps_app_id + def heremaps_isolines_app_id(self): + return self._heremaps_isolines_app_id @property - def heremaps_app_code(self): - return self._heremaps_app_code + def heremaps_isolines_app_code(self): + return self._heremaps_isolines_app_code + + @property + def heremaps_geocoder_app_id(self): + return self._heremaps_geocoder_app_id + + @property + def heremaps_geocoder_app_code(self): + return self._heremaps_geocoder_app_code @property def heremaps_geocoder_cost_per_hit(self): diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py b/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py index ab24a15..2b77531 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py @@ -16,21 +16,21 @@ class RedisConnection: sentinel = Sentinel([(self._config.host, self._config.port)], socket_timeout=self._config.timeout) - return sentinel.master_for( - self._config.sentinel_id, - socket_timeout=self._config.timeout, - db=self._config.db - ) + return sentinel.master_for(self._config.sentinel_id, + socket_timeout=self._config.timeout, + db=self._config.db, + retry_on_timeout=True) else: conn = StrictRedis(host=self._config.host, port=self._config.port, - db=self._config.db) + db=self._config.db, retry_on_timeout=True, + socket_timeout=self._config.timeout) return conn class RedisDBConfig: DEFAULT_USER_DB = 5 - DEFAULT_TIMEOUT = 2 # seconds + DEFAULT_TIMEOUT = 1.5 # seconds def __init__(self, key, db_conn): self._db_conn = db_conn @@ -63,8 +63,9 @@ class RedisDBConfig: self._sentinel_id = None def __str__(self): - return "Host: {0}, Port: {1}, Sentinel id: {2}, DB: {3}".format( - self.host, self.port, self.sentinel_id, self.db) + return "Host: {0}, Port: {1}, Sentinel id: {2}, DB: {3}, " \ + "Timeout: {4}".format(self.host, self.port, self.sentinel_id, + self.db, self.timeout) @property def host(self): diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index 633f24c..f3f0f04 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.4.0', + version='0.4.4', description='CartoDB Services API Python Library', diff --git a/server/lib/python/cartodb_services/test/test_helper.py b/server/lib/python/cartodb_services/test/test_helper.py index 9856aca..349b043 100644 --- a/server/lib/python/cartodb_services/test/test_helper.py +++ b/server/lib/python/cartodb_services/test/test_helper.py @@ -44,7 +44,7 @@ def build_plpy_mock(empty=False): def _plpy_execute_side_effect(*args, **kwargs): if args[0] == "SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as conf": - return [{'conf': '{"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}'}] + return [{'conf': '{"geocoder": {"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "app_id", "app_code": "code"}}'}] elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('mapzen_conf') as conf": return [{'conf': '{"routing_app_key": "app_key", "geocoder_app_key": "app_key"}'}] elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('logger_conf') as conf":