From f92191a69f38dbc520c0699567f02d120864780a Mon Sep 17 00:00:00 2001 From: csobier Date: Fri, 18 Mar 2016 14:11:59 -0400 Subject: [PATCH 01/17] added note about zcta under postal codes generator section --- doc/API.md | 7 ++++--- doc/general_concepts.md | 2 +- doc/geocoding_functions.md | 6 +++--- doc/isoline_functions.md | 2 +- doc/quota_information.md | 2 +- 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/doc/API.md b/doc/API.md index da60037..a895121 100644 --- a/doc/API.md +++ b/doc/API.md @@ -9,6 +9,7 @@ The isoline functions provide a way to generate isolines in terms of distance an ## Documentation * [Quickstart](quickstart.md) -* [General concepts](general_concepts.md) -* [Geocoding functions](geocoding_functions.md) -* [Isoline functions](isoline_functions.md) +* [General Concepts](general_concepts.md) +* [Geocoding Functions](geocoding_functions.md) +* [Isoline Functions](isoline_functions.md) +* [Quota Information](quota_information.md) diff --git a/doc/general_concepts.md b/doc/general_concepts.md index 10f0a13..3b4baa1 100644 --- a/doc/general_concepts.md +++ b/doc/general_concepts.md @@ -1,4 +1,4 @@ -# General concepts +# General Concepts The Data Services API offers geocoding and isoline services on top of the CartoDB SQL API by means of a set of functions. Each one of these functions is oriented to one kind of operation and returns the corresponding geometry (a `polygon` or a `point`), according to the input information. diff --git a/doc/geocoding_functions.md b/doc/geocoding_functions.md index 6defcaa..228f056 100644 --- a/doc/geocoding_functions.md +++ b/doc/geocoding_functions.md @@ -1,4 +1,4 @@ -# Geocoding functions +# Geocoding Functions The following geocoding functions are available, grouped by categories. @@ -181,6 +181,8 @@ UPDATE {tablename} SET the_geom = cdb_geocode_namedplace_point({city_column}, {p The following functions provide a postal code geocoding service that can be used to obtain points or polygon results. The postal code polygon geocoder covers the United States, France, Australia and Canada; a request for a different country will return an empty response. +**Note:** For the USA, US Census [Zip Code Tabulation Areas](https://www.census.gov/geo/reference/zctas.html) (ZCTA) are used to reference geocodes for USPS postal codes service areas. + ### cdb_geocode_postalcode_polygon(_postal_code text, country_name text_) #### Arguments @@ -208,8 +210,6 @@ SELECT cdb_geocode_postalcode_polygon('11211', 'USA') UPDATE {tablename} SET the_geom = cdb_geocode_postalcode_polygon({postal_code_column}, 'USA') ``` -**Note:** For the USA, US Census ZCTAs are considered. - ### cdb_geocode_postalcode_point(_code text, country_name text_) #### Arguments diff --git a/doc/isoline_functions.md b/doc/isoline_functions.md index 630f205..395defd 100644 --- a/doc/isoline_functions.md +++ b/doc/isoline_functions.md @@ -1,4 +1,4 @@ -# Isoline functions +# Isoline Functions The following functions provide an isolines generator service based on time or distance. This service uses the isolines service defined for the user (currently, only the Here isolines service is available). diff --git a/doc/quota_information.md b/doc/quota_information.md index d3cfaba..900d4f0 100644 --- a/doc/quota_information.md +++ b/doc/quota_information.md @@ -1,4 +1,4 @@ -# Quota information +# Quota Information **This Data Services API provides functions which are subject to quota limitations, and extra fees may apply**. Please check our [terms and conditions](https://cartodb.com/terms/). From 0ca1b840962ca364533b62f0cfd9778bcf7fbdbf Mon Sep 17 00:00:00 2001 From: csobier Date: Mon, 21 Mar 2016 16:42:48 -0400 Subject: [PATCH 02/17] added link to docs faq regarding zcta --- doc/geocoding_functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/geocoding_functions.md b/doc/geocoding_functions.md index 228f056..46f6e4c 100644 --- a/doc/geocoding_functions.md +++ b/doc/geocoding_functions.md @@ -181,7 +181,7 @@ UPDATE {tablename} SET the_geom = cdb_geocode_namedplace_point({city_column}, {p The following functions provide a postal code geocoding service that can be used to obtain points or polygon results. The postal code polygon geocoder covers the United States, France, Australia and Canada; a request for a different country will return an empty response. -**Note:** For the USA, US Census [Zip Code Tabulation Areas](https://www.census.gov/geo/reference/zctas.html) (ZCTA) are used to reference geocodes for USPS postal codes service areas. +**Note:** For the USA, US Census [Zip Code Tabulation Areas](https://www.census.gov/geo/reference/zctas.html) (ZCTA) are used to reference geocodes for USPS postal codes service areas. See the [FAQs](http://docs.cartodb.com/faqs/datasets-and-data/#why-does-cartodb-use-census-bureau-zctas-and-not-usps-zip-codes-for-postal-codes) about datasets and data for details. ### cdb_geocode_postalcode_polygon(_postal_code text, country_name text_) From 03ce547b8e1120724cf93dcae24e8452d41c2ea8 Mon Sep 17 00:00:00 2001 From: Carla Date: Thu, 31 Mar 2016 15:02:19 +0200 Subject: [PATCH 03/17] Update README.md --- README.md | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 809cd71..12e0cbf 100644 --- a/README.md +++ b/README.md @@ -23,11 +23,11 @@ Steps to deploy a new Data Services API version : ### Local install instructions -- install data services extension +- install data services geocoder extension ``` git clone git@github.com:CartoDB/data-services.git - data-services/geocoder/extension + cd data-services/geocoder/extension sudo make install ``` @@ -35,7 +35,7 @@ Steps to deploy a new Data Services API version : ``` cd client && sudo make install - cd server && sudo make install + cd server/extension && sudo make install ``` - install python library @@ -57,21 +57,23 @@ Steps to deploy a new Data Services API version : ``` - # select CDB_Conf_SetConf('redis_metadata_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); - # select CDB_Conf_SetConf('redis_metrics_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); + # SELECT CDB_Conf_SetConf('redis_metadata_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); + # SELECT CDB_Conf_SetConf('redis_metrics_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); - # select CDB_Conf_SetConf('heremaps_conf', '{"app_id": "APP_ID", "app_code": "APP_CODE"}'); - # select CDB_Conf_SetConf('user_config', '{"is_organization": false, "entity_name": ""}') + # SELECT CDB_Conf_SetConf('heremaps_conf', '{"app_id": "APP_ID", "app_code": "APP_CODE"}'); + # SELECT CDB_Conf_SetConf('user_config', '{"is_organization": false, "entity_name": ""}') + # SELECT CDB_Conf_SetConf('mapzen_conf', '{"routing_app_key": "ROUTING_API_KEY", "geocoder_app_key": "GEOCODER_API_KEY"}'); + ``` - configure plproxy to point to the same user database (you could do in a different one) ``` - select CDB_Conf_SetConf('geocoder_server_config', '{ "connection_str": "host=localhost port=5432 dbname=cartodb_dev_user_accf0647-d942-4e37-b129-8287c117e687_db user=postgres"}'); + SELECT CDB_Conf_SetConf('geocoder_server_config', '{ "connection_str": "host=localhost port=5432 dbname=cartodb_dev_user_accf0647-d942-4e37-b129-8287c117e687_db user=postgres"}'); ``` -- configure the search path in order to be able to execute the functions without use the schema: +- configure the search path in order to be able to execute the functions without using the schema: ``` - alter role "" set search_path='"$user", public, cartodb, cdb_dataservices_client'; + ALTER ROLE "" SET search_path='"$user", public, cartodb, cdb_dataservices_client'; ``` From 2668a1dfdc47fbde35c4f29f231feed2e5ebd9b5 Mon Sep 17 00:00:00 2001 From: Carla Date: Thu, 31 Mar 2016 15:29:18 +0200 Subject: [PATCH 04/17] Update README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 12e0cbf..7b77ae4 100644 --- a/README.md +++ b/README.md @@ -57,13 +57,13 @@ Steps to deploy a new Data Services API version : ``` - # SELECT CDB_Conf_SetConf('redis_metadata_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); - # SELECT CDB_Conf_SetConf('redis_metrics_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); + # SELECT CDB_Conf_SetConf('redis_metadata_config', '{"redis_host": "localhost", "redis_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); + # SELECT CDB_Conf_SetConf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); - # SELECT CDB_Conf_SetConf('heremaps_conf', '{"app_id": "APP_ID", "app_code": "APP_CODE"}'); + # SELECT CDB_Conf_SetConf('heremaps_conf', '{"app_id": "APP_ID", "app_code": "APP_CODE", "geocoder_cost_per_hit": "COST_PER_HIT"}'); # SELECT CDB_Conf_SetConf('user_config', '{"is_organization": false, "entity_name": ""}') # SELECT CDB_Conf_SetConf('mapzen_conf', '{"routing_app_key": "ROUTING_API_KEY", "geocoder_app_key": "GEOCODER_API_KEY"}'); - + # SELECT CDB_Conf_SetConf('logger_con', '{"geocoder_log_path": "/tmp/geocodings.log"}') ``` - configure plproxy to point to the same user database (you could do in a different one) From 1505e0e6ea0a4810cc3c6d60d7da57011e784d83 Mon Sep 17 00:00:00 2001 From: Carla Iriberri Date: Thu, 31 Mar 2016 15:52:52 +0200 Subject: [PATCH 05/17] Make isoline credentials independent --- .../cartodb_services/metrics/config.py | 44 ++++++++++++------- server/lib/python/cartodb_services/setup.py | 2 +- 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index a729cb4..7731b0f 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -54,10 +54,10 @@ class IsolinesRoutingConfig(ServiceConfig): ROUTING_CONFIG_KEYS = ['here_isolines_quota', 'soft_here_isolines_limit', 'period_end_date', 'username', 'orgname', - 'heremaps_app_id', 'heremaps_app_code', + 'heremaps_isolines_app_id', 'heremaps_isolines_app_code', 'geocoder_type'] - NOKIA_APP_ID_KEY = 'heremaps_app_id' - NOKIA_APP_CODE_KEY = 'heremaps_app_code' + NOKIA_APP_ID_KEY = 'heremaps_isolines_app_id' + NOKIA_APP_CODE_KEY = 'heremaps_isolines_app_code' QUOTA_KEY = 'here_isolines_quota' SOFT_LIMIT_KEY = 'soft_here_isolines_limit' USERNAME_KEY = 'username' @@ -102,8 +102,8 @@ class IsolinesRoutingConfig(ServiceConfig): self._soft_isolines_limit = True else: self._soft_isolines_limit = False - self._heremaps_app_id = db_config.heremaps_app_id - self._heremaps_app_code = db_config.heremaps_app_code + self._heremaps_app_id = db_config.heremaps_isolines_app_id + self._heremaps_app_code = db_config.heremaps_isolines_app_code @property def service_type(self): @@ -168,12 +168,12 @@ class GeocoderConfig(ServiceConfig): GEOCODER_CONFIG_KEYS = ['google_maps_client_id', 'google_maps_api_key', 'geocoding_quota', 'soft_geocoding_limit', 'geocoder_type', 'period_end_date', - 'heremaps_app_id', 'heremaps_app_code', + 'heremaps_geocoder_app_id', 'heremaps_geocoder_app_code', 'mapzen_geocoder_app_key', 'username', 'orgname'] NOKIA_GEOCODER_REDIS_MANDATORY_KEYS = ['geocoding_quota', 'soft_geocoding_limit'] NOKIA_GEOCODER = 'heremaps' - NOKIA_GEOCODER_APP_ID_KEY = 'heremaps_app_id' - NOKIA_GEOCODER_APP_CODE_KEY = 'heremaps_app_code' + NOKIA_GEOCODER_APP_ID_KEY = 'heremaps_geocoder_app_id' + NOKIA_GEOCODER_APP_CODE_KEY = 'heremaps_geocoder_app_code' GOOGLE_GEOCODER = 'google' GOOGLE_GEOCODER_API_KEY = 'google_maps_api_key' GOOGLE_GEOCODER_CLIENT_ID = 'google_maps_client_id' @@ -241,8 +241,8 @@ class GeocoderConfig(ServiceConfig): else: self._soft_geocoding_limit = False if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER: - self._heremaps_app_id = db_config.heremaps_app_id - self._heremaps_app_code = db_config.heremaps_app_code + self._heremaps_app_id = db_config.heremaps_geocoder_app_id + self._heremaps_app_code = db_config.heremaps_geocoder_app_code self._cost_per_hit = db_config.heremaps_geocoder_cost_per_hit elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER: self._google_maps_api_key = filtered_config[self.GOOGLE_GEOCODER_API_KEY] @@ -338,10 +338,12 @@ class ServicesDBConfig: raise ConfigException('Here maps configuration missing') else: heremaps_conf = json.loads(heremaps_conf_json) - self._heremaps_app_id = heremaps_conf['app_id'] - self._heremaps_app_code = heremaps_conf['app_code'] - self._heremaps_geocoder_cost_per_hit = heremaps_conf[ + self._heremaps_geocoder_app_id = heremaps_conf['geocoder']['app_id'] + self._heremaps_geocoder_app_code = heremaps_conf['geocoder']['app_code'] + self._heremaps_geocoder_cost_per_hit = heremaps_conf['geocoder'][ 'geocoder_cost_per_hit'] + self._heremaps_isolines_app_id = heremaps_conf['isolines']['app_id'] + self._heremaps_isolines_app_code = heremaps_conf['isolines']['app_code'] def _get_mapzen_config(self): mapzen_conf_json = self._get_conf('mapzen_conf') @@ -369,12 +371,20 @@ class ServicesDBConfig: raise ConfigException("Malformed config for {0}: {1}".format(key, e)) @property - def heremaps_app_id(self): - return self._heremaps_app_id + def heremaps_isolines_app_id(self): + return self._heremaps_isolines_app_id @property - def heremaps_app_code(self): - return self._heremaps_app_code + def heremaps_isolines_app_code(self): + return self._heremaps_isolines_app_code + + @property + def heremaps_geocoder_app_id(self): + return self._heremaps_geocoder_app_id + + @property + def heremaps_geocoder_app_code(self): + return self._heremaps_geocoder_app_code @property def heremaps_geocoder_cost_per_hit(self): diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index 633f24c..de5dd35 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.4.0', + version='0.4.1', description='CartoDB Services API Python Library', From 7de498cc954ab4cb17978f5b3f0d5181cf345310 Mon Sep 17 00:00:00 2001 From: Carla Date: Thu, 31 Mar 2016 16:01:22 +0200 Subject: [PATCH 06/17] Add redis/sentinel examples --- README.md | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 7b77ae4..73d7841 100644 --- a/README.md +++ b/README.md @@ -57,23 +57,27 @@ Steps to deploy a new Data Services API version : ``` - # SELECT CDB_Conf_SetConf('redis_metadata_config', '{"redis_host": "localhost", "redis_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); - # SELECT CDB_Conf_SetConf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); - - # SELECT CDB_Conf_SetConf('heremaps_conf', '{"app_id": "APP_ID", "app_code": "APP_CODE", "geocoder_cost_per_hit": "COST_PER_HIT"}'); - # SELECT CDB_Conf_SetConf('user_config', '{"is_organization": false, "entity_name": ""}') - # SELECT CDB_Conf_SetConf('mapzen_conf', '{"routing_app_key": "ROUTING_API_KEY", "geocoder_app_key": "GEOCODER_API_KEY"}'); - # SELECT CDB_Conf_SetConf('logger_con', '{"geocoder_log_path": "/tmp/geocodings.log"}') + # If sentinel is used: + SELECT CDB_Conf_SetConf('redis_metadata_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); + SELECT CDB_Conf_SetConf('redis_metrics_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "", "timeout": 0.1, "redis_db": 5}'); + # If sentinel is not used + SELECT CDB_Conf_SetConf('redis_metadata_config', '{"redis_host": "localhost", "redis_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); + SELECT CDB_Conf_SetConf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 6379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); + + SELECT CDB_Conf_SetConf('heremaps_conf', '{"app_id": "APP_ID", "app_code": "APP_CODE", "geocoder_cost_per_hit": "COST_PER_HIT"}'); + SELECT CDB_Conf_SetConf('user_config', '{"is_organization": false, "entity_name": ""}') + SELECT CDB_Conf_SetConf('mapzen_conf', '{"routing_app_key": "ROUTING_API_KEY", "geocoder_app_key": "GEOCODER_API_KEY"}'); + SELECT CDB_Conf_SetConf('logger_con', '{"geocoder_log_path": "/tmp/geocodings.log"}') ``` -- configure plproxy to point to the same user database (you could do in a different one) +- configure plproxy to point to the a database (you can use a specific database for the server or your same user) ``` - SELECT CDB_Conf_SetConf('geocoder_server_config', '{ "connection_str": "host=localhost port=5432 dbname=cartodb_dev_user_accf0647-d942-4e37-b129-8287c117e687_db user=postgres"}'); + SELECT CDB_Conf_SetConf('geocoder_server_config', '{ "connection_str": "host=localhost port=5432 dbname= user=postgres"}'); ``` - configure the search path in order to be able to execute the functions without using the schema: ``` - ALTER ROLE "" SET search_path='"$user", public, cartodb, cdb_dataservices_client'; + ALTER ROLE "" SET search_path="$user", public, cartodb, cdb_dataservices_client; ``` From 42b1d6f630d17ee4dfbf90c75e6253fe93edfecd Mon Sep 17 00:00:00 2001 From: Carla Date: Thu, 31 Mar 2016 16:11:03 +0200 Subject: [PATCH 07/17] Update README.md --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 73d7841..c53057b 100644 --- a/README.md +++ b/README.md @@ -60,9 +60,10 @@ Steps to deploy a new Data Services API version : # If sentinel is used: SELECT CDB_Conf_SetConf('redis_metadata_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); SELECT CDB_Conf_SetConf('redis_metrics_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "", "timeout": 0.1, "redis_db": 5}'); + # If sentinel is not used - SELECT CDB_Conf_SetConf('redis_metadata_config', '{"redis_host": "localhost", "redis_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); - SELECT CDB_Conf_SetConf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 6379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); + SELECT CDB_Conf_SetConf('redis_metadata_config', '{"redis_host": "localhost", "redis_port": 26379, "sentinel_master_id": "", "timeout": 0.1, "redis_db": 5}'); + SELECT CDB_Conf_SetConf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 6379, "sentinel_master_id": "", "timeout": 0.1, "redis_db": 5}'); SELECT CDB_Conf_SetConf('heremaps_conf', '{"app_id": "APP_ID", "app_code": "APP_CODE", "geocoder_cost_per_hit": "COST_PER_HIT"}'); SELECT CDB_Conf_SetConf('user_config', '{"is_organization": false, "entity_name": ""}') From 5f74d012a6950c7a55b7b23cc338c79061981e9e Mon Sep 17 00:00:00 2001 From: Carla Date: Thu, 31 Mar 2016 16:22:02 +0200 Subject: [PATCH 08/17] Update 00_install_test files --- server/extension/test/expected/00_install_test.out | 2 +- server/extension/test/sql/00_install_test.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/extension/test/expected/00_install_test.out b/server/extension/test/expected/00_install_test.out index c69ea04..d6951fd 100644 --- a/server/extension/test/expected/00_install_test.out +++ b/server/extension/test/expected/00_install_test.out @@ -19,7 +19,7 @@ SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localh (1 row) -SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}'); +SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"geocoder": {"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "dummy_id", "app_code": "dummy_code"}}'); cdb_conf_setconf ------------------ diff --git a/server/extension/test/sql/00_install_test.sql b/server/extension/test/sql/00_install_test.sql index 2e3cb2e..d86e983 100644 --- a/server/extension/test/sql/00_install_test.sql +++ b/server/extension/test/sql/00_install_test.sql @@ -11,7 +11,7 @@ CREATE EXTENSION cdb_dataservices_server; -- Mock the redis server connection to point to this very test db SELECT cartodb.cdb_conf_setconf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}'); SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}'); -SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}'); +SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"geocoder": {"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "dummy_id", "app_code": "dummy_code"}}'); SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing_app_key": "dummy_key", "geocoder_app_key": "dummy_key"}'); SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}'); From 62da0eff2129d8c52416d77b9ffe93d36c5657dd Mon Sep 17 00:00:00 2001 From: Carla Date: Thu, 31 Mar 2016 16:45:27 +0200 Subject: [PATCH 09/17] Update test_helper.py --- server/lib/python/cartodb_services/test/test_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/test/test_helper.py b/server/lib/python/cartodb_services/test/test_helper.py index 9856aca..349b043 100644 --- a/server/lib/python/cartodb_services/test/test_helper.py +++ b/server/lib/python/cartodb_services/test/test_helper.py @@ -44,7 +44,7 @@ def build_plpy_mock(empty=False): def _plpy_execute_side_effect(*args, **kwargs): if args[0] == "SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as conf": - return [{'conf': '{"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}'}] + return [{'conf': '{"geocoder": {"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "app_id", "app_code": "code"}}'}] elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('mapzen_conf') as conf": return [{'conf': '{"routing_app_key": "app_key", "geocoder_app_key": "app_key"}'}] elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('logger_conf') as conf": From d625b26e573a293f5fa93ffe001153a73be45eb0 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 1 Apr 2016 08:45:09 +0200 Subject: [PATCH 10/17] Fixed timeout and random problems --- .../cartodb_services/cartodb_services/mapzen/qps.py | 1 + .../cartodb_services/tools/redis_tools.py | 10 ++++++---- server/lib/python/cartodb_services/setup.py | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/qps.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/qps.py index 40ff1c7..fe8ebb2 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/qps.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/qps.py @@ -1,4 +1,5 @@ import time +import random from datetime import datetime from exceptions import TimeoutException diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py b/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py index ab24a15..24ba3c8 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py @@ -23,14 +23,15 @@ class RedisConnection: ) else: conn = StrictRedis(host=self._config.host, port=self._config.port, - db=self._config.db) + db=self._config.db, + socket_timeout=self._config.timeout) return conn class RedisDBConfig: DEFAULT_USER_DB = 5 - DEFAULT_TIMEOUT = 2 # seconds + DEFAULT_TIMEOUT = 1.5 # seconds def __init__(self, key, db_conn): self._db_conn = db_conn @@ -63,8 +64,9 @@ class RedisDBConfig: self._sentinel_id = None def __str__(self): - return "Host: {0}, Port: {1}, Sentinel id: {2}, DB: {3}".format( - self.host, self.port, self.sentinel_id, self.db) + return "Host: {0}, Port: {1}, Sentinel id: {2}, DB: {3}, " \ + "Timeout: {4}".format(self.host, self.port, self.sentinel_id, + self.db, self.timeout) @property def host(self): diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index 633f24c..de5dd35 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.4.0', + version='0.4.1', description='CartoDB Services API Python Library', From 8c5a4d32dee48e6166a3307a2081fea8cf087842 Mon Sep 17 00:00:00 2001 From: Carla Date: Fri, 1 Apr 2016 10:15:27 +0200 Subject: [PATCH 11/17] Update setup.py --- server/lib/python/cartodb_services/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index de5dd35..9285caa 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.4.1', + version='0.4.2', description='CartoDB Services API Python Library', From 5c23a43efa061ec502965b7e801581fa2e4a48ee Mon Sep 17 00:00:00 2001 From: csobier Date: Fri, 1 Apr 2016 13:09:16 -0400 Subject: [PATCH 12/17] added link to HERE coverage --- doc/isoline_functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/isoline_functions.md b/doc/isoline_functions.md index 395defd..d56c6d8 100644 --- a/doc/isoline_functions.md +++ b/doc/isoline_functions.md @@ -1,6 +1,6 @@ # Isoline Functions -The following functions provide an isolines generator service based on time or distance. This service uses the isolines service defined for the user (currently, only the Here isolines service is available). +The following functions provide an isolines generator service based on time or distance. This service uses the isolines service defined for the user (currently, only the [HERE](https://developer.here.com/coverage-info) isolines service is available). **This service is subject to quota limitations, and extra fees may apply**. Please view our [terms and conditions](https://cartodb.com/terms/) and check out the [Quota information section](http://docs.cartodb.com/cartodb-platform/dataservices-api/quota-information/) for details and recommendations related with quota usage. From a8dd211acff8f996d5afa2e39a2bb147da204231 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Mon, 4 Apr 2016 10:11:37 +0200 Subject: [PATCH 13/17] Use retry on timeout flag for Redis connections In order to avoid some fails due some high load peaks in our Redis databases, we are going to active this flag that retries 1 more time in case it receives a socket.Timeout error while reading or connecting --- .../cartodb_services/tools/redis_tools.py | 11 +++++------ server/lib/python/cartodb_services/setup.py | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py b/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py index 24ba3c8..2b77531 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py @@ -16,14 +16,13 @@ class RedisConnection: sentinel = Sentinel([(self._config.host, self._config.port)], socket_timeout=self._config.timeout) - return sentinel.master_for( - self._config.sentinel_id, - socket_timeout=self._config.timeout, - db=self._config.db - ) + return sentinel.master_for(self._config.sentinel_id, + socket_timeout=self._config.timeout, + db=self._config.db, + retry_on_timeout=True) else: conn = StrictRedis(host=self._config.host, port=self._config.port, - db=self._config.db, + db=self._config.db, retry_on_timeout=True, socket_timeout=self._config.timeout) return conn diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index 9285caa..7c66b55 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.4.2', + version='0.4.3', description='CartoDB Services API Python Library', From 79e69c6020a87f3fc2810c1d5449e169e773057b Mon Sep 17 00:00:00 2001 From: Carla Date: Tue, 5 Apr 2016 11:46:31 +0200 Subject: [PATCH 14/17] Update geocoding_functions.md --- doc/geocoding_functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/geocoding_functions.md b/doc/geocoding_functions.md index 46f6e4c..f6e6d52 100644 --- a/doc/geocoding_functions.md +++ b/doc/geocoding_functions.md @@ -94,7 +94,7 @@ UPDATE {tablename} SET the_geom = cdb_geocode_admin1_polygon({province_column}, ## City geocoder -The following functions provide a city geocoder service. It is recommended to use the more specific geocoding function -- the one that requires more parameters — in order for the result to be as accurate as possible when several cities share their name. +The following functions provide a city geocoder service. It is recommended to use the more specific geocoding function -- the one that requires more parameters — in order for the result to be as accurate as possible when several cities share their name. In case of coincidence, the point for the city with the most population will be returned. ### cdb_geocode_namedplace_point(_city_name text_) From a0377150f359bccf6f401721059f90e868a76ced Mon Sep 17 00:00:00 2001 From: Carla Date: Tue, 5 Apr 2016 14:16:47 +0200 Subject: [PATCH 15/17] Update geocoding_functions.md --- doc/geocoding_functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/geocoding_functions.md b/doc/geocoding_functions.md index f6e6d52..f913509 100644 --- a/doc/geocoding_functions.md +++ b/doc/geocoding_functions.md @@ -94,7 +94,7 @@ UPDATE {tablename} SET the_geom = cdb_geocode_admin1_polygon({province_column}, ## City geocoder -The following functions provide a city geocoder service. It is recommended to use the more specific geocoding function -- the one that requires more parameters — in order for the result to be as accurate as possible when several cities share their name. In case of coincidence, the point for the city with the most population will be returned. +The following functions provide a city geocoder service. It is recommended to use the more specific geocoding function -- the one that requires more parameters — in order for the result to be as accurate as possible when several cities share their name. If there are duplicate results for a city name, the city name with the highest population will be returned. ### cdb_geocode_namedplace_point(_city_name text_) From b4c1991c90055285372555abc0280101db8a3065 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 13 Apr 2016 18:12:04 +0200 Subject: [PATCH 16/17] Mapzen geocoder checks for the iso3 for the country passed --- server/extension/sql/20_geocode_street.sql | 5 ++++- .../cartodb_services/mapzen/geocoder.py | 19 +++++++++++++++---- .../cartodb_services/mapzen/types.py | 14 ++++++++++++++ server/lib/python/cartodb_services/setup.py | 2 +- 4 files changed, 34 insertions(+), 6 deletions(-) diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index 5e3fc2b..ae34242 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -97,7 +97,10 @@ RETURNS Geometry AS $$ try: geocoder = MapzenGeocoder(user_geocoder_config.mapzen_app_key) - coordinates = geocoder.geocode(searchtext=searchtext, country=country) + country_iso3 = None + if country: + country_iso3 = country_to_iso3(country) + coordinates = geocoder.geocode(searchtext=searchtext, country=country_iso3) if coordinates: quota_service.increment_success_service_use() plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py index be98761..9adbfea 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py @@ -17,8 +17,8 @@ class MapzenGeocoder: self._url = base_url @qps_retry - def geocode(self, searchtext, country=None): - request_params = self._build_requests_parameters(searchtext, country) + def geocode(self, searchtext, city=None, state_province=None, country=None): + request_params = self._build_requests_parameters(searchtext, city, state_province, country) response = requests.get(self._url, params=request_params) if response.status_code == requests.codes.ok: return self.__parse_response(response.text) @@ -27,15 +27,26 @@ class MapzenGeocoder: else: response.raise_for_status() - def _build_requests_parameters(self, searchtext, country=None): + def _build_requests_parameters(self, searchtext, city=None, + state_province=None, country=None): request_params = {} - request_params['text'] = searchtext + search_string = self._build_search_text(searchtext, city, state_province) + request_params['text'] = search_string request_params['layers'] = 'address' request_params['api_key'] = self._app_key if country: request_params['boundary.country'] = country return request_params + def _build_search_text(self, searchtext, city, state_province): + search_string = searchtext + if city: + search_string = "{0}, {1}".format(search_string, city) + if state_province: + search_string = "{0}, {1}".format(search_string, state_province) + + return search_string + def __parse_response(self, response): try: parsed_json_response = json.loads(response) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/types.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/types.py index fa3aeed..df82aba 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/types.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/types.py @@ -18,3 +18,17 @@ def polyline_to_linestring(polyline): geometry = None return geometry + +def country_to_iso3(country): + """ Convert country to its iso3 code """ + try: + country_plan = plpy.prepare("SELECT adm0_a3 as iso3 FROM admin0_synonyms WHERE lower(regexp_replace($1, " \ + "'[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text = name_; ", ['text']) + country_result = plpy.execute(country_plan, [country], 1) + if country_result: + return country_result[0]['iso3'] + else: + return None + except BaseException as e: + plpy.warning("Can't get the iso3 code from {0}: {1}".format(country, e)) + return None diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index 7c66b55..f3f0f04 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.4.3', + version='0.4.4', description='CartoDB Services API Python Library', From cb921614c3391241d88e182d545eae075db93aa9 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 14 Apr 2016 07:46:28 +0200 Subject: [PATCH 17/17] Version 0.6.1 sql files --- server/extension/Makefile | 4 +- .../cdb_dataservices_server--0.6.0--0.6.1.sql | 39 +++++++++++++++++++ .../cdb_dataservices_server--0.6.1--0.6.0.sql | 33 ++++++++++++++++ ...sql => cdb_dataservices_server--0.6.1.sql} | 37 ++++++++++++++++++ .../extension/cdb_dataservices_server.control | 2 +- .../cdb_dataservices_server--0.5.2--0.6.0.sql | 0 .../cdb_dataservices_server--0.6.0--0.5.2.sql | 0 .../cdb_dataservices_server--0.6.0.sql | 0 8 files changed, 112 insertions(+), 3 deletions(-) create mode 100644 server/extension/cdb_dataservices_server--0.6.0--0.6.1.sql create mode 100644 server/extension/cdb_dataservices_server--0.6.1--0.6.0.sql rename server/extension/{cdb_dataservices_server--0.5.2.sql => cdb_dataservices_server--0.6.1.sql} (95%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.5.2--0.6.0.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.6.0--0.5.2.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.6.0.sql (100%) diff --git a/server/extension/Makefile b/server/extension/Makefile index 6dd6db0..d764817 100644 --- a/server/extension/Makefile +++ b/server/extension/Makefile @@ -13,8 +13,8 @@ OLD_VERSIONS = $(wildcard old_versions/*.sql) # @see http://www.postgresql.org/docs/current/static/extend-pgxs.html DATA = $(NEW_EXTENSION_ARTIFACT) \ $(OLD_VERSIONS) \ - cdb_dataservices_server--0.6.0--0.5.2.sql \ - cdb_dataservices_server--0.5.2--0.6.0.sql + cdb_dataservices_server--0.6.1--0.6.0.sql \ + cdb_dataservices_server--0.6.0--0.6.1.sql REGRESS = $(notdir $(basename $(wildcard test/sql/*test.sql))) TEST_DIR = test/ diff --git a/server/extension/cdb_dataservices_server--0.6.0--0.6.1.sql b/server/extension/cdb_dataservices_server--0.6.0--0.6.1.sql new file mode 100644 index 0000000..7f3b5c2 --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.6.0--0.6.1.sql @@ -0,0 +1,39 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.6.1'" to load this file. \quit +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.mapzen import MapzenGeocoder + from cartodb_services.mapzen.types import country_to_iso3 + from cartodb_services.metrics import QuotaService + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + quota_service = QuotaService(user_geocoder_config, redis_conn) + + try: + geocoder = MapzenGeocoder(user_geocoder_config.mapzen_app_key) + country_iso3 = None + if country: + country_iso3 = country_to_iso3(country) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, + state_province=state_province, + country=country_iso3) + if coordinates: + quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using mapzen geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; \ No newline at end of file diff --git a/server/extension/cdb_dataservices_server--0.6.1--0.6.0.sql b/server/extension/cdb_dataservices_server--0.6.1--0.6.0.sql new file mode 100644 index 0000000..5ac3380 --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.6.1--0.6.0.sql @@ -0,0 +1,33 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.6.0'" to load this file. \quit +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.mapzen import MapzenGeocoder + from cartodb_services.metrics import QuotaService + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + quota_service = QuotaService(user_geocoder_config, redis_conn) + + try: + geocoder = MapzenGeocoder(user_geocoder_config.mapzen_app_key) + coordinates = geocoder.geocode(searchtext=searchtext, country=country) + if coordinates: + quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using mapzen geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; \ No newline at end of file diff --git a/server/extension/cdb_dataservices_server--0.5.2.sql b/server/extension/cdb_dataservices_server--0.6.1.sql similarity index 95% rename from server/extension/cdb_dataservices_server--0.5.2.sql rename to server/extension/cdb_dataservices_server--0.6.1.sql index d2906fb..8c8989a 100644 --- a/server/extension/cdb_dataservices_server--0.5.2.sql +++ b/server/extension/cdb_dataservices_server--0.6.1.sql @@ -175,6 +175,9 @@ RETURNS Geometry AS $$ elif user_geocoder_config.google_geocoder: google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapzen_geocoder: + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] else: plpy.error('Requested geocoder is not available') @@ -245,6 +248,40 @@ RETURNS Geometry AS $$ finally: quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.mapzen import MapzenGeocoder + from cartodb_services.metrics import QuotaService + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + quota_service = QuotaService(user_geocoder_config, redis_conn) + + try: + geocoder = MapzenGeocoder(user_geocoder_config.mapzen_app_key) + country_iso3 = None + if country: + country_iso3 = country_to_iso3(country) + coordinates = geocoder.geocode(searchtext=searchtext, country=country_iso3) + if coordinates: + quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_service_use() + error_msg = 'There was an error trying to geocode using mapzen geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService diff --git a/server/extension/cdb_dataservices_server.control b/server/extension/cdb_dataservices_server.control index f0182bc..682ea2a 100644 --- a/server/extension/cdb_dataservices_server.control +++ b/server/extension/cdb_dataservices_server.control @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices server extension' -default_version = '0.6.0' +default_version = '0.6.1' requires = 'plpythonu, postgis, cdb_geocoder' superuser = true schema = cdb_dataservices_server diff --git a/server/extension/cdb_dataservices_server--0.5.2--0.6.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.5.2--0.6.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.5.2--0.6.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.5.2--0.6.0.sql diff --git a/server/extension/cdb_dataservices_server--0.6.0--0.5.2.sql b/server/extension/old_versions/cdb_dataservices_server--0.6.0--0.5.2.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.6.0--0.5.2.sql rename to server/extension/old_versions/cdb_dataservices_server--0.6.0--0.5.2.sql diff --git a/server/extension/cdb_dataservices_server--0.6.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.6.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.6.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.6.0.sql