From d27b2a2b7878dbb68dd94833300fcba68bc91b10 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 26 Feb 2016 10:42:05 +0100 Subject: [PATCH 01/12] Block google services users to use isolines functions Due some service restrictions, google services users couldn't use the HERE services which includes the isolines functions. --- .../extension/cdb_dataservices_server--0.3.0--0.4.0.sql | 6 ++++++ server/extension/cdb_dataservices_server--0.4.0.sql | 6 ++++++ server/extension/sql/0.4.0/85_isodistance.sql | 3 +++ server/extension/sql/0.4.0/90_isochrone.sql | 3 +++ .../cartodb_services/cartodb_services/metrics/config.py | 9 ++++++++- 5 files changed, 26 insertions(+), 1 deletion(-) diff --git a/server/extension/cdb_dataservices_server--0.3.0--0.4.0.sql b/server/extension/cdb_dataservices_server--0.3.0--0.4.0.sql index 221898d..d56be82 100644 --- a/server/extension/cdb_dataservices_server--0.3.0--0.4.0.sql +++ b/server/extension/cdb_dataservices_server--0.3.0--0.4.0.sql @@ -87,6 +87,9 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] type = 'isodistance' + if user_isolines_config.google_services_user: + plpy.error('This service is not available for google service users.') + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) isolines = [] @@ -106,6 +109,9 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] type = 'isochrone' + if user_isolines_config.google_services_user: + plpy.error('This service is not available for google service users.') + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) isolines = [] diff --git a/server/extension/cdb_dataservices_server--0.4.0.sql b/server/extension/cdb_dataservices_server--0.4.0.sql index 85cffae..5aa3aac 100644 --- a/server/extension/cdb_dataservices_server--0.4.0.sql +++ b/server/extension/cdb_dataservices_server--0.4.0.sql @@ -866,6 +866,9 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] type = 'isodistance' + if user_isolines_config.google_services_user: + plpy.error('This service is not available for google service users.') + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) isolines = [] @@ -884,6 +887,9 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] type = 'isochrone' + if user_isolines_config.google_services_user: + plpy.error('This service is not available for google service users.') + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) isolines = [] diff --git a/server/extension/sql/0.4.0/85_isodistance.sql b/server/extension/sql/0.4.0/85_isodistance.sql index c6d99ae..3c3ac0e 100644 --- a/server/extension/sql/0.4.0/85_isodistance.sql +++ b/server/extension/sql/0.4.0/85_isodistance.sql @@ -6,6 +6,9 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] type = 'isodistance' + if user_isolines_config.google_services_user: + plpy.error('This service is not available for google service users.') + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) isolines = [] diff --git a/server/extension/sql/0.4.0/90_isochrone.sql b/server/extension/sql/0.4.0/90_isochrone.sql index 5ca7272..6237bd7 100644 --- a/server/extension/sql/0.4.0/90_isochrone.sql +++ b/server/extension/sql/0.4.0/90_isochrone.sql @@ -6,6 +6,9 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] type = 'isochrone' + if user_isolines_config.google_services_user: + plpy.error('This service is not available for google service users.') + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) isolines = [] diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index 484cbb7..2e395c5 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -32,7 +32,7 @@ class IsolinesRoutingConfig(ServiceConfig): ROUTING_CONFIG_KEYS = ['here_isolines_quota', 'soft_here_isolines_limit', 'period_end_date', 'username', 'orgname', - 'heremaps_app_id', 'heremaps_app_code'] + 'heremaps_app_id', 'heremaps_app_code', 'geocoder_type'] NOKIA_APP_ID_KEY = 'heremaps_app_id' NOKIA_APP_CODE_KEY = 'heremaps_app_code' QUOTA_KEY = 'here_isolines_quota' @@ -40,6 +40,8 @@ class IsolinesRoutingConfig(ServiceConfig): USERNAME_KEY = 'username' ORGNAME_KEY = 'orgname' PERIOD_END_DATE = 'period_end_date' + GEOCODER_TYPE_KEY = 'geocoder_type' + GOOGLE_GEOCODER = 'google' def __init__(self, redis_connection, username, orgname=None, heremaps_app_id=None, heremaps_app_code=None): @@ -74,6 +76,7 @@ class IsolinesRoutingConfig(ServiceConfig): user_config[self.PERIOD_END_DATE] = org_config[self.PERIOD_END_DATE] def __parse_config(self, filtered_config): + self._geocoder_type = filtered_config[self.GEOCODER_TYPE_KEY].lower() self._isolines_quota = float(filtered_config[self.QUOTA_KEY]) self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE]) if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true': @@ -107,6 +110,10 @@ class IsolinesRoutingConfig(ServiceConfig): def heremaps_app_code(self): return self._heremaps_app_code + @property + def google_services_user(self): + return self._geocoder_type == self.GOOGLE_GEOCODER + class InternalGeocoderConfig(ServiceConfig): From 93463b30329d3f0e00b8db3347fe1a8d6b429f5e Mon Sep 17 00:00:00 2001 From: Luis Bosque Date: Wed, 17 Feb 2016 18:13:24 +0100 Subject: [PATCH 02/12] Support using a common redis connection If sentinel_host or sentinel_cluster_id is not provided it will try to connect with a redis_host parameter --- server/extension/Makefile | 15 +- .../cdb_dataservices_server--0.4.0--0.5.0.sql | 52 + .../cdb_dataservices_server--0.5.0--0.4.0.sql | 49 + .../cdb_dataservices_server--0.5.0.sql | 912 ++++++++++++++++++ .../extension/cdb_dataservices_server.control | 2 +- server/extension/sql/0.5.0/00_header.sql | 3 + .../extension/sql/0.5.0/10_redis_helper.sql | 61 ++ .../extension/sql/0.5.0/15_config_helper.sql | 51 + .../extension/sql/0.5.0/20_geocode_street.sql | 84 ++ server/extension/sql/0.5.0/30_admin0.sql | 50 + server/extension/sql/0.5.0/40_admin1.sql | 117 +++ server/extension/sql/0.5.0/50_namedplaces.sql | 164 ++++ server/extension/sql/0.5.0/60_postalcodes.sql | 219 +++++ server/extension/sql/0.5.0/70_ips.sql | 61 ++ .../extension/sql/0.5.0/80_routing_helper.sql | 51 + server/extension/sql/0.5.0/85_isodistance.sql | 18 + server/extension/sql/0.5.0/90_isochrone.sql | 18 + .../sql/0.5.0/99_geocoder_server_user.sql | 15 + .../test/0.5.0/expected/00_install_test.out | 36 + .../test/0.5.0/expected/20_street_test.out | 12 + .../test/0.5.0/expected/30_admin0_test.out | 47 + .../test/0.5.0/expected/40_admin1_test.out | 81 ++ .../0.5.0/expected/50_namedplaces_test.out | 136 +++ .../0.5.0/expected/60_postalcodes_test.out | 163 ++++ .../test/0.5.0/expected/70_ips_test.out | 40 + .../0.5.0/expected/85_isodistance_test.out | 12 + .../test/0.5.0/expected/90_isochrone_test.out | 12 + .../99_remove_geocoder_api_user_test.out | 5 + .../test/0.5.0/sql/00_install_test.sql | 25 + .../test/0.5.0/sql/20_street_test.sql | 7 + .../test/0.5.0/sql/30_admin0_test.sql | 32 + .../test/0.5.0/sql/40_admin1_test.sql | 48 + .../test/0.5.0/sql/50_namedplaces_test.sql | 72 ++ .../test/0.5.0/sql/60_postalcodes_test.sql | 117 +++ .../extension/test/0.5.0/sql/70_ips_test.sql | 24 + .../test/0.5.0/sql/85_isodistance_test.sql | 7 + .../test/0.5.0/sql/90_isochrone_test.sql | 7 + .../sql/99_remove_geocoder_api_user_test.sql | 5 + .../cartodb_services/tools/redis_tools.py | 25 +- 39 files changed, 2839 insertions(+), 16 deletions(-) create mode 100644 server/extension/cdb_dataservices_server--0.4.0--0.5.0.sql create mode 100644 server/extension/cdb_dataservices_server--0.5.0--0.4.0.sql create mode 100644 server/extension/cdb_dataservices_server--0.5.0.sql create mode 100644 server/extension/sql/0.5.0/00_header.sql create mode 100644 server/extension/sql/0.5.0/10_redis_helper.sql create mode 100644 server/extension/sql/0.5.0/15_config_helper.sql create mode 100644 server/extension/sql/0.5.0/20_geocode_street.sql create mode 100644 server/extension/sql/0.5.0/30_admin0.sql create mode 100644 server/extension/sql/0.5.0/40_admin1.sql create mode 100644 server/extension/sql/0.5.0/50_namedplaces.sql create mode 100644 server/extension/sql/0.5.0/60_postalcodes.sql create mode 100644 server/extension/sql/0.5.0/70_ips.sql create mode 100644 server/extension/sql/0.5.0/80_routing_helper.sql create mode 100644 server/extension/sql/0.5.0/85_isodistance.sql create mode 100644 server/extension/sql/0.5.0/90_isochrone.sql create mode 100644 server/extension/sql/0.5.0/99_geocoder_server_user.sql create mode 100644 server/extension/test/0.5.0/expected/00_install_test.out create mode 100644 server/extension/test/0.5.0/expected/20_street_test.out create mode 100644 server/extension/test/0.5.0/expected/30_admin0_test.out create mode 100644 server/extension/test/0.5.0/expected/40_admin1_test.out create mode 100644 server/extension/test/0.5.0/expected/50_namedplaces_test.out create mode 100644 server/extension/test/0.5.0/expected/60_postalcodes_test.out create mode 100644 server/extension/test/0.5.0/expected/70_ips_test.out create mode 100644 server/extension/test/0.5.0/expected/85_isodistance_test.out create mode 100644 server/extension/test/0.5.0/expected/90_isochrone_test.out create mode 100644 server/extension/test/0.5.0/expected/99_remove_geocoder_api_user_test.out create mode 100644 server/extension/test/0.5.0/sql/00_install_test.sql create mode 100644 server/extension/test/0.5.0/sql/20_street_test.sql create mode 100644 server/extension/test/0.5.0/sql/30_admin0_test.sql create mode 100644 server/extension/test/0.5.0/sql/40_admin1_test.sql create mode 100644 server/extension/test/0.5.0/sql/50_namedplaces_test.sql create mode 100644 server/extension/test/0.5.0/sql/60_postalcodes_test.sql create mode 100644 server/extension/test/0.5.0/sql/70_ips_test.sql create mode 100644 server/extension/test/0.5.0/sql/85_isodistance_test.sql create mode 100644 server/extension/test/0.5.0/sql/90_isochrone_test.sql create mode 100644 server/extension/test/0.5.0/sql/99_remove_geocoder_api_user_test.sql diff --git a/server/extension/Makefile b/server/extension/Makefile index 337931b..0d8091c 100644 --- a/server/extension/Makefile +++ b/server/extension/Makefile @@ -15,14 +15,17 @@ DATA = $(NEW_EXTENSION_ARTIFACT) \ cdb_dataservices_server--0.1.0.sql \ cdb_dataservices_server--0.2.0.sql \ cdb_dataservices_server--0.3.0.sql \ + cdb_dataservices_server--0.4.0.sql \ cdb_dataservices_server--0.1.0--0.0.1.sql \ cdb_dataservices_server--0.0.1--0.1.0.sql \ - cdb_dataservices_server--0.2.0--0.1.0.sql \ - cdb_dataservices_server--0.1.0--0.2.0.sql \ - cdb_dataservices_server--0.2.0--0.3.0.sql \ - cdb_dataservices_server--0.3.0--0.2.0.sql \ - cdb_dataservices_server--0.3.0--0.4.0.sql \ - cdb_dataservices_server--0.4.0--0.3.0.sql + cdb_dataservices_server--0.2.0--0.1.0.sql \ + cdb_dataservices_server--0.1.0--0.2.0.sql \ + cdb_dataservices_server--0.2.0--0.3.0.sql \ + cdb_dataservices_server--0.3.0--0.2.0.sql \ + cdb_dataservices_server--0.3.0--0.4.0.sql \ + cdb_dataservices_server--0.4.0--0.3.0.sql \ + cdb_dataservices_server--0.5.0--0.4.0.sql \ + cdb_dataservices_server--0.4.0--0.5.0.sql REGRESS = $(notdir $(basename $(wildcard test/$(EXTVERSION)/sql/*test.sql))) TEST_DIR = test/$(EXTVERSION) diff --git a/server/extension/cdb_dataservices_server--0.4.0--0.5.0.sql b/server/extension/cdb_dataservices_server--0.4.0--0.5.0.sql new file mode 100644 index 0000000..e6b5295 --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.4.0--0.5.0.sql @@ -0,0 +1,52 @@ +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_redis_conf_v2(config_key text) +RETURNS cdb_dataservices_server._redis_conf_params AS $$ + conf_query = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(config_key) + conf = plpy.execute(conf_query)[0]['conf'] + if conf is None: + plpy.error("There is no redis configuration defined") + else: + import json + params = json.loads(conf) + return { + "sentinel_host": params['sentinel_host'], + "sentinel_port": params['sentinel_port'], + "sentinel_master_id": params['sentinel_master_id'], + "redis_host": params['redis_host'] + "timeout": params['timeout'], + "redis_db": params['redis_db'] + } +$$ LANGUAGE plpythonu; + +-- Get the connection to redis from cache or create a new one +CREATE OR REPLACE FUNCTION cdb_dataservices_server._connect_to_redis(user_id text) +RETURNS boolean AS $$ + cache_key = "redis_connection_{0}".format(user_id) + if cache_key in GD: + return False + else: + from cartodb_services.tools import RedisConnection + metadata_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, + c.sentinel_master_id, c.redis_host, c.timeout, c.redis_db + from cdb_dataservices_server._get_redis_conf_v2('redis_metadata_config') c;""")[0] + metrics_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, + c.sentinel_master_id, c.redis_host, c.timeout, c.redis_db + from cdb_dataservices_server._get_redis_conf_v2('redis_metrics_config') c;""")[0] + redis_metadata_connection = RedisConnection(metadata_config_params['sentinel_host'], + metadata_config_params['sentinel_port'], + metadata_config_params['sentinel_master_id'], + metadata_config_params['redis_host'], + timeout=metadata_config_params['timeout'], + redis_db=metadata_config_params['redis_db']).redis_connection() + redis_metrics_connection = RedisConnection(metrics_config_params['sentinel_host'], + metrics_config_params['sentinel_port'], + metrics_config_params['sentinel_master_id'], + metrics_config_params['redis_host'], + timeout=metrics_config_params['timeout'], + redis_db=metrics_config_params['redis_db']).redis_connection() + GD[cache_key] = { + 'redis_metadata_connection': redis_metadata_connection, + 'redis_metrics_connection': redis_metrics_connection, + } + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; diff --git a/server/extension/cdb_dataservices_server--0.5.0--0.4.0.sql b/server/extension/cdb_dataservices_server--0.5.0--0.4.0.sql new file mode 100644 index 0000000..83b5827 --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.5.0--0.4.0.sql @@ -0,0 +1,49 @@ +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_redis_conf_v2(config_key text) +RETURNS cdb_dataservices_server._redis_conf_params AS $$ + conf_query = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(config_key) + conf = plpy.execute(conf_query)[0]['conf'] + if conf is None: + plpy.error("There is no redis configuration defined") + else: + import json + params = json.loads(conf) + return { + "sentinel_host": params['sentinel_host'], + "sentinel_port": params['sentinel_port'], + "sentinel_master_id": params['sentinel_master_id'], + "timeout": params['timeout'], + "redis_db": params['redis_db'] + } +$$ LANGUAGE plpythonu; + +-- Get the connection to redis from cache or create a new one +CREATE OR REPLACE FUNCTION cdb_dataservices_server._connect_to_redis(user_id text) +RETURNS boolean AS $$ + cache_key = "redis_connection_{0}".format(user_id) + if cache_key in GD: + return False + else: + from cartodb_services.tools import RedisConnection + metadata_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, + c.sentinel_master_id, c.timeout, c.redis_db + from cdb_dataservices_server._get_redis_conf_v2('redis_metadata_config') c;""")[0] + metrics_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, + c.sentinel_master_id, c.timeout, c.redis_db + from cdb_dataservices_server._get_redis_conf_v2('redis_metrics_config') c;""")[0] + redis_metadata_connection = RedisConnection(metadata_config_params['sentinel_host'], + metadata_config_params['sentinel_port'], + metadata_config_params['sentinel_master_id'], + timeout=metadata_config_params['timeout'], + redis_db=metadata_config_params['redis_db']).redis_connection() + redis_metrics_connection = RedisConnection(metrics_config_params['sentinel_host'], + metrics_config_params['sentinel_port'], + metrics_config_params['sentinel_master_id'], + timeout=metrics_config_params['timeout'], + redis_db=metrics_config_params['redis_db']).redis_connection() + GD[cache_key] = { + 'redis_metadata_connection': redis_metadata_connection, + 'redis_metrics_connection': redis_metrics_connection, + } + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; diff --git a/server/extension/cdb_dataservices_server--0.5.0.sql b/server/extension/cdb_dataservices_server--0.5.0.sql new file mode 100644 index 0000000..05bac88 --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.5.0.sql @@ -0,0 +1,912 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_dataservices_server" to load this file. \quit +CREATE TYPE cdb_dataservices_server._redis_conf_params AS ( + sentinel_host text, + sentinel_port int, + sentinel_master_id text, + redis_host text, + redis_db text, + timeout float +); + +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_redis_conf_v2(config_key text) +RETURNS cdb_dataservices_server._redis_conf_params AS $$ + conf_query = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(config_key) + conf = plpy.execute(conf_query)[0]['conf'] + if conf is None: + plpy.error("There is no redis configuration defined") + else: + import json + params = json.loads(conf) + return { + "sentinel_host": params['sentinel_host'], + "sentinel_port": params['sentinel_port'], + "sentinel_master_id": params['sentinel_master_id'], + "redis_host": params['redis_host'], + "timeout": params['timeout'], + "redis_db": params['redis_db'] + } +$$ LANGUAGE plpythonu; + +-- Get the connection to redis from cache or create a new one +CREATE OR REPLACE FUNCTION cdb_dataservices_server._connect_to_redis(user_id text) +RETURNS boolean AS $$ + cache_key = "redis_connection_{0}".format(user_id) + if cache_key in GD: + return False + else: + from cartodb_services.tools import RedisConnection + metadata_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, + c.sentinel_master_id, c.redis_host, c.timeout, c.redis_db + from cdb_dataservices_server._get_redis_conf_v2('redis_metadata_config') c;""")[0] + metrics_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, + c.sentinel_master_id, c.redis_host, c.timeout, c.redis_db + from cdb_dataservices_server._get_redis_conf_v2('redis_metrics_config') c;""")[0] + redis_metadata_connection = RedisConnection(metadata_config_params['sentinel_host'], + metadata_config_params['sentinel_port'], + metadata_config_params['sentinel_master_id'], + metadata_config_params['redis_host'], + timeout=metadata_config_params['timeout'], + redis_db=metadata_config_params['redis_db']).redis_connection() + redis_metrics_connection = RedisConnection(metrics_config_params['sentinel_host'], + metrics_config_params['sentinel_port'], + metrics_config_params['sentinel_master_id'], + metrics_config_params['redis_host'], + timeout=metrics_config_params['timeout'], + redis_db=metrics_config_params['redis_db']).redis_connection() + GD[cache_key] = { + 'redis_metadata_connection': redis_metadata_connection, + 'redis_metrics_connection': redis_metrics_connection, + } + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + import json + from cartodb_services.metrics import GeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf'] + if not heremaps_conf_json: + heremaps_app_id = None + heremaps_app_code = None + else: + heremaps_conf = json.loads(heremaps_conf_json) + heremaps_app_id = heremaps_conf['app_id'] + heremaps_app_code = heremaps_conf['app_code'] + geocoder_config = GeocoderConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code) + # --Think about the security concerns with this kind of global cache, it should be only available + # --for this user session but... + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + import json + from cartodb_services.metrics import RoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf'] + if not heremaps_conf_json: + heremaps_app_id = None + heremaps_app_code = None + else: + heremaps_conf = json.loads(heremaps_conf_json) + heremaps_app_id = heremaps_conf['app_id'] + heremaps_app_code = heremaps_conf['app_code'] + routing_config = RoutingConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code) + # --Think about the security concerns with this kind of global cache, it should be only available + # --for this user session but... + GD[cache_key] = routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; +-- Geocodes a street address given a searchtext and a state and/or country +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + plpy.error('Requested geocoder is not available') + +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.here import HereMapsGeocoder + from cartodb_services.metrics import QuotaService + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + # -- Check the quota + quota_service = QuotaService(user_geocoder_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reach the limit of your quota') + + try: + geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + quota_service.increment_success_geocoder_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using here maps geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.google import GoogleMapsGeocoder + from cartodb_services.metrics import QuotaService + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + quota_service = QuotaService(user_geocoder_config, redis_conn) + + try: + geocoder = GoogleMapsGeocoder(user_geocoder_config.google_client_id, user_geocoder_config.google_api_key) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + quota_service.increment_success_geocoder_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using google maps geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin0_polygon(country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT n.the_geom as geom INTO ret + FROM (SELECT q, lower(regexp_replace(q, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text x + FROM (SELECT country_name q) g) d + LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x + LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.the_geom, s.adm0_a3; + + RETURN ret; + END +$$ LANGUAGE plpgsql; +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [admin1_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"]) + rv = plpy.execute(plan, [admin1_name, country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT q, ( + SELECT the_geom + FROM global_province_polygons + WHERE d.c = ANY (synonyms) + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM ( + SELECT + trim(replace(lower(admin1_name),'.',' ')) c, admin1_name q + ) d + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + WITH p AS (SELECT r.c, r.q, (SELECT iso3 FROM country_decoder WHERE lower(country_name) = ANY (synonyms)) i FROM (SELECT trim(replace(lower(admin1_name),'.',' ')) c, country_name q) r) + SELECT + geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_province_polygons + WHERE p.c = ANY (synonyms) + AND iso3 = p.i + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM p) n; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [city_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"]) + rv = plpy.execute(plan, [city_name, country_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"]) + rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH best AS (SELECT s AS q, (SELECT the_geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT city_name as s) p), + next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT city_name as s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT q, geom, TRUE AS success FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH p AS (SELECT r.s, r.c, (SELECT iso2 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT city_name AS s, country_name::text AS c) r), + best AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE c = p.c AND geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH inputcountry AS ( + SELECT iso2 as isoTwo FROM country_decoder WHERE lower(country_name) = ANY (synonyms) LIMIT 1 + ), + p AS ( + SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder, inputcountry WHERE lower(r.a1) = ANY (synonyms) AND admin1_decoder.iso2 = inputcountry.isoTwo LIMIT 1) i FROM (SELECT city_name AS s, admin1_name::text AS a1) r), + best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"]) + rv = plpy.execute(plan, [ip], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_ipaddress_point(ip text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + + new_ip INET; + BEGIN + BEGIN + IF family(ip::inet) = 6 THEN + new_ip := ip::inet; + ELSE + new_ip := ('::ffff:' || ip)::inet; + END IF; + EXCEPTION WHEN OTHERS THEN + SELECT NULL as geom INTO ret; + RETURN ret; + END; + + WITH + ips AS (SELECT ip s, new_ip net), + matches AS (SELECT s, (SELECT the_geom FROM ip_address_locations WHERE network_start_ip <= ips.net ORDER BY network_start_ip DESC LIMIT 1) geom FROM ips) + SELECT geom INTO ret + FROM matches; + RETURN ret; +END +$$ LANGUAGE plpgsql; +CREATE TYPE cdb_dataservices_server.isoline AS (center geometry(Geometry,4326), data_range integer, the_geom geometry(Multipolygon,4326)); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_routing_isolines(username TEXT, orgname TEXT, type TEXT, source geometry(Geometry, 4326), mode TEXT, data_range integer[], options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.here import HereMapsRoutingIsoline + from cartodb_services.metrics import QuotaService + from cartodb_services.here.types import geo_polyline_to_multipolygon + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_routing_config = GD["user_routing_config_{0}".format(username)] + + quota_service = QuotaService(user_routing_config, redis_conn) + + try: + client = HereMapsRoutingIsoline(user_routing_config.heremaps_app_id, user_routing_config.heremaps_app_code, base_url = HereMapsRoutingIsoline.PRODUCTION_ROUTING_BASE_URL) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + source_str = 'geo!%f,%f' % (lat, lon) + else: + source_str = None + + if type == 'isodistance': + resp = client.calculate_isodistance(source_str, mode, data_range, options) + elif type == 'isochrone': + resp = client.calculate_isochrone(source_str, mode, data_range, options) + + if resp: + result = [] + for isoline in resp: + data_range_n = isoline['range'] + polyline = isoline['geom'] + multipolygon = geo_polyline_to_multipolygon(polyline) + result.append([source, data_range_n, multipolygon]) + quota_service.increment_success_geocoder_use() + quota_service.increment_isolines_service_use(len(resp)) + return result + else: + quota_service.increment_empty_geocoder_use() + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to obtain isodistances using here maps geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu SECURITY DEFINER; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_routing_config_{0}".format(username)] + type = 'isodistance' + + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + isolines = [] + for element in result: + isoline = element['isoline'] + isoline = isoline.translate(None, "()").split(',') + isolines.append(isoline) + + return isolines +$$ LANGUAGE plpythonu; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_routing_config_{0}".format(username)] + type = 'isochrone' + + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + isolines = [] + for element in result: + isoline = element['isoline'] + isoline = isoline.translate(None, "()").split(',') + isolines.append(isoline) + + return isolines +$$ LANGUAGE plpythonu; +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT * + FROM pg_catalog.pg_user + WHERE usename = 'geocoder_api') THEN + + CREATE USER geocoder_api; + END IF; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO geocoder_api; + GRANT USAGE ON SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT USAGE ON SCHEMA public TO geocoder_api; + GRANT SELECT ON ALL TABLES IN SCHEMA public TO geocoder_api; +END$$; diff --git a/server/extension/cdb_dataservices_server.control b/server/extension/cdb_dataservices_server.control index 895ee6d..677c0e8 100644 --- a/server/extension/cdb_dataservices_server.control +++ b/server/extension/cdb_dataservices_server.control @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices server extension' -default_version = '0.4.0' +default_version = '0.5.0' requires = 'plpythonu, postgis, cdb_geocoder' superuser = true schema = cdb_dataservices_server diff --git a/server/extension/sql/0.5.0/00_header.sql b/server/extension/sql/0.5.0/00_header.sql new file mode 100644 index 0000000..2c8035c --- /dev/null +++ b/server/extension/sql/0.5.0/00_header.sql @@ -0,0 +1,3 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_dataservices_server" to load this file. \quit diff --git a/server/extension/sql/0.5.0/10_redis_helper.sql b/server/extension/sql/0.5.0/10_redis_helper.sql new file mode 100644 index 0000000..9200b26 --- /dev/null +++ b/server/extension/sql/0.5.0/10_redis_helper.sql @@ -0,0 +1,61 @@ +CREATE TYPE cdb_dataservices_server._redis_conf_params AS ( + sentinel_host text, + sentinel_port int, + sentinel_master_id text, + redis_host text, + redis_db text, + timeout float +); + +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_redis_conf_v2(config_key text) +RETURNS cdb_dataservices_server._redis_conf_params AS $$ + conf_query = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(config_key) + conf = plpy.execute(conf_query)[0]['conf'] + if conf is None: + plpy.error("There is no redis configuration defined") + else: + import json + params = json.loads(conf) + return { + "sentinel_host": params['sentinel_host'], + "sentinel_port": params['sentinel_port'], + "sentinel_master_id": params['sentinel_master_id'], + "redis_host": params['redis_host'], + "timeout": params['timeout'], + "redis_db": params['redis_db'] + } +$$ LANGUAGE plpythonu; + +-- Get the connection to redis from cache or create a new one +CREATE OR REPLACE FUNCTION cdb_dataservices_server._connect_to_redis(user_id text) +RETURNS boolean AS $$ + cache_key = "redis_connection_{0}".format(user_id) + if cache_key in GD: + return False + else: + from cartodb_services.tools import RedisConnection + metadata_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, + c.sentinel_master_id, c.redis_host, c.timeout, c.redis_db + from cdb_dataservices_server._get_redis_conf_v2('redis_metadata_config') c;""")[0] + metrics_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, + c.sentinel_master_id, c.redis_host, c.timeout, c.redis_db + from cdb_dataservices_server._get_redis_conf_v2('redis_metrics_config') c;""")[0] + redis_metadata_connection = RedisConnection(metadata_config_params['sentinel_host'], + metadata_config_params['sentinel_port'], + metadata_config_params['sentinel_master_id'], + metadata_config_params['redis_host'], + timeout=metadata_config_params['timeout'], + redis_db=metadata_config_params['redis_db']).redis_connection() + redis_metrics_connection = RedisConnection(metrics_config_params['sentinel_host'], + metrics_config_params['sentinel_port'], + metrics_config_params['sentinel_master_id'], + metrics_config_params['redis_host'], + timeout=metrics_config_params['timeout'], + redis_db=metrics_config_params['redis_db']).redis_connection() + GD[cache_key] = { + 'redis_metadata_connection': redis_metadata_connection, + 'redis_metrics_connection': redis_metrics_connection, + } + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; diff --git a/server/extension/sql/0.5.0/15_config_helper.sql b/server/extension/sql/0.5.0/15_config_helper.sql new file mode 100644 index 0000000..47aad42 --- /dev/null +++ b/server/extension/sql/0.5.0/15_config_helper.sql @@ -0,0 +1,51 @@ +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + import json + from cartodb_services.metrics import GeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf'] + if not heremaps_conf_json: + heremaps_app_id = None + heremaps_app_code = None + else: + heremaps_conf = json.loads(heremaps_conf_json) + heremaps_app_id = heremaps_conf['app_id'] + heremaps_app_code = heremaps_conf['app_code'] + geocoder_config = GeocoderConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code) + # --Think about the security concerns with this kind of global cache, it should be only available + # --for this user session but... + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + import json + from cartodb_services.metrics import RoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf'] + if not heremaps_conf_json: + heremaps_app_id = None + heremaps_app_code = None + else: + heremaps_conf = json.loads(heremaps_conf_json) + heremaps_app_id = heremaps_conf['app_id'] + heremaps_app_code = heremaps_conf['app_code'] + routing_config = RoutingConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code) + # --Think about the security concerns with this kind of global cache, it should be only available + # --for this user session but... + GD[cache_key] = routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; diff --git a/server/extension/sql/0.5.0/20_geocode_street.sql b/server/extension/sql/0.5.0/20_geocode_street.sql new file mode 100644 index 0000000..de77f62 --- /dev/null +++ b/server/extension/sql/0.5.0/20_geocode_street.sql @@ -0,0 +1,84 @@ +-- Geocodes a street address given a searchtext and a state and/or country +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + plpy.error('Requested geocoder is not available') + +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.here import HereMapsGeocoder + from cartodb_services.metrics import QuotaService + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + # -- Check the quota + quota_service = QuotaService(user_geocoder_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reach the limit of your quota') + + try: + geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + quota_service.increment_success_geocoder_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using here maps geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.google import GoogleMapsGeocoder + from cartodb_services.metrics import QuotaService + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + quota_service = QuotaService(user_geocoder_config, redis_conn) + + try: + geocoder = GoogleMapsGeocoder(user_geocoder_config.google_client_id, user_geocoder_config.google_api_key) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + quota_service.increment_success_geocoder_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using google maps geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; diff --git a/server/extension/sql/0.5.0/30_admin0.sql b/server/extension/sql/0.5.0/30_admin0.sql new file mode 100644 index 0000000..ec8fe5c --- /dev/null +++ b/server/extension/sql/0.5.0/30_admin0.sql @@ -0,0 +1,50 @@ +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin0_polygon(country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT n.the_geom as geom INTO ret + FROM (SELECT q, lower(regexp_replace(q, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text x + FROM (SELECT country_name q) g) d + LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x + LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.the_geom, s.adm0_a3; + + RETURN ret; + END +$$ LANGUAGE plpgsql; diff --git a/server/extension/sql/0.5.0/40_admin1.sql b/server/extension/sql/0.5.0/40_admin1.sql new file mode 100644 index 0000000..dc8e037 --- /dev/null +++ b/server/extension/sql/0.5.0/40_admin1.sql @@ -0,0 +1,117 @@ +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [admin1_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"]) + rv = plpy.execute(plan, [admin1_name, country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT q, ( + SELECT the_geom + FROM global_province_polygons + WHERE d.c = ANY (synonyms) + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM ( + SELECT + trim(replace(lower(admin1_name),'.',' ')) c, admin1_name q + ) d + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + WITH p AS (SELECT r.c, r.q, (SELECT iso3 FROM country_decoder WHERE lower(country_name) = ANY (synonyms)) i FROM (SELECT trim(replace(lower(admin1_name),'.',' ')) c, country_name q) r) + SELECT + geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_province_polygons + WHERE p.c = ANY (synonyms) + AND iso3 = p.i + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM p) n; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + diff --git a/server/extension/sql/0.5.0/50_namedplaces.sql b/server/extension/sql/0.5.0/50_namedplaces.sql new file mode 100644 index 0000000..80306cb --- /dev/null +++ b/server/extension/sql/0.5.0/50_namedplaces.sql @@ -0,0 +1,164 @@ +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [city_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"]) + rv = plpy.execute(plan, [city_name, country_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"]) + rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH best AS (SELECT s AS q, (SELECT the_geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT city_name as s) p), + next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT city_name as s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT q, geom, TRUE AS success FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH p AS (SELECT r.s, r.c, (SELECT iso2 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT city_name AS s, country_name::text AS c) r), + best AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE c = p.c AND geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH inputcountry AS ( + SELECT iso2 as isoTwo FROM country_decoder WHERE lower(country_name) = ANY (synonyms) LIMIT 1 + ), + p AS ( + SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder, inputcountry WHERE lower(r.a1) = ANY (synonyms) AND admin1_decoder.iso2 = inputcountry.isoTwo LIMIT 1) i FROM (SELECT city_name AS s, admin1_name::text AS a1) r), + best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + diff --git a/server/extension/sql/0.5.0/60_postalcodes.sql b/server/extension/sql/0.5.0/60_postalcodes.sql new file mode 100644 index 0000000..4ba321b --- /dev/null +++ b/server/extension/sql/0.5.0/60_postalcodes.sql @@ -0,0 +1,219 @@ +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; diff --git a/server/extension/sql/0.5.0/70_ips.sql b/server/extension/sql/0.5.0/70_ips.sql new file mode 100644 index 0000000..f76190f --- /dev/null +++ b/server/extension/sql/0.5.0/70_ips.sql @@ -0,0 +1,61 @@ +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"]) + rv = plpy.execute(plan, [ip], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_ipaddress_point(ip text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + + new_ip INET; + BEGIN + BEGIN + IF family(ip::inet) = 6 THEN + new_ip := ip::inet; + ELSE + new_ip := ('::ffff:' || ip)::inet; + END IF; + EXCEPTION WHEN OTHERS THEN + SELECT NULL as geom INTO ret; + RETURN ret; + END; + + WITH + ips AS (SELECT ip s, new_ip net), + matches AS (SELECT s, (SELECT the_geom FROM ip_address_locations WHERE network_start_ip <= ips.net ORDER BY network_start_ip DESC LIMIT 1) geom FROM ips) + SELECT geom INTO ret + FROM matches; + RETURN ret; +END +$$ LANGUAGE plpgsql; diff --git a/server/extension/sql/0.5.0/80_routing_helper.sql b/server/extension/sql/0.5.0/80_routing_helper.sql new file mode 100644 index 0000000..634c2d7 --- /dev/null +++ b/server/extension/sql/0.5.0/80_routing_helper.sql @@ -0,0 +1,51 @@ +CREATE TYPE cdb_dataservices_server.isoline AS (center geometry(Geometry,4326), data_range integer, the_geom geometry(Multipolygon,4326)); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_routing_isolines(username TEXT, orgname TEXT, type TEXT, source geometry(Geometry, 4326), mode TEXT, data_range integer[], options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.here import HereMapsRoutingIsoline + from cartodb_services.metrics import QuotaService + from cartodb_services.here.types import geo_polyline_to_multipolygon + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_routing_config = GD["user_routing_config_{0}".format(username)] + + quota_service = QuotaService(user_routing_config, redis_conn) + + try: + client = HereMapsRoutingIsoline(user_routing_config.heremaps_app_id, user_routing_config.heremaps_app_code, base_url = HereMapsRoutingIsoline.PRODUCTION_ROUTING_BASE_URL) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + source_str = 'geo!%f,%f' % (lat, lon) + else: + source_str = None + + if type == 'isodistance': + resp = client.calculate_isodistance(source_str, mode, data_range, options) + elif type == 'isochrone': + resp = client.calculate_isochrone(source_str, mode, data_range, options) + + if resp: + result = [] + for isoline in resp: + data_range_n = isoline['range'] + polyline = isoline['geom'] + multipolygon = geo_polyline_to_multipolygon(polyline) + result.append([source, data_range_n, multipolygon]) + quota_service.increment_success_geocoder_use() + quota_service.increment_isolines_service_use(len(resp)) + return result + else: + quota_service.increment_empty_geocoder_use() + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to obtain isodistances using here maps geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu SECURITY DEFINER; diff --git a/server/extension/sql/0.5.0/85_isodistance.sql b/server/extension/sql/0.5.0/85_isodistance.sql new file mode 100644 index 0000000..b5b553c --- /dev/null +++ b/server/extension/sql/0.5.0/85_isodistance.sql @@ -0,0 +1,18 @@ +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_routing_config_{0}".format(username)] + type = 'isodistance' + + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + isolines = [] + for element in result: + isoline = element['isoline'] + isoline = isoline.translate(None, "()").split(',') + isolines.append(isoline) + + return isolines +$$ LANGUAGE plpythonu; diff --git a/server/extension/sql/0.5.0/90_isochrone.sql b/server/extension/sql/0.5.0/90_isochrone.sql new file mode 100644 index 0000000..31f9d3a --- /dev/null +++ b/server/extension/sql/0.5.0/90_isochrone.sql @@ -0,0 +1,18 @@ +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_routing_config_{0}".format(username)] + type = 'isochrone' + + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + isolines = [] + for element in result: + isoline = element['isoline'] + isoline = isoline.translate(None, "()").split(',') + isolines.append(isoline) + + return isolines +$$ LANGUAGE plpythonu; diff --git a/server/extension/sql/0.5.0/99_geocoder_server_user.sql b/server/extension/sql/0.5.0/99_geocoder_server_user.sql new file mode 100644 index 0000000..e011682 --- /dev/null +++ b/server/extension/sql/0.5.0/99_geocoder_server_user.sql @@ -0,0 +1,15 @@ +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT * + FROM pg_catalog.pg_user + WHERE usename = 'geocoder_api') THEN + + CREATE USER geocoder_api; + END IF; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO geocoder_api; + GRANT USAGE ON SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT USAGE ON SCHEMA public TO geocoder_api; + GRANT SELECT ON ALL TABLES IN SCHEMA public TO geocoder_api; +END$$; diff --git a/server/extension/test/0.5.0/expected/00_install_test.out b/server/extension/test/0.5.0/expected/00_install_test.out new file mode 100644 index 0000000..db041d0 --- /dev/null +++ b/server/extension/test/0.5.0/expected/00_install_test.out @@ -0,0 +1,36 @@ +-- Install dependencies +CREATE EXTENSION postgis; +CREATE EXTENSION schema_triggers; +CREATE EXTENSION plpythonu; +CREATE EXTENSION cartodb; +CREATE EXTENSION cdb_geocoder; +-- Install the extension +CREATE EXTENSION cdb_dataservices_server; +-- Mock the redis server connection to point to this very test db +SELECT cartodb.cdb_conf_setconf('redis_metrics_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); + cdb_conf_setconf +------------------ + +(1 row) + +SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); + cdb_conf_setconf +------------------ + +(1 row) + +-- Mock the varnish invalidation function +-- (used by cdb_geocoder tests) +CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$ +BEGIN + RETURN; +END +$$ +LANGUAGE plpgsql; +-- Set user quota +SELECT cartodb.CDB_SetUserQuotaInBytes(0); + cdb_setuserquotainbytes +------------------------- + 0 +(1 row) + diff --git a/server/extension/test/0.5.0/expected/20_street_test.out b/server/extension/test/0.5.0/expected/20_street_test.out new file mode 100644 index 0000000..8fbce7a --- /dev/null +++ b/server/extension/test/0.5.0/expected/20_street_test.out @@ -0,0 +1,12 @@ +-- Check for namedplaces signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_street_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, text'); + exists +-------- + t +(1 row) + diff --git a/server/extension/test/0.5.0/expected/30_admin0_test.out b/server/extension/test/0.5.0/expected/30_admin0_test.out new file mode 100644 index 0000000..55c7639 --- /dev/null +++ b/server/extension/test/0.5.0/expected/30_admin0_test.out @@ -0,0 +1,47 @@ +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_dataservices_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'Spain'); + cdb_geocode_admin0_polygon +---------------------------- + +(1 row) + +-- Insert some dummy synonym +INSERT INTO admin0_synonyms (name, adm0_a3) VALUES ('Spain', 'ESP'); +-- Insert some dummy geometry to return +INSERT INTO ne_admin0_v3 (adm0_a3, the_geom) VALUES('ESP', ST_GeomFromText( + 'POLYGON((-71.1031880899493 42.3152774590236, + -71.1031627617667 42.3152960829043, + -71.102923838298 42.3149156848307, + -71.1031880899493 42.3152774590236))',4326) +); +-- This should return the polygon inserted above +SELECT cdb_dataservices_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'Spain'); + cdb_geocode_admin0_polygon +-------------------------------------------------------------------------------------------------------------------------------------------------------------------- + 0103000020E61000000100000004000000D0EA37A29AC651C00FD603035B284540FEFCFB379AC651C0C0503E9F5B284540FFDDDD4D96C651C033AC3B284F284540D0EA37A29AC651C00FD603035B284540 +(1 row) + +-- Check for admin0 signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_admin0_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_admin0_polygon' + AND oidvectortypes(p.proargtypes) = 'text'); + exists +-------- + t +(1 row) + diff --git a/server/extension/test/0.5.0/expected/40_admin1_test.out b/server/extension/test/0.5.0/expected/40_admin1_test.out new file mode 100644 index 0000000..4894b0c --- /dev/null +++ b/server/extension/test/0.5.0/expected/40_admin1_test.out @@ -0,0 +1,81 @@ +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California'); + cdb_geocode_admin1_polygon +---------------------------- + +(1 row) + +SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California', 'United States'); + cdb_geocode_admin1_polygon +---------------------------- + +(1 row) + +-- Insert dummy data into country decoder table +INSERT INTO country_decoder (synonyms, iso3) VALUES (Array['united states'], 'USA'); +-- Insert some dummy data and geometry to return +INSERT INTO global_province_polygons (synonyms, iso3, the_geom) VALUES (Array['california'], 'USA', ST_GeomFromText( + 'POLYGON((-71.1031880899493 42.3152774590236, + -71.1031627617667 42.3152960829043, + -71.102923838298 42.3149156848307, + -71.1031880899493 42.3152774590236))',4326) +); +-- This should return the polygon inserted above +SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California'); + cdb_geocode_admin1_polygon +-------------------------------------------------------------------------------------------------------------------------------------------------------------------- + 0103000020E61000000100000004000000D0EA37A29AC651C00FD603035B284540FEFCFB379AC651C0C0503E9F5B284540FFDDDD4D96C651C033AC3B284F284540D0EA37A29AC651C00FD603035B284540 +(1 row) + +SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California', 'United States'); + cdb_geocode_admin1_polygon +-------------------------------------------------------------------------------------------------------------------------------------------------------------------- + 0103000020E61000000100000004000000D0EA37A29AC651C00FD603035B284540FEFCFB379AC651C0C0503E9F5B284540FFDDDD4D96C651C033AC3B284F284540D0EA37A29AC651C00FD603035B284540 +(1 row) + +-- Check for admin1 signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_admin1_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_admin1_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_admin1_polygon' + AND oidvectortypes(p.proargtypes) = 'text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_admin1_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text'); + exists +-------- + t +(1 row) + diff --git a/server/extension/test/0.5.0/expected/50_namedplaces_test.out b/server/extension/test/0.5.0/expected/50_namedplaces_test.out new file mode 100644 index 0000000..7755bf4 --- /dev/null +++ b/server/extension/test/0.5.0/expected/50_namedplaces_test.out @@ -0,0 +1,136 @@ +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx'); + cdb_geocode_namedplace_point +------------------------------ + +(1 row) + +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain'); + cdb_geocode_namedplace_point +------------------------------ + +(1 row) + +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain'); + cdb_geocode_namedplace_point +------------------------------ + +(1 row) + +-- Insert dummy data into points table +INSERT INTO global_cities_points_limited (geoname_id, name, iso2, admin1, admin2, population, lowername, the_geom) VALUES (3128760, 'Elche', 'ES', 'Valencia', 'AL', 34534, 'elche', ST_GeomFromText( + 'POINT(0.6983 39.26787)',4326) +); +-- Insert dummy data into alternates table +INSERT INTO global_cities_alternates_limited (geoname_id, name, preferred, lowername, admin1_geonameid, iso2, admin1, the_geom) VALUES (3128760, 'Elx', true, 'elx', '000000', 'ES', 'Valencia', ST_GeomFromText( + 'POINT(0.6983 39.26787)',4326) +); +-- Insert dummy data into country decoder table +INSERT INTO country_decoder (synonyms, iso2) VALUES (Array['spain'], 'ES'); +-- Insert dummy data into admin1 decoder table +INSERT INTO admin1_decoder (admin1, synonyms, iso2) VALUES ('Valencia', Array['valencia', 'Valencia'], 'ES'); +-- This should return the point inserted above +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx'); + cdb_geocode_namedplace_point +---------------------------------------------------- + 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 +(1 row) + +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche'); + cdb_geocode_namedplace_point +---------------------------------------------------- + 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 +(1 row) + +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain'); + cdb_geocode_namedplace_point +---------------------------------------------------- + 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 +(1 row) + +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'Spain'); + cdb_geocode_namedplace_point +---------------------------------------------------- + 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 +(1 row) + +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain'); + cdb_geocode_namedplace_point +---------------------------------------------------- + 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 +(1 row) + +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'valencia', 'Spain'); + cdb_geocode_namedplace_point +---------------------------------------------------- + 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 +(1 row) + +-- Check for namedplaces signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + exists +-------- + t +(1 row) + diff --git a/server/extension/test/0.5.0/expected/60_postalcodes_test.out b/server/extension/test/0.5.0/expected/60_postalcodes_test.out new file mode 100644 index 0000000..b2a5c91 --- /dev/null +++ b/server/extension/test/0.5.0/expected/60_postalcodes_test.out @@ -0,0 +1,163 @@ +-- Make sure dbs are clean +DELETE FROM global_postal_code_points; +DELETE FROM global_postal_code_polygons; +DELETE FROM country_decoder; +DELETE FROM available_services; +DELETE FROM admin0_synonyms; +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204'); + cdb_geocode_postalcode_point +------------------------------ + +(1 row) + +-- Insert dummy data into ip_address_locations +INSERT INTO global_postal_code_points (the_geom, iso3, postal_code, postal_code_num) VALUES ( + '0101000020E61000000000000000E040408036B47414764840', + 'ESP', + '03204', + 3204 +); +INSERT INTO global_postal_code_polygons (the_geom, iso3, postal_code, postal_code_num) VALUES ( + '0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040', + 'ESP', + '03204', + 3204 +); +INSERT INTO country_decoder (iso3, synonyms) VALUES ( + 'ESP', + Array['spain', 'Spain', 'ESP'] +); +INSERT INTO available_services (adm0_a3, admin0, postal_code_points, postal_code_polygons) VALUES ( + 'ESP', + 't', + 't', + 't' +); +INSERT INTO admin0_synonyms (adm0_a3, name, name_, rank) VALUES ( + 'ESP', + 'Spain', + 'spain', + 3 +); +-- This should return the polygon inserted above +SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204'); + cdb_geocode_postalcode_point +---------------------------------------------------- + 0101000020E61000000000000000E040408036B47414764840 +(1 row) + +SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204', 'spain'); + cdb_geocode_postalcode_point +---------------------------------------------------- + 0101000020E61000000000000000E040408036B47414764840 +(1 row) + +SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', '03204'); + cdb_geocode_postalcode_polygon +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + 0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040 +(1 row) + +SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', '03204', 'spain'); + cdb_geocode_postalcode_polygon +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + 0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040 +(1 row) + +-- Clean dbs +DELETE FROM global_postal_code_points; +DELETE FROM global_postal_code_polygons; +DELETE FROM country_decoder; +DELETE FROM available_services; +DELETE FROM admin0_synonyms; +-- Check for namedplaces signatures (point and polygon) +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_postalcode_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_postalcode_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_postalcode_point' + AND oidvectortypes(p.proargtypes) = 'text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_postalcode_point' + AND oidvectortypes(p.proargtypes) = 'text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text'); + exists +-------- + t +(1 row) + diff --git a/server/extension/test/0.5.0/expected/70_ips_test.out b/server/extension/test/0.5.0/expected/70_ips_test.out new file mode 100644 index 0000000..48012f7 --- /dev/null +++ b/server/extension/test/0.5.0/expected/70_ips_test.out @@ -0,0 +1,40 @@ +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_dataservices_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', '0.0.0.0'); + cdb_geocode_ipaddress_point +----------------------------- + +(1 row) + +-- Insert dummy data into ip_address_locations +INSERT INTO ip_address_locations VALUES ('::ffff:0.0.0.0'::inet, (ST_SetSRID(ST_MakePoint('40.40', '3.71'), 4326))); +-- This should return the polygon inserted above +SELECT cdb_dataservices_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', '0.0.0.0'); + cdb_geocode_ipaddress_point +---------------------------------------------------- + 0101000020E61000003333333333334440AE47E17A14AE0D40 +(1 row) + +-- Check for namedplaces signatures (point and polygon) +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_ipaddress_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_ipaddress_point' + AND oidvectortypes(p.proargtypes) = 'text'); + exists +-------- + t +(1 row) + diff --git a/server/extension/test/0.5.0/expected/85_isodistance_test.out b/server/extension/test/0.5.0/expected/85_isodistance_test.out new file mode 100644 index 0000000..5705284 --- /dev/null +++ b/server/extension/test/0.5.0/expected/85_isodistance_test.out @@ -0,0 +1,12 @@ +-- Check for isodistance signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_isodistance' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, integer[], text[]'); + exists +-------- + t +(1 row) + diff --git a/server/extension/test/0.5.0/expected/90_isochrone_test.out b/server/extension/test/0.5.0/expected/90_isochrone_test.out new file mode 100644 index 0000000..7ee0f04 --- /dev/null +++ b/server/extension/test/0.5.0/expected/90_isochrone_test.out @@ -0,0 +1,12 @@ +-- Check for isochrone signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_isochrone' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, integer[], text[]'); + exists +-------- + t +(1 row) + diff --git a/server/extension/test/0.5.0/expected/99_remove_geocoder_api_user_test.out b/server/extension/test/0.5.0/expected/99_remove_geocoder_api_user_test.out new file mode 100644 index 0000000..ad4fee7 --- /dev/null +++ b/server/extension/test/0.5.0/expected/99_remove_geocoder_api_user_test.out @@ -0,0 +1,5 @@ +REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_dataservices_server FROM geocoder_api; +REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA public FROM geocoder_api; +REVOKE USAGE ON SCHEMA cdb_dataservices_server FROM geocoder_api; +REVOKE USAGE ON SCHEMA public FROM geocoder_api; +REVOKE SELECT ON ALL TABLES IN SCHEMA public FROM geocoder_api; diff --git a/server/extension/test/0.5.0/sql/00_install_test.sql b/server/extension/test/0.5.0/sql/00_install_test.sql new file mode 100644 index 0000000..da91608 --- /dev/null +++ b/server/extension/test/0.5.0/sql/00_install_test.sql @@ -0,0 +1,25 @@ +-- Install dependencies +CREATE EXTENSION postgis; +CREATE EXTENSION schema_triggers; +CREATE EXTENSION plpythonu; +CREATE EXTENSION cartodb; +CREATE EXTENSION cdb_geocoder; + +-- Install the extension +CREATE EXTENSION cdb_dataservices_server; + +-- Mock the redis server connection to point to this very test db +SELECT cartodb.cdb_conf_setconf('redis_metrics_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); +SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); + +-- Mock the varnish invalidation function +-- (used by cdb_geocoder tests) +CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$ +BEGIN + RETURN; +END +$$ +LANGUAGE plpgsql; + +-- Set user quota +SELECT cartodb.CDB_SetUserQuotaInBytes(0); diff --git a/server/extension/test/0.5.0/sql/20_street_test.sql b/server/extension/test/0.5.0/sql/20_street_test.sql new file mode 100644 index 0000000..86e0368 --- /dev/null +++ b/server/extension/test/0.5.0/sql/20_street_test.sql @@ -0,0 +1,7 @@ +-- Check for namedplaces signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_street_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, text'); diff --git a/server/extension/test/0.5.0/sql/30_admin0_test.sql b/server/extension/test/0.5.0/sql/30_admin0_test.sql new file mode 100644 index 0000000..3249c60 --- /dev/null +++ b/server/extension/test/0.5.0/sql/30_admin0_test.sql @@ -0,0 +1,32 @@ +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_dataservices_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'Spain'); + +-- Insert some dummy synonym +INSERT INTO admin0_synonyms (name, adm0_a3) VALUES ('Spain', 'ESP'); + +-- Insert some dummy geometry to return +INSERT INTO ne_admin0_v3 (adm0_a3, the_geom) VALUES('ESP', ST_GeomFromText( + 'POLYGON((-71.1031880899493 42.3152774590236, + -71.1031627617667 42.3152960829043, + -71.102923838298 42.3149156848307, + -71.1031880899493 42.3152774590236))',4326) +); + +-- This should return the polygon inserted above +SELECT cdb_dataservices_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'Spain'); + +-- Check for admin0 signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_admin0_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_admin0_polygon' + AND oidvectortypes(p.proargtypes) = 'text'); diff --git a/server/extension/test/0.5.0/sql/40_admin1_test.sql b/server/extension/test/0.5.0/sql/40_admin1_test.sql new file mode 100644 index 0000000..7b3748c --- /dev/null +++ b/server/extension/test/0.5.0/sql/40_admin1_test.sql @@ -0,0 +1,48 @@ +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California'); +SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California', 'United States'); + +-- Insert dummy data into country decoder table +INSERT INTO country_decoder (synonyms, iso3) VALUES (Array['united states'], 'USA'); + +-- Insert some dummy data and geometry to return +INSERT INTO global_province_polygons (synonyms, iso3, the_geom) VALUES (Array['california'], 'USA', ST_GeomFromText( + 'POLYGON((-71.1031880899493 42.3152774590236, + -71.1031627617667 42.3152960829043, + -71.102923838298 42.3149156848307, + -71.1031880899493 42.3152774590236))',4326) +); + +-- This should return the polygon inserted above +SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California'); +SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California', 'United States'); + +-- Check for admin1 signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_admin1_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_admin1_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_admin1_polygon' + AND oidvectortypes(p.proargtypes) = 'text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_admin1_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text'); diff --git a/server/extension/test/0.5.0/sql/50_namedplaces_test.sql b/server/extension/test/0.5.0/sql/50_namedplaces_test.sql new file mode 100644 index 0000000..306a682 --- /dev/null +++ b/server/extension/test/0.5.0/sql/50_namedplaces_test.sql @@ -0,0 +1,72 @@ +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain'); + +-- Insert dummy data into points table +INSERT INTO global_cities_points_limited (geoname_id, name, iso2, admin1, admin2, population, lowername, the_geom) VALUES (3128760, 'Elche', 'ES', 'Valencia', 'AL', 34534, 'elche', ST_GeomFromText( + 'POINT(0.6983 39.26787)',4326) +); + +-- Insert dummy data into alternates table +INSERT INTO global_cities_alternates_limited (geoname_id, name, preferred, lowername, admin1_geonameid, iso2, admin1, the_geom) VALUES (3128760, 'Elx', true, 'elx', '000000', 'ES', 'Valencia', ST_GeomFromText( + 'POINT(0.6983 39.26787)',4326) +); + +-- Insert dummy data into country decoder table +INSERT INTO country_decoder (synonyms, iso2) VALUES (Array['spain'], 'ES'); + +-- Insert dummy data into admin1 decoder table +INSERT INTO admin1_decoder (admin1, synonyms, iso2) VALUES ('Valencia', Array['valencia', 'Valencia'], 'ES'); + +-- This should return the point inserted above +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'Spain'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'valencia', 'Spain'); + +-- Check for namedplaces signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); diff --git a/server/extension/test/0.5.0/sql/60_postalcodes_test.sql b/server/extension/test/0.5.0/sql/60_postalcodes_test.sql new file mode 100644 index 0000000..55d1c85 --- /dev/null +++ b/server/extension/test/0.5.0/sql/60_postalcodes_test.sql @@ -0,0 +1,117 @@ +-- Make sure dbs are clean +DELETE FROM global_postal_code_points; +DELETE FROM global_postal_code_polygons; +DELETE FROM country_decoder; +DELETE FROM available_services; +DELETE FROM admin0_synonyms; + +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204'); + +-- Insert dummy data into ip_address_locations +INSERT INTO global_postal_code_points (the_geom, iso3, postal_code, postal_code_num) VALUES ( + '0101000020E61000000000000000E040408036B47414764840', + 'ESP', + '03204', + 3204 +); + +INSERT INTO global_postal_code_polygons (the_geom, iso3, postal_code, postal_code_num) VALUES ( + '0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040', + 'ESP', + '03204', + 3204 +); + +INSERT INTO country_decoder (iso3, synonyms) VALUES ( + 'ESP', + Array['spain', 'Spain', 'ESP'] +); + +INSERT INTO available_services (adm0_a3, admin0, postal_code_points, postal_code_polygons) VALUES ( + 'ESP', + 't', + 't', + 't' +); + +INSERT INTO admin0_synonyms (adm0_a3, name, name_, rank) VALUES ( + 'ESP', + 'Spain', + 'spain', + 3 +); + +-- This should return the polygon inserted above +SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204'); + +SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204', 'spain'); + +SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', '03204'); + +SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', '03204', 'spain'); + +-- Clean dbs +DELETE FROM global_postal_code_points; +DELETE FROM global_postal_code_polygons; +DELETE FROM country_decoder; +DELETE FROM available_services; +DELETE FROM admin0_synonyms; + +-- Check for namedplaces signatures (point and polygon) +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_postalcode_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_postalcode_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_postalcode_point' + AND oidvectortypes(p.proargtypes) = 'text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_postalcode_point' + AND oidvectortypes(p.proargtypes) = 'text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text'); diff --git a/server/extension/test/0.5.0/sql/70_ips_test.sql b/server/extension/test/0.5.0/sql/70_ips_test.sql new file mode 100644 index 0000000..6111bd0 --- /dev/null +++ b/server/extension/test/0.5.0/sql/70_ips_test.sql @@ -0,0 +1,24 @@ +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_dataservices_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', '0.0.0.0'); + +-- Insert dummy data into ip_address_locations +INSERT INTO ip_address_locations VALUES ('::ffff:0.0.0.0'::inet, (ST_SetSRID(ST_MakePoint('40.40', '3.71'), 4326))); + +-- This should return the polygon inserted above +SELECT cdb_dataservices_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', '0.0.0.0'); + +-- Check for namedplaces signatures (point and polygon) +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_ipaddress_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_ipaddress_point' + AND oidvectortypes(p.proargtypes) = 'text'); diff --git a/server/extension/test/0.5.0/sql/85_isodistance_test.sql b/server/extension/test/0.5.0/sql/85_isodistance_test.sql new file mode 100644 index 0000000..f66b68a --- /dev/null +++ b/server/extension/test/0.5.0/sql/85_isodistance_test.sql @@ -0,0 +1,7 @@ +-- Check for isodistance signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_isodistance' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, integer[], text[]'); diff --git a/server/extension/test/0.5.0/sql/90_isochrone_test.sql b/server/extension/test/0.5.0/sql/90_isochrone_test.sql new file mode 100644 index 0000000..113fb28 --- /dev/null +++ b/server/extension/test/0.5.0/sql/90_isochrone_test.sql @@ -0,0 +1,7 @@ +-- Check for isochrone signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_isochrone' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, integer[], text[]'); diff --git a/server/extension/test/0.5.0/sql/99_remove_geocoder_api_user_test.sql b/server/extension/test/0.5.0/sql/99_remove_geocoder_api_user_test.sql new file mode 100644 index 0000000..ad4fee7 --- /dev/null +++ b/server/extension/test/0.5.0/sql/99_remove_geocoder_api_user_test.sql @@ -0,0 +1,5 @@ +REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_dataservices_server FROM geocoder_api; +REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA public FROM geocoder_api; +REVOKE USAGE ON SCHEMA cdb_dataservices_server FROM geocoder_api; +REVOKE USAGE ON SCHEMA public FROM geocoder_api; +REVOKE SELECT ON ALL TABLES IN SCHEMA public FROM geocoder_api; diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py b/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py index 4ca9009..8a24bb3 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py @@ -1,4 +1,5 @@ from redis.sentinel import Sentinel +from redis import StrictRedis class RedisConnection: @@ -6,24 +7,30 @@ class RedisConnection: REDIS_DEFAULT_USER_DB = 5 REDIS_DEFAULT_TIMEOUT = 2 #seconds REDIS_SENTINEL_DEFAULT_PORT = 26379 + REDIS_DEFAULT_PORT = 6379 def __init__(self, sentinel_host, sentinel_port, sentinel_master_id, - redis_db=REDIS_DEFAULT_USER_DB, **kwargs): + redis_host, redis_db=REDIS_DEFAULT_USER_DB, **kwargs): self.sentinel_host = sentinel_host self.sentinel_port = sentinel_port self.sentinel_master_id = sentinel_master_id + self.redis_host = redis_host self.timeout = kwargs['timeout'] if 'timeout' in kwargs else self.REDIS_DEFAULT_TIMEOUT self.redis_db = redis_db + self.redis_port = self.REDIS_DEFAULT_PORT def redis_connection(self): return self.__create_redis_connection() def __create_redis_connection(self): - sentinel = Sentinel([(self.sentinel_host, - self.REDIS_SENTINEL_DEFAULT_PORT)], - socket_timeout=self.timeout) - return sentinel.master_for( - self.sentinel_master_id, - socket_timeout=self.timeout, - db=self.redis_db - ) + if (self.sentinel_host == None) or (self.sentinel_master_id == None): + return StrictRedis(host=self.redis_host, port=self.redis_port, db=self.redis_db) + else: + sentinel = Sentinel([(self.sentinel_host, + self.REDIS_SENTINEL_DEFAULT_PORT)], + socket_timeout=self.timeout) + return sentinel.master_for( + self.sentinel_master_id, + socket_timeout=self.timeout, + db=self.redis_db + ) From 04617332a2ffb59e2882e9cb2776a3632c933126 Mon Sep 17 00:00:00 2001 From: Luis Bosque Date: Wed, 17 Feb 2016 18:24:12 +0100 Subject: [PATCH 03/12] Add _redis_conf_params type alter In order to add or remove redis_host attribute to upgrade/downgrade path --- server/extension/cdb_dataservices_server--0.4.0--0.5.0.sql | 2 ++ server/extension/cdb_dataservices_server--0.5.0--0.4.0.sql | 2 ++ 2 files changed, 4 insertions(+) diff --git a/server/extension/cdb_dataservices_server--0.4.0--0.5.0.sql b/server/extension/cdb_dataservices_server--0.4.0--0.5.0.sql index e6b5295..ff54a09 100644 --- a/server/extension/cdb_dataservices_server--0.4.0--0.5.0.sql +++ b/server/extension/cdb_dataservices_server--0.4.0--0.5.0.sql @@ -1,3 +1,5 @@ +ALTER TYPE cdb_dataservices_server._redis_conf_params ADD ATTRIBUTE redis_host text; + -- Get the Redis configuration from the _conf table -- CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_redis_conf_v2(config_key text) RETURNS cdb_dataservices_server._redis_conf_params AS $$ diff --git a/server/extension/cdb_dataservices_server--0.5.0--0.4.0.sql b/server/extension/cdb_dataservices_server--0.5.0--0.4.0.sql index 83b5827..fcc28c6 100644 --- a/server/extension/cdb_dataservices_server--0.5.0--0.4.0.sql +++ b/server/extension/cdb_dataservices_server--0.5.0--0.4.0.sql @@ -1,3 +1,5 @@ +ALTER TYPE cdb_dataservices_server._redis_conf_params DROP ATTRIBUTE IF EXISTS redis_host; + -- Get the Redis configuration from the _conf table -- CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_redis_conf_v2(config_key text) RETURNS cdb_dataservices_server._redis_conf_params AS $$ From ab32065e1d223d4a2769c3eab02165ad2e971661 Mon Sep 17 00:00:00 2001 From: Luis Bosque Date: Thu, 18 Feb 2016 15:13:15 +0100 Subject: [PATCH 04/12] Script to help to enable the extension in dev env --- scripts/create_extension_in_db.sh | 35 +++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100755 scripts/create_extension_in_db.sh diff --git a/scripts/create_extension_in_db.sh b/scripts/create_extension_in_db.sh new file mode 100755 index 0000000..d7c4176 --- /dev/null +++ b/scripts/create_extension_in_db.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +TARGET_DB=$1 +PGUSER=${2:-postgres} +PYTHON_LIBRARY_NAME='cartodb_services' + +function usage { + echo "Usage: ${0} [dbuser]" +} + +[[ -z $TARGET_DB ]] && echo "Missing DB parameter" && usage && exit 1 + +python -c "import ${PYTHON_LIBRARY_NAME}" +if [[ $? != 0 ]] +then + echo "Missing ${PYTHON_LIBRARY_NAME} python library" + echo "Trying to install.." + cd server/lib/python/cartodb_services && sudo python setup.py install + python -c "import ${PYTHON_LIBRARY_NAME}" 2> /dev/null + if [[ $? != 0 ]] + then + echo "There are some problems with python library. Debug manually" + exit 1 + fi +fi + +CREATE_EXTENSION_COMMAND="CREATE EXTENSION IF NOT EXISTS" + +CDB_GEOCODER_EXTENSION_CREATE="${CREATE_EXTENSION_COMMAND} cdb_geocoder" +CDB_DATASERVICES_SERVER_CREATE="${CREATE_EXTENSION_COMMAND} cdb_dataservices_server" + +echo "* Creating extension cdb_geocoder" +psql -U ${PGUSER} -d ${TARGET_DB} -c "${CDB_GEOCODER_EXTENSION_CREATE}" +echo "* Creating extension cdb_dataservices_server" +psql -U ${PGUSER} -d ${TARGET_DB} -c "${CDB_DATASERVICES_SERVER_CREATE}" From 472d8c2360342d202424f832bf70aa4d69be4617 Mon Sep 17 00:00:00 2001 From: Luis Bosque Date: Fri, 19 Feb 2016 11:18:03 +0100 Subject: [PATCH 05/12] Install cdb_geocoder extension with non-admin user --- scripts/create_extension_in_db.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/create_extension_in_db.sh b/scripts/create_extension_in_db.sh index d7c4176..71ed678 100755 --- a/scripts/create_extension_in_db.sh +++ b/scripts/create_extension_in_db.sh @@ -1,7 +1,8 @@ #!/bin/bash TARGET_DB=$1 -PGUSER=${2:-postgres} +ADMIN_PGUSER="postgres" +PGUSER=${2:-${ADMIN_PGUSER}} PYTHON_LIBRARY_NAME='cartodb_services' function usage { @@ -32,4 +33,4 @@ CDB_DATASERVICES_SERVER_CREATE="${CREATE_EXTENSION_COMMAND} cdb_dataservices_ser echo "* Creating extension cdb_geocoder" psql -U ${PGUSER} -d ${TARGET_DB} -c "${CDB_GEOCODER_EXTENSION_CREATE}" echo "* Creating extension cdb_dataservices_server" -psql -U ${PGUSER} -d ${TARGET_DB} -c "${CDB_DATASERVICES_SERVER_CREATE}" +psql -U ${ADMIN_PGUSER} -d ${TARGET_DB} -c "${CDB_DATASERVICES_SERVER_CREATE}" From 6c5b1b4e99af02ebaa76d35e044267a822212ba2 Mon Sep 17 00:00:00 2001 From: Luis Bosque Date: Mon, 22 Feb 2016 09:50:11 +0100 Subject: [PATCH 06/12] Use strictredis if sentinel_master_id is not null Also, use only a redis_host and redis_port attributes either if the connection is to redis or to sentinel --- .../cdb_dataservices_server--0.5.0.sql | 24 ++++++++----------- .../extension/sql/0.5.0/10_redis_helper.sql | 24 ++++++++----------- .../cartodb_services/tools/redis_tools.py | 20 +++++++--------- 3 files changed, 29 insertions(+), 39 deletions(-) diff --git a/server/extension/cdb_dataservices_server--0.5.0.sql b/server/extension/cdb_dataservices_server--0.5.0.sql index 05bac88..9f027b8 100644 --- a/server/extension/cdb_dataservices_server--0.5.0.sql +++ b/server/extension/cdb_dataservices_server--0.5.0.sql @@ -2,10 +2,9 @@ -- Complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "CREATE EXTENSION cdb_dataservices_server" to load this file. \quit CREATE TYPE cdb_dataservices_server._redis_conf_params AS ( - sentinel_host text, - sentinel_port int, sentinel_master_id text, redis_host text, + redis_port int, redis_db text, timeout float ); @@ -21,10 +20,9 @@ RETURNS cdb_dataservices_server._redis_conf_params AS $$ import json params = json.loads(conf) return { - "sentinel_host": params['sentinel_host'], - "sentinel_port": params['sentinel_port'], "sentinel_master_id": params['sentinel_master_id'], "redis_host": params['redis_host'], + "redis_port": params['redis_port'], "timeout": params['timeout'], "redis_db": params['redis_db'] } @@ -38,22 +36,20 @@ RETURNS boolean AS $$ return False else: from cartodb_services.tools import RedisConnection - metadata_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, - c.sentinel_master_id, c.redis_host, c.timeout, c.redis_db + metadata_config_params = plpy.execute("""select c.sentinel_master_id, c.redis_host, + c.redis_port, c.timeout, c.redis_db from cdb_dataservices_server._get_redis_conf_v2('redis_metadata_config') c;""")[0] - metrics_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, - c.sentinel_master_id, c.redis_host, c.timeout, c.redis_db + metrics_config_params = plpy.execute("""select c.sentinel_master_id, c.redis_host, + c.redis_port, c.timeout, c.redis_db from cdb_dataservices_server._get_redis_conf_v2('redis_metrics_config') c;""")[0] - redis_metadata_connection = RedisConnection(metadata_config_params['sentinel_host'], - metadata_config_params['sentinel_port'], - metadata_config_params['sentinel_master_id'], + redis_metadata_connection = RedisConnection(metadata_config_params['sentinel_master_id'], metadata_config_params['redis_host'], + metadata_config_params['redis_port'], timeout=metadata_config_params['timeout'], redis_db=metadata_config_params['redis_db']).redis_connection() - redis_metrics_connection = RedisConnection(metrics_config_params['sentinel_host'], - metrics_config_params['sentinel_port'], - metrics_config_params['sentinel_master_id'], + redis_metrics_connection = RedisConnection(metrics_config_params['sentinel_master_id'], metrics_config_params['redis_host'], + metrics_config_params['redis_port'], timeout=metrics_config_params['timeout'], redis_db=metrics_config_params['redis_db']).redis_connection() GD[cache_key] = { diff --git a/server/extension/sql/0.5.0/10_redis_helper.sql b/server/extension/sql/0.5.0/10_redis_helper.sql index 9200b26..97223f4 100644 --- a/server/extension/sql/0.5.0/10_redis_helper.sql +++ b/server/extension/sql/0.5.0/10_redis_helper.sql @@ -1,8 +1,7 @@ CREATE TYPE cdb_dataservices_server._redis_conf_params AS ( - sentinel_host text, - sentinel_port int, sentinel_master_id text, redis_host text, + redis_port int, redis_db text, timeout float ); @@ -18,10 +17,9 @@ RETURNS cdb_dataservices_server._redis_conf_params AS $$ import json params = json.loads(conf) return { - "sentinel_host": params['sentinel_host'], - "sentinel_port": params['sentinel_port'], "sentinel_master_id": params['sentinel_master_id'], "redis_host": params['redis_host'], + "redis_port": params['redis_port'], "timeout": params['timeout'], "redis_db": params['redis_db'] } @@ -35,22 +33,20 @@ RETURNS boolean AS $$ return False else: from cartodb_services.tools import RedisConnection - metadata_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, - c.sentinel_master_id, c.redis_host, c.timeout, c.redis_db + metadata_config_params = plpy.execute("""select c.sentinel_master_id, c.redis_host, + c.redis_port, c.timeout, c.redis_db from cdb_dataservices_server._get_redis_conf_v2('redis_metadata_config') c;""")[0] - metrics_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, - c.sentinel_master_id, c.redis_host, c.timeout, c.redis_db + metrics_config_params = plpy.execute("""select c.sentinel_master_id, c.redis_host, + c.redis_port, c.timeout, c.redis_db from cdb_dataservices_server._get_redis_conf_v2('redis_metrics_config') c;""")[0] - redis_metadata_connection = RedisConnection(metadata_config_params['sentinel_host'], - metadata_config_params['sentinel_port'], - metadata_config_params['sentinel_master_id'], + redis_metadata_connection = RedisConnection(metadata_config_params['sentinel_master_id'], metadata_config_params['redis_host'], + metadata_config_params['redis_port'], timeout=metadata_config_params['timeout'], redis_db=metadata_config_params['redis_db']).redis_connection() - redis_metrics_connection = RedisConnection(metrics_config_params['sentinel_host'], - metrics_config_params['sentinel_port'], - metrics_config_params['sentinel_master_id'], + redis_metrics_connection = RedisConnection(metrics_config_params['sentinel_master_id'], metrics_config_params['redis_host'], + metrics_config_params['redis_port'], timeout=metrics_config_params['timeout'], redis_db=metrics_config_params['redis_db']).redis_connection() GD[cache_key] = { diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py b/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py index 8a24bb3..24888dd 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py @@ -6,28 +6,26 @@ class RedisConnection: REDIS_DEFAULT_USER_DB = 5 REDIS_DEFAULT_TIMEOUT = 2 #seconds - REDIS_SENTINEL_DEFAULT_PORT = 26379 - REDIS_DEFAULT_PORT = 6379 + #REDIS_SENTINEL_DEFAULT_PORT = 26379 + #REDIS_DEFAULT_PORT = 6379 - def __init__(self, sentinel_host, sentinel_port, sentinel_master_id, - redis_host, redis_db=REDIS_DEFAULT_USER_DB, **kwargs): - self.sentinel_host = sentinel_host - self.sentinel_port = sentinel_port - self.sentinel_master_id = sentinel_master_id + def __init__(self, sentinel_master_id, redis_host, redis_port, + redis_db=REDIS_DEFAULT_USER_DB, **kwargs): self.redis_host = redis_host + self.redis_port = redis_port + self.sentinel_master_id = sentinel_master_id self.timeout = kwargs['timeout'] if 'timeout' in kwargs else self.REDIS_DEFAULT_TIMEOUT self.redis_db = redis_db - self.redis_port = self.REDIS_DEFAULT_PORT def redis_connection(self): return self.__create_redis_connection() def __create_redis_connection(self): - if (self.sentinel_host == None) or (self.sentinel_master_id == None): + if self.sentinel_master_id == None: return StrictRedis(host=self.redis_host, port=self.redis_port, db=self.redis_db) else: - sentinel = Sentinel([(self.sentinel_host, - self.REDIS_SENTINEL_DEFAULT_PORT)], + sentinel = Sentinel([(self.redis_host, + self.redis_port)], socket_timeout=self.timeout) return sentinel.master_for( self.sentinel_master_id, From 2a807af6df17a92cb3c94596e0d2de1d2f07a1dc Mon Sep 17 00:00:00 2001 From: Luis Bosque Date: Mon, 22 Feb 2016 10:06:09 +0100 Subject: [PATCH 07/12] Fix upgrade/downgrade paths to 0.5.0 --- .../cdb_dataservices_server--0.4.0--0.5.0.sql | 28 +++++++++---------- .../cdb_dataservices_server--0.5.0--0.4.0.sql | 5 +++- .../test/0.5.0/expected/00_install_test.out | 4 +-- .../test/0.5.0/sql/00_install_test.sql | 4 +-- 4 files changed, 22 insertions(+), 19 deletions(-) diff --git a/server/extension/cdb_dataservices_server--0.4.0--0.5.0.sql b/server/extension/cdb_dataservices_server--0.4.0--0.5.0.sql index ff54a09..11e2e71 100644 --- a/server/extension/cdb_dataservices_server--0.4.0--0.5.0.sql +++ b/server/extension/cdb_dataservices_server--0.4.0--0.5.0.sql @@ -1,4 +1,7 @@ -ALTER TYPE cdb_dataservices_server._redis_conf_params ADD ATTRIBUTE redis_host text; +ALTER TYPE cdb_dataservices_server._redis_conf_params ADD ATTRIBUTE redis_host text + ADD ATTRIBUTE redis_port int + DROP ATTRIBUTE IF EXISTS sentinel_host + DROP ATTRIBUTE IF EXISTS sentinel_port; -- Get the Redis configuration from the _conf table -- CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_redis_conf_v2(config_key text) @@ -11,10 +14,9 @@ RETURNS cdb_dataservices_server._redis_conf_params AS $$ import json params = json.loads(conf) return { - "sentinel_host": params['sentinel_host'], - "sentinel_port": params['sentinel_port'], "sentinel_master_id": params['sentinel_master_id'], - "redis_host": params['redis_host'] + "redis_host": params['redis_host'], + "redis_port": params['redis_port'], "timeout": params['timeout'], "redis_db": params['redis_db'] } @@ -28,22 +30,20 @@ RETURNS boolean AS $$ return False else: from cartodb_services.tools import RedisConnection - metadata_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, - c.sentinel_master_id, c.redis_host, c.timeout, c.redis_db + metadata_config_params = plpy.execute("""select c.sentinel_master_id, c.redis_host, + c.redis_port, c.timeout, c.redis_db from cdb_dataservices_server._get_redis_conf_v2('redis_metadata_config') c;""")[0] - metrics_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, - c.sentinel_master_id, c.redis_host, c.timeout, c.redis_db + metrics_config_params = plpy.execute("""select c.sentinel_master_id, c.redis_host, + c.redis_port, c.timeout, c.redis_db from cdb_dataservices_server._get_redis_conf_v2('redis_metrics_config') c;""")[0] - redis_metadata_connection = RedisConnection(metadata_config_params['sentinel_host'], - metadata_config_params['sentinel_port'], - metadata_config_params['sentinel_master_id'], + redis_metadata_connection = RedisConnection(metadata_config_params['sentinel_master_id'], metadata_config_params['redis_host'], + metadata_config_params['redis_port'], timeout=metadata_config_params['timeout'], redis_db=metadata_config_params['redis_db']).redis_connection() - redis_metrics_connection = RedisConnection(metrics_config_params['sentinel_host'], - metrics_config_params['sentinel_port'], - metrics_config_params['sentinel_master_id'], + redis_metrics_connection = RedisConnection(metrics_config_params['sentinel_master_id'], metrics_config_params['redis_host'], + metrics_config_params['redis_port'], timeout=metrics_config_params['timeout'], redis_db=metrics_config_params['redis_db']).redis_connection() GD[cache_key] = { diff --git a/server/extension/cdb_dataservices_server--0.5.0--0.4.0.sql b/server/extension/cdb_dataservices_server--0.5.0--0.4.0.sql index fcc28c6..db9371d 100644 --- a/server/extension/cdb_dataservices_server--0.5.0--0.4.0.sql +++ b/server/extension/cdb_dataservices_server--0.5.0--0.4.0.sql @@ -1,4 +1,7 @@ -ALTER TYPE cdb_dataservices_server._redis_conf_params DROP ATTRIBUTE IF EXISTS redis_host; +ALTER TYPE cdb_dataservices_server._redis_conf_params DROP ATTRIBUTE IF EXISTS redis_host, + DROP ATTRIBUTE IF EXISTS redis_port, + ADD ATTRIBUTE sentinel_host text, + ADD ATTRIBUTE sentinel_port int; -- Get the Redis configuration from the _conf table -- CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_redis_conf_v2(config_key text) diff --git a/server/extension/test/0.5.0/expected/00_install_test.out b/server/extension/test/0.5.0/expected/00_install_test.out index db041d0..d55743f 100644 --- a/server/extension/test/0.5.0/expected/00_install_test.out +++ b/server/extension/test/0.5.0/expected/00_install_test.out @@ -7,13 +7,13 @@ CREATE EXTENSION cdb_geocoder; -- Install the extension CREATE EXTENSION cdb_dataservices_server; -- Mock the redis server connection to point to this very test db -SELECT cartodb.cdb_conf_setconf('redis_metrics_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); +SELECT cartodb.cdb_conf_setconf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); cdb_conf_setconf ------------------ (1 row) -SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); +SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localhost", "redis_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); cdb_conf_setconf ------------------ diff --git a/server/extension/test/0.5.0/sql/00_install_test.sql b/server/extension/test/0.5.0/sql/00_install_test.sql index da91608..ad69ffb 100644 --- a/server/extension/test/0.5.0/sql/00_install_test.sql +++ b/server/extension/test/0.5.0/sql/00_install_test.sql @@ -9,8 +9,8 @@ CREATE EXTENSION cdb_geocoder; CREATE EXTENSION cdb_dataservices_server; -- Mock the redis server connection to point to this very test db -SELECT cartodb.cdb_conf_setconf('redis_metrics_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); -SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); +SELECT cartodb.cdb_conf_setconf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); +SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localhost", "redis_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); -- Mock the varnish invalidation function -- (used by cdb_geocoder tests) From d2e73a69fa2f974f317437bfbbb05e3cea22f567 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 23 Feb 2016 12:35:12 +0100 Subject: [PATCH 08/12] Python library supports Mapzen routing --- .../cartodb_services/mapzen/__init__.py | 1 + .../cartodb_services/mapzen/exceptions.py | 16 +++ .../cartodb_services/mapzen/routing.py | 118 ++++++++++++++++++ .../cartodb_services/mapzen/types.py | 16 +++ .../cartodb_services/metrics/__init__.py | 2 +- .../cartodb_services/metrics/config.py | 21 ++++ .../cartodb_services/tools/__init__.py | 1 + .../cartodb_services/tools/coordinates.py | 19 +++ .../cartodb_services/tools/redis_tools.py | 2 - .../python/cartodb_services/requirements.txt | 1 + server/lib/python/cartodb_services/setup.py | 2 +- .../test/test_heremapsrouting.py | 1 - .../test/test_mapzenrouting.py | 77 ++++++++++++ 13 files changed, 272 insertions(+), 5 deletions(-) create mode 100644 server/lib/python/cartodb_services/cartodb_services/mapzen/__init__.py create mode 100644 server/lib/python/cartodb_services/cartodb_services/mapzen/exceptions.py create mode 100644 server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py create mode 100644 server/lib/python/cartodb_services/cartodb_services/mapzen/types.py create mode 100644 server/lib/python/cartodb_services/cartodb_services/tools/coordinates.py create mode 100644 server/lib/python/cartodb_services/test/test_mapzenrouting.py diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/__init__.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/__init__.py new file mode 100644 index 0000000..c1a8c1f --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/__init__.py @@ -0,0 +1 @@ +from routing import MapzenRouting, MapzenRoutingResponse diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/exceptions.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/exceptions.py new file mode 100644 index 0000000..92912ec --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/exceptions.py @@ -0,0 +1,16 @@ +#!/usr/local/bin/python +# -*- coding: utf-8 -*- +import json + + +class WrongParams(Exception): + def __init__(self, value): + self.value = value + + def __str__(self): + return repr('Wrong parameters passed: ' + json.dumps(self.value)) + + +class MalformedResult(Exception): + def __str__(self): + return repr('Result structure is malformed') diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py new file mode 100644 index 0000000..2b08d46 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py @@ -0,0 +1,118 @@ +import requests +import json +import re +from polyline.codec import PolylineCodec + +from exceptions import WrongParams +from cartodb_services.tools import Coordinate + + +class MapzenRouting: + 'A Mapzen Routing wrapper for python' + + PRODUCTION_ROUTING_BASE_URL = 'https://valhalla.mapzen.com/route' + + ACCEPTED_MODES = { + "walk": "pedestrian", + "car": "auto", + "public_transport": "bus", + "bicycle": "bicycle" + } + + AUTO_SHORTEST = 'auto_shortest' + + OPTIONAL_PARAMS = [ + 'mode_type', + ] + + METRICS_UNITS = 'kilometers' + IMPERIAL_UNITS = 'miles' + + def __init__(self, app_key, base_url=PRODUCTION_ROUTING_BASE_URL): + self._app_key = app_key + self._url = base_url + + def calculate_route_point_to_point(self, origin, destination, mode, + options=[], units=METRICS_UNITS): + parsed_options = self.__parse_options(options) + mode_param = self.__parse_mode_param(mode, parsed_options) + directions = self.__parse_directions(origin, destination) + json_request_params = self.__parse_json_parameters(directions, + mode_param, + units) + request_params = self.__parse_request_parameters(json_request_params) + response = requests.get(self._url, params=request_params) + if response.status_code == requests.codes.ok: + return self.__parse_routing_response(response.text) + else: + response.raise_for_status() + + def __parse_options(self, options): + return dict(option.split('=') for option in options) + + def __parse_request_parameters(self, json_request): + request_options = {"json": json_request} + request_options.update({'api_key': self._app_key}) + + return request_options + + def __parse_json_parameters(self, directions, mode, units): + json_options = directions + json_options.update({'costing': self.ACCEPTED_MODES[mode]}) + json_options.update({"directions_options": {'units': units, + 'narrative': False}}) + + return json.dumps(json_options) + + def __parse_directions(self, origin, destination): + return {"locations": [ + {"lon": origin.longitude, "lat": origin.latitude}, + {"lon": destination.longitude, "lat": destination.latitude} + ]} + + def __parse_routing_response(self, response): + try: + parsed_json_response = json.loads(response) + legs = parsed_json_response['trip']['legs'][0] + shape = PolylineCodec().decode(legs['shape']) + length = legs['summary']['length'] + duration = legs['summary']['time'] + routing_response = MapzenRoutingResponse(shape, length, duration) + + return routing_response + except IndexError: + return [] + except KeyError: + raise MalformedResult() + + def __parse_mode_param(self, mode, options): + if mode in self.ACCEPTED_MODES: + mode_source = self.ACCEPTED_MODES[mode] + else: + raise WrongParams("{0} is not an accepted mode type".format(mode)) + + if mode == self.ACCEPTED_MODES['car'] and 'mode_type' in options and \ + options['mode_type'] == 'shortest': + mode = self.AUTO_SHORTEST + + return mode + + +class MapzenRoutingResponse: + + def __init__(self, shape, length, duration): + self._shape = shape + self._length = length + self._duration = duration + + @property + def shape(self): + return self._shape + + @property + def length(self): + return self._length + + @property + def duration(self): + return self._duration diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/types.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/types.py new file mode 100644 index 0000000..bfaef3b --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/types.py @@ -0,0 +1,16 @@ +import plpy + + +def polyline_to_linestring(polyline): + """Convert a Mapzen polyline shape to a PostGIS multipolygon""" + coordinates = [] + for point in polyline: + # Divide by 10 because mapzen uses one more decimal than the + # google standard (https://mapzen.com/documentation/turn-by-turn/decoding/) + coordinates.append("%s %s" % (point[1]/10, point[0]/10)) + wkt_coordinates = ','.join(coordinates) + + sql = "SELECT ST_GeomFromText('LINESTRING({0})', 4326) as geom".format(wkt_coordinates) + geometry = plpy.execute(sql, 1)[0]['geom'] + + return geometry diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py b/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py index 208b2be..af7bed5 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py @@ -1,3 +1,3 @@ -from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, ConfigException +from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException from quota import QuotaService from user import UserMetricsService diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index 2e395c5..d2b3b80 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -27,6 +27,27 @@ class ServiceConfig(object): def organization(self): return self._orgname +class RoutingConfig(ServiceConfig): + + ROUTING_CONFIG_KEYS = ['username', 'orgname', 'mapzen_app_key'] + MAPZEN_APP_KEY = 'mapzen_app_key' + USERNAME_KEY = 'username' + ORGNAME_KEY = 'orgname' + + def __init__(self, redis_connection, username, orgname=None, + mapzen_app_key=None): + super(RoutingConfig, self).__init__(redis_connection, username, + orgname) + self._mapzen_app_key = mapzen_app_key + + @property + def service_type(self): + return 'routing_mapzen' + + @property + def mapzen_app_key(self): + return self._mapzen_app_key + class IsolinesRoutingConfig(ServiceConfig): diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py b/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py index dc49770..200dcd4 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py @@ -1 +1,2 @@ from redis_tools import RedisConnection +from coordinates import Coordinate \ No newline at end of file diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/coordinates.py b/server/lib/python/cartodb_services/cartodb_services/tools/coordinates.py new file mode 100644 index 0000000..c8c0748 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/tools/coordinates.py @@ -0,0 +1,19 @@ +class Coordinate: + """Class that represents a generic form of coordinates to be used + by the services """ + + def __init__(self, longitude, latitude): + self._longitude = longitude + self._latitude = latitude + + @property + def latitude(self): + return self._latitude + + @property + def longitude(self): + return self._longitude + + def to_json(self): + return "{{\"lon\": {0},\"lat\": {1}}}".format(self._longitude, + self._latitude) diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py b/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py index 24888dd..cc04f9b 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/redis_tools.py @@ -6,8 +6,6 @@ class RedisConnection: REDIS_DEFAULT_USER_DB = 5 REDIS_DEFAULT_TIMEOUT = 2 #seconds - #REDIS_SENTINEL_DEFAULT_PORT = 26379 - #REDIS_DEFAULT_PORT = 6379 def __init__(self, sentinel_master_id, redis_host, redis_port, redis_db=REDIS_DEFAULT_USER_DB, **kwargs): diff --git a/server/lib/python/cartodb_services/requirements.txt b/server/lib/python/cartodb_services/requirements.txt index 17e41c8..1363d69 100644 --- a/server/lib/python/cartodb_services/requirements.txt +++ b/server/lib/python/cartodb_services/requirements.txt @@ -5,6 +5,7 @@ python-dateutil==2.2 googlemaps==2.4.2 # Dependency for googlemaps package requests<=2.9.1 +polyline==1.1 # Test mock==1.3.0 diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index 3774847..408f147 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.2.0', + version='0.3.0', description='CartoDB Services API Python Library', diff --git a/server/lib/python/cartodb_services/test/test_heremapsrouting.py b/server/lib/python/cartodb_services/test/test_heremapsrouting.py index 1d27f30..3b30b7b 100644 --- a/server/lib/python/cartodb_services/test/test_heremapsrouting.py +++ b/server/lib/python/cartodb_services/test/test_heremapsrouting.py @@ -148,7 +148,6 @@ class HereMapsRoutingIsolineTestCase(unittest.TestCase): u'32.9699707,0.9462833']) def test_calculate_isochrone_with_valid_params(self, req_mock): - print self.isoline_url url = "{0}?start=geo%2133.0%2C1.0&mode=shortest%3Bcar".format(self.isoline_url) req_mock.register_uri('GET', url, text=self.GOOD_RESPONSE) response = self.routing.calculate_isochrone('geo!33.0,1.0', 'car', diff --git a/server/lib/python/cartodb_services/test/test_mapzenrouting.py b/server/lib/python/cartodb_services/test/test_mapzenrouting.py new file mode 100644 index 0000000..5e76536 --- /dev/null +++ b/server/lib/python/cartodb_services/test/test_mapzenrouting.py @@ -0,0 +1,77 @@ +#!/usr/local/bin/python +# -*- coding: utf-8 -*- + +import unittest +import requests_mock +import re +from nose.tools import assert_raises +from urlparse import urlparse, parse_qs + +from cartodb_services.mapzen import MapzenRouting, MapzenRoutingResponse +from cartodb_services.mapzen.exceptions import WrongParams +from cartodb_services.tools import Coordinate + +requests_mock.Mocker.TEST_PREFIX = 'test_' + + +@requests_mock.Mocker() +class MapzenRoutingTestCase(unittest.TestCase): + + GOOD_SHAPE = [(38.5, -120.2), (43.2, -126.4)] + + GOOD_RESPONSE = """{{ + "id": "ethervoid-route", + "trip": {{ + "status": 0, + "status_message": "Found route between points", + "legs": [ + {{ + "shape": "_p~iF~ps|U_~t[~|yd@", + "summary": {{ + "length": 444.59, + "time": 16969 + }} + }} + ], + "units": "kilometers", + "summary": {{ + "length": 444.59, + "time": 16969 + }}, + "locations": [ + {{ + "lon": -120.2, + "lat": 38.5, + "type": "break" + }}, + {{ + "lon": -126.4, + "lat": 43.2, + "type": "break" + }} + ] + }} + }}""".format(GOOD_SHAPE) + + MALFORMED_RESPONSE = """{"manolo": "escobar"}""" + + def setUp(self): + self.routing = MapzenRouting('api_key') + self.url = MapzenRouting.PRODUCTION_ROUTING_BASE_URL + + def test_calculate_simple_routing_with_valid_params(self, req_mock): + req_mock.register_uri('GET', requests_mock.ANY, text=self.GOOD_RESPONSE) + origin = Coordinate('-120.2','38.5') + destination = Coordinate('-126.4','43.2') + response = self.routing.calculate_route_point_to_point(origin, destination,'car') + + self.assertEqual(response.shape, self.GOOD_SHAPE) + self.assertEqual(response.length, 444.59) + self.assertEqual(response.duration, 16969) + + def test_uknown_mode_raise_exception(self, req_mock): + req_mock.register_uri('GET', requests_mock.ANY, text=self.GOOD_RESPONSE) + origin = Coordinate('-120.2','38.5') + destination = Coordinate('-126.4','43.2') + + assert_raises(WrongParams, self.routing.calculate_route_point_to_point, origin, destination, 'unknown') From e6b0e3794b191f555f3bc5a336b82ff118043567 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 23 Feb 2016 17:54:17 +0100 Subject: [PATCH 09/12] Mapzen integration for the server side --- .../cdb_dataservices_server--0.4.0--0.5.0.sql | 106 ++++++++- .../cdb_dataservices_server--0.5.0--0.4.0.sql | 7 + .../cdb_dataservices_server--0.5.0.sql | 122 +++++++++- server/extension/sql/0.5.0/00_header.sql | 4 +- .../sql/0.5.0/100_routing_helper.sql | 55 +++++ .../sql/0.5.0/105_route_point_to_point.sql | 18 ++ .../extension/sql/0.5.0/15_config_helper.sql | 32 ++- .../extension/sql/0.5.0/20_geocode_street.sql | 85 +------ server/extension/sql/0.5.0/30_admin0.sql | 51 +--- server/extension/sql/0.5.0/40_admin1.sql | 118 +--------- server/extension/sql/0.5.0/50_namedplaces.sql | 165 +------------ server/extension/sql/0.5.0/60_postalcodes.sql | 220 +----------------- server/extension/sql/0.5.0/70_ips.sql | 62 +---- .../sql/0.5.0/80_isolines_helper.sql | 1 + .../extension/sql/0.5.0/80_routing_helper.sql | 51 ---- server/extension/sql/0.5.0/85_isodistance.sql | 19 +- server/extension/sql/0.5.0/90_isochrone.sql | 19 +- ..._user.sql => 999_geocoder_server_user.sql} | 0 .../test/0.5.0/expected/00_install_test.out | 6 + .../test/0.5.0/expected/20_street_test.out | 13 +- .../test/0.5.0/expected/30_admin0_test.out | 48 +--- .../test/0.5.0/expected/40_admin1_test.out | 82 +------ .../0.5.0/expected/50_namedplaces_test.out | 137 +---------- .../0.5.0/expected/60_postalcodes_test.out | 164 +------------ .../test/0.5.0/expected/70_ips_test.out | 41 +--- .../0.5.0/expected/85_isodistance_test.out | 13 +- .../test/0.5.0/expected/90_isochrone_test.out | 13 +- .../expected/95_route_point_to_point_test.out | 12 + ... => 999_remove_geocoder_api_user_test.out} | 0 .../test/0.5.0/sql/00_install_test.sql | 1 + .../test/0.5.0/sql/20_street_test.sql | 8 +- .../test/0.5.0/sql/30_admin0_test.sql | 33 +-- .../test/0.5.0/sql/40_admin1_test.sql | 49 +--- .../test/0.5.0/sql/50_namedplaces_test.sql | 73 +----- .../test/0.5.0/sql/60_postalcodes_test.sql | 118 +--------- .../extension/test/0.5.0/sql/70_ips_test.sql | 25 +- .../test/0.5.0/sql/85_isodistance_test.sql | 8 +- .../test/0.5.0/sql/90_isochrone_test.sql | 8 +- .../sql/95_route_point_to_point_test.sql | 7 + ... => 999_remove_geocoder_api_user_test.sql} | 0 40 files changed, 373 insertions(+), 1621 deletions(-) mode change 100644 => 120000 server/extension/sql/0.5.0/00_header.sql create mode 100644 server/extension/sql/0.5.0/100_routing_helper.sql create mode 100644 server/extension/sql/0.5.0/105_route_point_to_point.sql mode change 100644 => 120000 server/extension/sql/0.5.0/20_geocode_street.sql mode change 100644 => 120000 server/extension/sql/0.5.0/30_admin0.sql mode change 100644 => 120000 server/extension/sql/0.5.0/40_admin1.sql mode change 100644 => 120000 server/extension/sql/0.5.0/50_namedplaces.sql mode change 100644 => 120000 server/extension/sql/0.5.0/60_postalcodes.sql mode change 100644 => 120000 server/extension/sql/0.5.0/70_ips.sql create mode 120000 server/extension/sql/0.5.0/80_isolines_helper.sql delete mode 100644 server/extension/sql/0.5.0/80_routing_helper.sql mode change 100644 => 120000 server/extension/sql/0.5.0/85_isodistance.sql mode change 100644 => 120000 server/extension/sql/0.5.0/90_isochrone.sql rename server/extension/sql/0.5.0/{99_geocoder_server_user.sql => 999_geocoder_server_user.sql} (100%) mode change 100644 => 120000 server/extension/test/0.5.0/expected/20_street_test.out mode change 100644 => 120000 server/extension/test/0.5.0/expected/30_admin0_test.out mode change 100644 => 120000 server/extension/test/0.5.0/expected/40_admin1_test.out mode change 100644 => 120000 server/extension/test/0.5.0/expected/50_namedplaces_test.out mode change 100644 => 120000 server/extension/test/0.5.0/expected/60_postalcodes_test.out mode change 100644 => 120000 server/extension/test/0.5.0/expected/70_ips_test.out mode change 100644 => 120000 server/extension/test/0.5.0/expected/85_isodistance_test.out mode change 100644 => 120000 server/extension/test/0.5.0/expected/90_isochrone_test.out create mode 100644 server/extension/test/0.5.0/expected/95_route_point_to_point_test.out rename server/extension/test/0.5.0/expected/{99_remove_geocoder_api_user_test.out => 999_remove_geocoder_api_user_test.out} (100%) mode change 100644 => 120000 server/extension/test/0.5.0/sql/20_street_test.sql mode change 100644 => 120000 server/extension/test/0.5.0/sql/30_admin0_test.sql mode change 100644 => 120000 server/extension/test/0.5.0/sql/40_admin1_test.sql mode change 100644 => 120000 server/extension/test/0.5.0/sql/50_namedplaces_test.sql mode change 100644 => 120000 server/extension/test/0.5.0/sql/60_postalcodes_test.sql mode change 100644 => 120000 server/extension/test/0.5.0/sql/70_ips_test.sql mode change 100644 => 120000 server/extension/test/0.5.0/sql/85_isodistance_test.sql mode change 100644 => 120000 server/extension/test/0.5.0/sql/90_isochrone_test.sql create mode 100644 server/extension/test/0.5.0/sql/95_route_point_to_point_test.sql rename server/extension/test/0.5.0/sql/{99_remove_geocoder_api_user_test.sql => 999_remove_geocoder_api_user_test.sql} (100%) diff --git a/server/extension/cdb_dataservices_server--0.4.0--0.5.0.sql b/server/extension/cdb_dataservices_server--0.4.0--0.5.0.sql index 11e2e71..06abdb3 100644 --- a/server/extension/cdb_dataservices_server--0.4.0--0.5.0.sql +++ b/server/extension/cdb_dataservices_server--0.4.0--0.5.0.sql @@ -1,7 +1,7 @@ -ALTER TYPE cdb_dataservices_server._redis_conf_params ADD ATTRIBUTE redis_host text - ADD ATTRIBUTE redis_port int - DROP ATTRIBUTE IF EXISTS sentinel_host - DROP ATTRIBUTE IF EXISTS sentinel_port; +ALTER TYPE cdb_dataservices_server._redis_conf_params ADD ATTRIBUTE redis_host text; +ALTER TYPE cdb_dataservices_server._redis_conf_params ADD ATTRIBUTE redis_port int; +ALTER TYPE cdb_dataservices_server._redis_conf_params DROP ATTRIBUTE IF EXISTS sentinel_host; +ALTER TYPE cdb_dataservices_server._redis_conf_params DROP ATTRIBUTE IF EXISTS sentinel_port; -- Get the Redis configuration from the _conf table -- CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_redis_conf_v2(config_key text) @@ -52,3 +52,101 @@ RETURNS boolean AS $$ } return True $$ LANGUAGE plpythonu SECURITY DEFINER; + +-- Mapzen routing integration + +CREATE TYPE cdb_dataservices_server.simple_route AS ( + shape geometry(LineString,4326), + length real, + duration integer +); + +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + import json + from cartodb_services.metrics import RoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + mapzen_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('mapzen_conf') as mapzen_conf", 1)[0]['mapzen_conf'] + if not mapzen_conf_json: + mapzen_app_key = None + else: + mapzen_conf = json.loads(mapzen_conf_json) + mapzen_app_key = mapzen_conf['routing_app_key'] + routing_config = RoutingConfig(redis_conn, username, orgname, mapzen_app_key) + GD[cache_key] = routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_point_to_point( + username TEXT, + orgname TEXT, + origin geometry(Point, 4326), + destination geometry(Point, 4326), + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + import json + from cartodb_services.mapzen import MapzenRouting, MapzenRoutingResponse + from cartodb_services.mapzen.types import polyline_to_linestring + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Coordinate + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_routing_config = GD["user_routing_config_{0}".format(username)] + + quota_service = QuotaService(user_routing_config, redis_conn) + + try: + client = MapzenRouting(user_routing_config.mapzen_app_key) + + orig_lat = plpy.execute("SELECT ST_Y('%s') AS lat" % origin)[0]['lat'] + orig_lon = plpy.execute("SELECT ST_X('%s') AS lon" % origin)[0]['lon'] + origin_coordinates = Coordinate(orig_lon, orig_lat) + dest_lat = plpy.execute("SELECT ST_Y('%s') AS lat" % destination)[0]['lat'] + dest_lon = plpy.execute("SELECT ST_X('%s') AS lon" % destination)[0]['lon'] + dest_coordinates = Coordinate(dest_lon, dest_lat) + + resp = client.calculate_route_point_to_point(origin_coordinates, dest_coordinates, mode, options, units) + + if resp: + shape_linestring = polyline_to_linestring(resp.shape) + quota_service.increment_success_geocoder_use() + return [shape_linestring, resp.length, resp.duration] + else: + quota_service.increment_empty_geocoder_use() + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to obtain route using mapzen provider: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point( + username TEXT, + orgname TEXT, + origin geometry(Point, 4326), + destination geometry(Point, 4326), + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_point_to_point($1, $2, $3, $4, $5, $6, $7) as route;", ["text", "text", "geometry(Point, 4326)", "geometry(Point, 4326)", "text", "text[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, origin, destination, mode, options, units]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] +$$ LANGUAGE plpythonu; diff --git a/server/extension/cdb_dataservices_server--0.5.0--0.4.0.sql b/server/extension/cdb_dataservices_server--0.5.0--0.4.0.sql index db9371d..17d9203 100644 --- a/server/extension/cdb_dataservices_server--0.5.0--0.4.0.sql +++ b/server/extension/cdb_dataservices_server--0.5.0--0.4.0.sql @@ -52,3 +52,10 @@ RETURNS boolean AS $$ } return True $$ LANGUAGE plpythonu SECURITY DEFINER; + +-- Mapzen integration + +DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_route_point_to_point(TEXT, TEXT, geometry(Point, 4326), geometry(Point, 4326), TEXT, text[], text); +DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_mapzen_route_point_to_point(TEXT, TEXT, geometry(Point, 4326), geometry(Point, 4326), TEXT, text[], text); +DROP FUNCTION IF EXISTS cdb_dataservices_server._get_routing_config(text, text); +DROP TYPE IF EXISTS cdb_dataservices_server.simple_route; diff --git a/server/extension/cdb_dataservices_server--0.5.0.sql b/server/extension/cdb_dataservices_server--0.5.0.sql index 9f027b8..9d1d8f6 100644 --- a/server/extension/cdb_dataservices_server--0.5.0.sql +++ b/server/extension/cdb_dataservices_server--0.5.0.sql @@ -1,6 +1,79 @@ --DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES -- Complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "CREATE EXTENSION cdb_dataservices_server" to load this file. \quit +CREATE TYPE cdb_dataservices_server.simple_route AS ( + shape geometry(LineString,4326), + length real, + duration integer +); + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_point_to_point( + username TEXT, + orgname TEXT, + origin geometry(Point, 4326), + destination geometry(Point, 4326), + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + import json + from cartodb_services.mapzen import MapzenRouting, MapzenRoutingResponse + from cartodb_services.mapzen.types import polyline_to_linestring + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Coordinate + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_routing_config = GD["user_routing_config_{0}".format(username)] + + quota_service = QuotaService(user_routing_config, redis_conn) + + try: + client = MapzenRouting(user_routing_config.mapzen_app_key) + + orig_lat = plpy.execute("SELECT ST_Y('%s') AS lat" % origin)[0]['lat'] + orig_lon = plpy.execute("SELECT ST_X('%s') AS lon" % origin)[0]['lon'] + origin_coordinates = Coordinate(orig_lon, orig_lat) + dest_lat = plpy.execute("SELECT ST_Y('%s') AS lat" % destination)[0]['lat'] + dest_lon = plpy.execute("SELECT ST_X('%s') AS lon" % destination)[0]['lon'] + dest_coordinates = Coordinate(dest_lon, dest_lat) + + resp = client.calculate_route_point_to_point(origin_coordinates, dest_coordinates, mode, options, units) + + if resp: + shape_linestring = polyline_to_linestring(resp.shape) + quota_service.increment_success_geocoder_use() + return [shape_linestring, resp.length, resp.duration] + else: + quota_service.increment_empty_geocoder_use() + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to obtain route using mapzen provider: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu SECURITY DEFINER; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point( + username TEXT, + orgname TEXT, + origin geometry(Point, 4326), + destination geometry(Point, 4326), + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_point_to_point($1, $2, $3, $4, $5, $6, $7) as route;", ["text", "text", "geometry(Point, 4326)", "geometry(Point, 4326)", "text", "text[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, origin, destination, mode, options, units]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] +$$ LANGUAGE plpythonu; CREATE TYPE cdb_dataservices_server._redis_conf_params AS ( sentinel_master_id text, redis_host text, @@ -85,14 +158,14 @@ RETURNS boolean AS $$ $$ LANGUAGE plpythonu SECURITY DEFINER; -- Get the Redis configuration from the _conf table -- -CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_isolines_routing_config(username text, orgname text) RETURNS boolean AS $$ - cache_key = "user_routing_config_{0}".format(username) + cache_key = "user_isolines_routing_config_{0}".format(username) if cache_key in GD: return False else: import json - from cartodb_services.metrics import RoutingConfig + from cartodb_services.metrics import IsolinesRoutingConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf'] @@ -103,7 +176,31 @@ RETURNS boolean AS $$ heremaps_conf = json.loads(heremaps_conf_json) heremaps_app_id = heremaps_conf['app_id'] heremaps_app_code = heremaps_conf['app_code'] - routing_config = RoutingConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code) + isolines_routing_config = IsolinesRoutingConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code) + # --Think about the security concerns with this kind of global cache, it should be only available + # --for this user session but... + GD[cache_key] = isolines_routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + import json + from cartodb_services.metrics import RoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + mapzen_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('mapzen_conf') as mapzen_conf", 1)[0]['mapzen_conf'] + if not mapzen_conf_json: + mapzen_app_key = None + else: + mapzen_conf = json.loads(mapzen_conf_json) + mapzen_app_key = mapzen_conf['routing_app_key'] + routing_config = RoutingConfig(redis_conn, username, orgname, mapzen_app_key) # --Think about the security concerns with this kind of global cache, it should be only available # --for this user session but... GD[cache_key] = routing_config @@ -814,12 +911,15 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ from cartodb_services.here.types import geo_polyline_to_multipolygon redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_routing_config = GD["user_routing_config_{0}".format(username)] + user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] - quota_service = QuotaService(user_routing_config, redis_conn) + # -- Check the quota + quota_service = QuotaService(user_isolines_routing_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reach the limit of your quota') try: - client = HereMapsRoutingIsoline(user_routing_config.heremaps_app_id, user_routing_config.heremaps_app_code, base_url = HereMapsRoutingIsoline.PRODUCTION_ROUTING_BASE_URL) + client = HereMapsRoutingIsoline(user_isolines_routing_config.heremaps_app_id, user_isolines_routing_config.heremaps_app_code, base_url = HereMapsRoutingIsoline.PRODUCTION_ROUTING_BASE_URL) if source: lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] @@ -859,8 +959,8 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT RETURNS SETOF cdb_dataservices_server.isoline AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_isolines_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] type = 'isodistance' here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) @@ -877,8 +977,8 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, RETURNS SETOF cdb_dataservices_server.isoline AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_isolines_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] type = 'isochrone' here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) diff --git a/server/extension/sql/0.5.0/00_header.sql b/server/extension/sql/0.5.0/00_header.sql deleted file mode 100644 index 2c8035c..0000000 --- a/server/extension/sql/0.5.0/00_header.sql +++ /dev/null @@ -1,3 +0,0 @@ ---DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES --- Complain if script is sourced in psql, rather than via CREATE EXTENSION -\echo Use "CREATE EXTENSION cdb_dataservices_server" to load this file. \quit diff --git a/server/extension/sql/0.5.0/00_header.sql b/server/extension/sql/0.5.0/00_header.sql new file mode 120000 index 0000000..6d12de2 --- /dev/null +++ b/server/extension/sql/0.5.0/00_header.sql @@ -0,0 +1 @@ +../0.4.0/00_header.sql \ No newline at end of file diff --git a/server/extension/sql/0.5.0/100_routing_helper.sql b/server/extension/sql/0.5.0/100_routing_helper.sql new file mode 100644 index 0000000..e23a1c5 --- /dev/null +++ b/server/extension/sql/0.5.0/100_routing_helper.sql @@ -0,0 +1,55 @@ +CREATE TYPE cdb_dataservices_server.simple_route AS ( + shape geometry(LineString,4326), + length real, + duration integer +); + + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_route_point_to_point( + username TEXT, + orgname TEXT, + origin geometry(Point, 4326), + destination geometry(Point, 4326), + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + import json + from cartodb_services.mapzen import MapzenRouting, MapzenRoutingResponse + from cartodb_services.mapzen.types import polyline_to_linestring + from cartodb_services.metrics import QuotaService + from cartodb_services.tools import Coordinate + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_routing_config = GD["user_routing_config_{0}".format(username)] + + quota_service = QuotaService(user_routing_config, redis_conn) + + try: + client = MapzenRouting(user_routing_config.mapzen_app_key) + + orig_lat = plpy.execute("SELECT ST_Y('%s') AS lat" % origin)[0]['lat'] + orig_lon = plpy.execute("SELECT ST_X('%s') AS lon" % origin)[0]['lon'] + origin_coordinates = Coordinate(orig_lon, orig_lat) + dest_lat = plpy.execute("SELECT ST_Y('%s') AS lat" % destination)[0]['lat'] + dest_lon = plpy.execute("SELECT ST_X('%s') AS lon" % destination)[0]['lon'] + dest_coordinates = Coordinate(dest_lon, dest_lat) + + resp = client.calculate_route_point_to_point(origin_coordinates, dest_coordinates, mode, options, units) + + if resp: + shape_linestring = polyline_to_linestring(resp.shape) + quota_service.increment_success_geocoder_use() + return [shape_linestring, resp.length, resp.duration] + else: + quota_service.increment_empty_geocoder_use() + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to obtain route using mapzen provider: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu SECURITY DEFINER; diff --git a/server/extension/sql/0.5.0/105_route_point_to_point.sql b/server/extension/sql/0.5.0/105_route_point_to_point.sql new file mode 100644 index 0000000..77a54a3 --- /dev/null +++ b/server/extension/sql/0.5.0/105_route_point_to_point.sql @@ -0,0 +1,18 @@ +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point( + username TEXT, + orgname TEXT, + origin geometry(Point, 4326), + destination geometry(Point, 4326), + mode TEXT, + options text[] DEFAULT ARRAY[]::text[], + units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_server.simple_route AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_routing_config = GD["user_routing_config_{0}".format(username)] + + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_point_to_point($1, $2, $3, $4, $5, $6, $7) as route;", ["text", "text", "geometry(Point, 4326)", "geometry(Point, 4326)", "text", "text[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, origin, destination, mode, options, units]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] +$$ LANGUAGE plpythonu; diff --git a/server/extension/sql/0.5.0/15_config_helper.sql b/server/extension/sql/0.5.0/15_config_helper.sql index 47aad42..b70d63a 100644 --- a/server/extension/sql/0.5.0/15_config_helper.sql +++ b/server/extension/sql/0.5.0/15_config_helper.sql @@ -25,14 +25,14 @@ RETURNS boolean AS $$ $$ LANGUAGE plpythonu SECURITY DEFINER; -- Get the Redis configuration from the _conf table -- -CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_isolines_routing_config(username text, orgname text) RETURNS boolean AS $$ - cache_key = "user_routing_config_{0}".format(username) + cache_key = "user_isolines_routing_config_{0}".format(username) if cache_key in GD: return False else: import json - from cartodb_services.metrics import RoutingConfig + from cartodb_services.metrics import IsolinesRoutingConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf'] @@ -43,7 +43,31 @@ RETURNS boolean AS $$ heremaps_conf = json.loads(heremaps_conf_json) heremaps_app_id = heremaps_conf['app_id'] heremaps_app_code = heremaps_conf['app_code'] - routing_config = RoutingConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code) + isolines_routing_config = IsolinesRoutingConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code) + # --Think about the security concerns with this kind of global cache, it should be only available + # --for this user session but... + GD[cache_key] = isolines_routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + import json + from cartodb_services.metrics import RoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + mapzen_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('mapzen_conf') as mapzen_conf", 1)[0]['mapzen_conf'] + if not mapzen_conf_json: + mapzen_app_key = None + else: + mapzen_conf = json.loads(mapzen_conf_json) + mapzen_app_key = mapzen_conf['routing_app_key'] + routing_config = RoutingConfig(redis_conn, username, orgname, mapzen_app_key) # --Think about the security concerns with this kind of global cache, it should be only available # --for this user session but... GD[cache_key] = routing_config diff --git a/server/extension/sql/0.5.0/20_geocode_street.sql b/server/extension/sql/0.5.0/20_geocode_street.sql deleted file mode 100644 index de77f62..0000000 --- a/server/extension/sql/0.5.0/20_geocode_street.sql +++ /dev/null @@ -1,84 +0,0 @@ --- Geocodes a street address given a searchtext and a state and/or country -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) -RETURNS Geometry AS $$ - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] - - if user_geocoder_config.heremaps_geocoder: - here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) - return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] - elif user_geocoder_config.google_geocoder: - google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) - return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] - else: - plpy.error('Requested geocoder is not available') - -$$ LANGUAGE plpythonu; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) -RETURNS Geometry AS $$ - from cartodb_services.here import HereMapsGeocoder - from cartodb_services.metrics import QuotaService - - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] - - # -- Check the quota - quota_service = QuotaService(user_geocoder_config, redis_conn) - if not quota_service.check_user_quota(): - plpy.error('You have reach the limit of your quota') - - try: - geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code) - coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) - if coordinates: - quota_service.increment_success_geocoder_use() - plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) - point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] - return point['st_setsrid'] - else: - quota_service.increment_empty_geocoder_use() - return None - except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() - quota_service.increment_failed_geocoder_use() - error_msg = 'There was an error trying to geocode using here maps geocoder: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) - finally: - quota_service.increment_total_geocoder_use() -$$ LANGUAGE plpythonu; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) -RETURNS Geometry AS $$ - from cartodb_services.google import GoogleMapsGeocoder - from cartodb_services.metrics import QuotaService - - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] - quota_service = QuotaService(user_geocoder_config, redis_conn) - - try: - geocoder = GoogleMapsGeocoder(user_geocoder_config.google_client_id, user_geocoder_config.google_api_key) - coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) - if coordinates: - quota_service.increment_success_geocoder_use() - plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) - point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] - return point['st_setsrid'] - else: - quota_service.increment_empty_geocoder_use() - return None - except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() - quota_service.increment_failed_geocoder_use() - error_msg = 'There was an error trying to geocode using google maps geocoder: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) - finally: - quota_service.increment_total_geocoder_use() -$$ LANGUAGE plpythonu; diff --git a/server/extension/sql/0.5.0/20_geocode_street.sql b/server/extension/sql/0.5.0/20_geocode_street.sql new file mode 120000 index 0000000..30ce5a4 --- /dev/null +++ b/server/extension/sql/0.5.0/20_geocode_street.sql @@ -0,0 +1 @@ +../0.4.0/20_geocode_street.sql \ No newline at end of file diff --git a/server/extension/sql/0.5.0/30_admin0.sql b/server/extension/sql/0.5.0/30_admin0.sql deleted file mode 100644 index ec8fe5c..0000000 --- a/server/extension/sql/0.5.0/30_admin0.sql +++ /dev/null @@ -1,50 +0,0 @@ -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) -RETURNS Geometry AS $$ - from cartodb_services.metrics import QuotaService - from cartodb_services.metrics import InternalGeocoderConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) - - quota_service = QuotaService(user_geocoder_config, redis_conn) - try: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"]) - rv = plpy.execute(plan, [country_name], 1) - result = rv[0]["mypolygon"] - if result: - quota_service.increment_success_geocoder_use() - return result - else: - quota_service.increment_empty_geocoder_use() - return None - except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() - quota_service.increment_failed_geocoder_use() - error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) - finally: - quota_service.increment_total_geocoder_use() -$$ LANGUAGE plpythonu; - - --------------------------------------------------------------------------------- - --- Implementation of the server extension --- Note: these functions depend on the cdb_geocoder extension -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin0_polygon(country_name text) -RETURNS Geometry AS $$ - DECLARE - ret Geometry; - BEGIN - SELECT n.the_geom as geom INTO ret - FROM (SELECT q, lower(regexp_replace(q, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text x - FROM (SELECT country_name q) g) d - LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x - LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.the_geom, s.adm0_a3; - - RETURN ret; - END -$$ LANGUAGE plpgsql; diff --git a/server/extension/sql/0.5.0/30_admin0.sql b/server/extension/sql/0.5.0/30_admin0.sql new file mode 120000 index 0000000..5b199e6 --- /dev/null +++ b/server/extension/sql/0.5.0/30_admin0.sql @@ -0,0 +1 @@ +../0.4.0/30_admin0.sql \ No newline at end of file diff --git a/server/extension/sql/0.5.0/40_admin1.sql b/server/extension/sql/0.5.0/40_admin1.sql deleted file mode 100644 index dc8e037..0000000 --- a/server/extension/sql/0.5.0/40_admin1.sql +++ /dev/null @@ -1,117 +0,0 @@ ----- cdb_geocode_admin1_polygon(admin1_name text) -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) -RETURNS Geometry AS $$ - from cartodb_services.metrics import QuotaService - from cartodb_services.metrics import InternalGeocoderConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) - - quota_service = QuotaService(user_geocoder_config, redis_conn) - try: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"]) - rv = plpy.execute(plan, [admin1_name], 1) - result = rv[0]["mypolygon"] - if result: - quota_service.increment_success_geocoder_use() - return result - else: - quota_service.increment_empty_geocoder_use() - return None - except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() - quota_service.increment_failed_geocoder_use() - error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) - finally: - quota_service.increment_total_geocoder_use() -$$ LANGUAGE plpythonu; - ----- cdb_geocode_admin1_polygon(admin1_name text, country_name text) -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) -RETURNS Geometry AS $$ - from cartodb_services.metrics import QuotaService - from cartodb_services.metrics import InternalGeocoderConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) - - quota_service = QuotaService(user_geocoder_config, redis_conn) - try: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"]) - rv = plpy.execute(plan, [admin1_name, country_name], 1) - result = rv[0]["mypolygon"] - if result: - quota_service.increment_success_geocoder_use() - return result - else: - quota_service.increment_empty_geocoder_use() - return None - except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() - quota_service.increment_failed_geocoder_use() - error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) - finally: - quota_service.increment_total_geocoder_use() -$$ LANGUAGE plpythonu; - --------------------------------------------------------------------------------- - --- Implementation of the server extension --- Note: these functions depend on the cdb_geocoder extension - ----- cdb_geocode_admin1_polygon(admin1_name text) -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text) -RETURNS Geometry AS $$ - DECLARE - ret Geometry; - BEGIN - SELECT geom INTO ret - FROM ( - SELECT q, ( - SELECT the_geom - FROM global_province_polygons - WHERE d.c = ANY (synonyms) - ORDER BY frequency DESC LIMIT 1 - ) geom - FROM ( - SELECT - trim(replace(lower(admin1_name),'.',' ')) c, admin1_name q - ) d - ) v; - - RETURN ret; - END -$$ LANGUAGE plpgsql; - ----- cdb_geocode_admin1_polygon(admin1_name text, country_name text) -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text, country_name text) -RETURNS Geometry AS $$ - DECLARE - ret Geometry; - BEGIN - WITH p AS (SELECT r.c, r.q, (SELECT iso3 FROM country_decoder WHERE lower(country_name) = ANY (synonyms)) i FROM (SELECT trim(replace(lower(admin1_name),'.',' ')) c, country_name q) r) - SELECT - geom INTO ret - FROM ( - SELECT - q, ( - SELECT the_geom - FROM global_province_polygons - WHERE p.c = ANY (synonyms) - AND iso3 = p.i - ORDER BY frequency DESC LIMIT 1 - ) geom - FROM p) n; - - RETURN ret; - END -$$ LANGUAGE plpgsql; - diff --git a/server/extension/sql/0.5.0/40_admin1.sql b/server/extension/sql/0.5.0/40_admin1.sql new file mode 120000 index 0000000..40dbd30 --- /dev/null +++ b/server/extension/sql/0.5.0/40_admin1.sql @@ -0,0 +1 @@ +../0.4.0/40_admin1.sql \ No newline at end of file diff --git a/server/extension/sql/0.5.0/50_namedplaces.sql b/server/extension/sql/0.5.0/50_namedplaces.sql deleted file mode 100644 index 80306cb..0000000 --- a/server/extension/sql/0.5.0/50_namedplaces.sql +++ /dev/null @@ -1,164 +0,0 @@ ----- cdb_geocode_namedplace_point(city_name text) -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) -RETURNS Geometry AS $$ - from cartodb_services.metrics import QuotaService - from cartodb_services.metrics import InternalGeocoderConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) - - quota_service = QuotaService(user_geocoder_config, redis_conn) - try: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"]) - rv = plpy.execute(plan, [city_name], 1) - result = rv[0]["mypoint"] - if result: - quota_service.increment_success_geocoder_use() - return result - else: - quota_service.increment_empty_geocoder_use() - return None - except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() - quota_service.increment_failed_geocoder_use() - error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) - finally: - quota_service.increment_total_geocoder_use() -$$ LANGUAGE plpythonu; - ----- cdb_geocode_namedplace_point(city_name text, country_name text) -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) -RETURNS Geometry AS $$ - from cartodb_services.metrics import QuotaService - from cartodb_services.metrics import InternalGeocoderConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) - - quota_service = QuotaService(user_geocoder_config, redis_conn) - try: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"]) - rv = plpy.execute(plan, [city_name, country_name], 1) - result = rv[0]["mypoint"] - if result: - quota_service.increment_success_geocoder_use() - return result - else: - quota_service.increment_empty_geocoder_use() - return None - except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() - quota_service.increment_failed_geocoder_use() - error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) - finally: - quota_service.increment_total_geocoder_use() -$$ LANGUAGE plpythonu; - ----- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) -RETURNS Geometry AS $$ - from cartodb_services.metrics import QuotaService - from cartodb_services.metrics import InternalGeocoderConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) - - quota_service = QuotaService(user_geocoder_config, redis_conn) - try: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"]) - rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) - result = rv[0]["mypoint"] - if result: - quota_service.increment_success_geocoder_use() - return result - else: - quota_service.increment_empty_geocoder_use() - return None - except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() - quota_service.increment_failed_geocoder_use() - error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) - finally: - quota_service.increment_total_geocoder_use() -$$ LANGUAGE plpythonu; - --------------------------------------------------------------------------------- - --- Implementation of the server extension --- Note: these functions depend on the cdb_geocoder extension - ----- cdb_geocode_namedplace_point(city_name text) -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text) -RETURNS Geometry AS $$ - DECLARE - ret Geometry; - BEGIN - SELECT geom INTO ret - FROM ( - WITH best AS (SELECT s AS q, (SELECT the_geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT city_name as s) p), - next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT city_name as s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) - SELECT q, geom, TRUE AS success FROM best WHERE geom IS NOT NULL - UNION ALL - SELECT q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next - ) v; - - RETURN ret; - END -$$ LANGUAGE plpgsql; - ----- cdb_geocode_namedplace_point(city_name text, country_name text) -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, country_name text) -RETURNS Geometry AS $$ - DECLARE - ret Geometry; - BEGIN - SELECT geom INTO ret - FROM ( - WITH p AS (SELECT r.s, r.c, (SELECT iso2 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT city_name AS s, country_name::text AS c) r), - best AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), - next AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE c = p.c AND geom IS NOT NULL)) - SELECT geom FROM best WHERE geom IS NOT NULL - UNION ALL - SELECT geom FROM next - ) v; - - RETURN ret; - END -$$ LANGUAGE plpgsql; - ----- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) -RETURNS Geometry AS $$ - DECLARE - ret Geometry; - BEGIN - SELECT geom INTO ret - FROM ( - WITH inputcountry AS ( - SELECT iso2 as isoTwo FROM country_decoder WHERE lower(country_name) = ANY (synonyms) LIMIT 1 - ), - p AS ( - SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder, inputcountry WHERE lower(r.a1) = ANY (synonyms) AND admin1_decoder.iso2 = inputcountry.isoTwo LIMIT 1) i FROM (SELECT city_name AS s, admin1_name::text AS a1) r), - best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), - next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) - SELECT geom FROM best WHERE geom IS NOT NULL - UNION ALL - SELECT geom FROM next - ) v; - - RETURN ret; - END -$$ LANGUAGE plpgsql; - diff --git a/server/extension/sql/0.5.0/50_namedplaces.sql b/server/extension/sql/0.5.0/50_namedplaces.sql new file mode 120000 index 0000000..eb4d63f --- /dev/null +++ b/server/extension/sql/0.5.0/50_namedplaces.sql @@ -0,0 +1 @@ +../0.4.0/50_namedplaces.sql \ No newline at end of file diff --git a/server/extension/sql/0.5.0/60_postalcodes.sql b/server/extension/sql/0.5.0/60_postalcodes.sql deleted file mode 100644 index 4ba321b..0000000 --- a/server/extension/sql/0.5.0/60_postalcodes.sql +++ /dev/null @@ -1,219 +0,0 @@ -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text) -RETURNS Geometry AS $$ - from cartodb_services.metrics import QuotaService - from cartodb_services.metrics import InternalGeocoderConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) - - quota_service = QuotaService(user_geocoder_config, redis_conn) - try: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"]) - rv = plpy.execute(plan, [code], 1) - result = rv[0]["mypoint"] - if result: - quota_service.increment_success_geocoder_use() - return result - else: - quota_service.increment_empty_geocoder_use() - return None - except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() - quota_service.increment_failed_geocoder_use() - error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) - finally: - quota_service.increment_total_geocoder_use() -$$ LANGUAGE plpythonu; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text) -RETURNS Geometry AS $$ - from cartodb_services.metrics import QuotaService - from cartodb_services.metrics import InternalGeocoderConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) - - quota_service = QuotaService(user_geocoder_config, redis_conn) - try: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"]) - rv = plpy.execute(plan, [code, country], 1) - result = rv[0]["mypoint"] - if result: - quota_service.increment_success_geocoder_use() - return result - else: - quota_service.increment_empty_geocoder_use() - return None - except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() - quota_service.increment_failed_geocoder_use() - error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) - finally: - quota_service.increment_total_geocoder_use() -$$ LANGUAGE plpythonu; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text) -RETURNS Geometry AS $$ - from cartodb_services.metrics import QuotaService - from cartodb_services.metrics import InternalGeocoderConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) - - quota_service = QuotaService(user_geocoder_config, redis_conn) - try: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"]) - rv = plpy.execute(plan, [code], 1) - result = rv[0]["mypolygon"] - if result: - quota_service.increment_success_geocoder_use() - return result - else: - quota_service.increment_empty_geocoder_use() - return None - except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() - quota_service.increment_failed_geocoder_use() - error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) - finally: - quota_service.increment_total_geocoder_use() -$$ LANGUAGE plpythonu; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text) -RETURNS Geometry AS $$ - from cartodb_services.metrics import QuotaService - from cartodb_services.metrics import InternalGeocoderConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) - - quota_service = QuotaService(user_geocoder_config, redis_conn) - try: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"]) - rv = plpy.execute(plan, [code, country], 1) - result = rv[0]["mypolygon"] - if result: - quota_service.increment_success_geocoder_use() - return result - else: - quota_service.increment_empty_geocoder_use() - return None - except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() - quota_service.increment_failed_geocoder_use() - error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) - finally: - quota_service.increment_total_geocoder_use() -$$ LANGUAGE plpythonu; - --------------------------------------------------------------------------------- - --- Implementation of the server extension --- Note: these functions depend on the cdb_geocoder extension -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text) -RETURNS Geometry AS $$ - DECLARE - ret Geometry; - BEGIN - SELECT geom INTO ret - FROM ( - SELECT - q, ( - SELECT the_geom - FROM global_postal_code_points - WHERE postal_code = upper(d.q) - LIMIT 1 - ) geom - FROM (SELECT code q) d - ) v; - - RETURN ret; -END -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text, country text) -RETURNS Geometry AS $$ - DECLARE - ret Geometry; - BEGIN - SELECT geom INTO ret - FROM ( - SELECT - q, ( - SELECT the_geom - FROM global_postal_code_points - WHERE postal_code = upper(d.q) - AND iso3 = ( - SELECT iso3 FROM country_decoder WHERE - lower(country) = ANY (synonyms) LIMIT 1 - ) - LIMIT 1 - ) geom - FROM (SELECT code q) d - ) v; - - RETURN ret; -END -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text) -RETURNS Geometry AS $$ - DECLARE - ret Geometry; - BEGIN - SELECT geom INTO ret - FROM ( - SELECT - q, ( - SELECT the_geom - FROM global_postal_code_polygons - WHERE postal_code = upper(d.q) - LIMIT 1 - ) geom - FROM (SELECT code q) d - ) v; - - RETURN ret; -END -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text, country text) -RETURNS Geometry AS $$ - DECLARE - ret Geometry; - BEGIN - SELECT geom INTO ret - FROM ( - SELECT - q, ( - SELECT the_geom - FROM global_postal_code_polygons - WHERE postal_code = upper(d.q) - AND iso3 = ( - SELECT iso3 FROM country_decoder WHERE - lower(country) = ANY (synonyms) LIMIT 1 - ) - LIMIT 1 - ) geom - FROM (SELECT code q) d - ) v; - - RETURN ret; -END -$$ LANGUAGE plpgsql; diff --git a/server/extension/sql/0.5.0/60_postalcodes.sql b/server/extension/sql/0.5.0/60_postalcodes.sql new file mode 120000 index 0000000..176dd6c --- /dev/null +++ b/server/extension/sql/0.5.0/60_postalcodes.sql @@ -0,0 +1 @@ +../0.4.0/60_postalcodes.sql \ No newline at end of file diff --git a/server/extension/sql/0.5.0/70_ips.sql b/server/extension/sql/0.5.0/70_ips.sql deleted file mode 100644 index f76190f..0000000 --- a/server/extension/sql/0.5.0/70_ips.sql +++ /dev/null @@ -1,61 +0,0 @@ -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) -RETURNS Geometry AS $$ - from cartodb_services.metrics import QuotaService - from cartodb_services.metrics import InternalGeocoderConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) - - quota_service = QuotaService(user_geocoder_config, redis_conn) - try: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"]) - rv = plpy.execute(plan, [ip], 1) - result = rv[0]["mypoint"] - if result: - quota_service.increment_success_geocoder_use() - return result - else: - quota_service.increment_empty_geocoder_use() - return None - except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() - quota_service.increment_failed_geocoder_use() - error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) - finally: - quota_service.increment_total_geocoder_use() -$$ LANGUAGE plpythonu; - --------------------------------------------------------------------------------- - --- Implementation of the server extension --- Note: these functions depend on the cdb_geocoder extension -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_ipaddress_point(ip text) -RETURNS Geometry AS $$ - DECLARE - ret Geometry; - - new_ip INET; - BEGIN - BEGIN - IF family(ip::inet) = 6 THEN - new_ip := ip::inet; - ELSE - new_ip := ('::ffff:' || ip)::inet; - END IF; - EXCEPTION WHEN OTHERS THEN - SELECT NULL as geom INTO ret; - RETURN ret; - END; - - WITH - ips AS (SELECT ip s, new_ip net), - matches AS (SELECT s, (SELECT the_geom FROM ip_address_locations WHERE network_start_ip <= ips.net ORDER BY network_start_ip DESC LIMIT 1) geom FROM ips) - SELECT geom INTO ret - FROM matches; - RETURN ret; -END -$$ LANGUAGE plpgsql; diff --git a/server/extension/sql/0.5.0/70_ips.sql b/server/extension/sql/0.5.0/70_ips.sql new file mode 120000 index 0000000..f96f249 --- /dev/null +++ b/server/extension/sql/0.5.0/70_ips.sql @@ -0,0 +1 @@ +../0.4.0/70_ips.sql \ No newline at end of file diff --git a/server/extension/sql/0.5.0/80_isolines_helper.sql b/server/extension/sql/0.5.0/80_isolines_helper.sql new file mode 120000 index 0000000..b665362 --- /dev/null +++ b/server/extension/sql/0.5.0/80_isolines_helper.sql @@ -0,0 +1 @@ +../0.4.0/80_isolines_helper.sql \ No newline at end of file diff --git a/server/extension/sql/0.5.0/80_routing_helper.sql b/server/extension/sql/0.5.0/80_routing_helper.sql deleted file mode 100644 index 634c2d7..0000000 --- a/server/extension/sql/0.5.0/80_routing_helper.sql +++ /dev/null @@ -1,51 +0,0 @@ -CREATE TYPE cdb_dataservices_server.isoline AS (center geometry(Geometry,4326), data_range integer, the_geom geometry(Multipolygon,4326)); - -CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_routing_isolines(username TEXT, orgname TEXT, type TEXT, source geometry(Geometry, 4326), mode TEXT, data_range integer[], options text[]) -RETURNS SETOF cdb_dataservices_server.isoline AS $$ - import json - from cartodb_services.here import HereMapsRoutingIsoline - from cartodb_services.metrics import QuotaService - from cartodb_services.here.types import geo_polyline_to_multipolygon - - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - user_routing_config = GD["user_routing_config_{0}".format(username)] - - quota_service = QuotaService(user_routing_config, redis_conn) - - try: - client = HereMapsRoutingIsoline(user_routing_config.heremaps_app_id, user_routing_config.heremaps_app_code, base_url = HereMapsRoutingIsoline.PRODUCTION_ROUTING_BASE_URL) - - if source: - lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] - lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] - source_str = 'geo!%f,%f' % (lat, lon) - else: - source_str = None - - if type == 'isodistance': - resp = client.calculate_isodistance(source_str, mode, data_range, options) - elif type == 'isochrone': - resp = client.calculate_isochrone(source_str, mode, data_range, options) - - if resp: - result = [] - for isoline in resp: - data_range_n = isoline['range'] - polyline = isoline['geom'] - multipolygon = geo_polyline_to_multipolygon(polyline) - result.append([source, data_range_n, multipolygon]) - quota_service.increment_success_geocoder_use() - quota_service.increment_isolines_service_use(len(resp)) - return result - else: - quota_service.increment_empty_geocoder_use() - except BaseException as e: - import sys, traceback - type_, value_, traceback_ = sys.exc_info() - quota_service.increment_failed_geocoder_use() - error_msg = 'There was an error trying to obtain isodistances using here maps geocoder: {0}'.format(e) - plpy.notice(traceback.format_tb(traceback_)) - plpy.error(error_msg) - finally: - quota_service.increment_total_geocoder_use() -$$ LANGUAGE plpythonu SECURITY DEFINER; diff --git a/server/extension/sql/0.5.0/85_isodistance.sql b/server/extension/sql/0.5.0/85_isodistance.sql deleted file mode 100644 index b5b553c..0000000 --- a/server/extension/sql/0.5.0/85_isodistance.sql +++ /dev/null @@ -1,18 +0,0 @@ -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) -RETURNS SETOF cdb_dataservices_server.isoline AS $$ - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_isolines_config = GD["user_routing_config_{0}".format(username)] - type = 'isodistance' - - here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) - result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) - isolines = [] - for element in result: - isoline = element['isoline'] - isoline = isoline.translate(None, "()").split(',') - isolines.append(isoline) - - return isolines -$$ LANGUAGE plpythonu; diff --git a/server/extension/sql/0.5.0/85_isodistance.sql b/server/extension/sql/0.5.0/85_isodistance.sql new file mode 120000 index 0000000..3b2e4dc --- /dev/null +++ b/server/extension/sql/0.5.0/85_isodistance.sql @@ -0,0 +1 @@ +../0.4.0/85_isodistance.sql \ No newline at end of file diff --git a/server/extension/sql/0.5.0/90_isochrone.sql b/server/extension/sql/0.5.0/90_isochrone.sql deleted file mode 100644 index 31f9d3a..0000000 --- a/server/extension/sql/0.5.0/90_isochrone.sql +++ /dev/null @@ -1,18 +0,0 @@ -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) -RETURNS SETOF cdb_dataservices_server.isoline AS $$ - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_isolines_config = GD["user_routing_config_{0}".format(username)] - type = 'isochrone' - - here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) - result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) - isolines = [] - for element in result: - isoline = element['isoline'] - isoline = isoline.translate(None, "()").split(',') - isolines.append(isoline) - - return isolines -$$ LANGUAGE plpythonu; diff --git a/server/extension/sql/0.5.0/90_isochrone.sql b/server/extension/sql/0.5.0/90_isochrone.sql new file mode 120000 index 0000000..59276d2 --- /dev/null +++ b/server/extension/sql/0.5.0/90_isochrone.sql @@ -0,0 +1 @@ +../0.4.0/90_isochrone.sql \ No newline at end of file diff --git a/server/extension/sql/0.5.0/99_geocoder_server_user.sql b/server/extension/sql/0.5.0/999_geocoder_server_user.sql similarity index 100% rename from server/extension/sql/0.5.0/99_geocoder_server_user.sql rename to server/extension/sql/0.5.0/999_geocoder_server_user.sql diff --git a/server/extension/test/0.5.0/expected/00_install_test.out b/server/extension/test/0.5.0/expected/00_install_test.out index d55743f..2da3255 100644 --- a/server/extension/test/0.5.0/expected/00_install_test.out +++ b/server/extension/test/0.5.0/expected/00_install_test.out @@ -19,6 +19,12 @@ SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localh (1 row) +SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"app_key": "dummy_key"}'); + cdb_conf_setconf +------------------ + +(1 row) + -- Mock the varnish invalidation function -- (used by cdb_geocoder tests) CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$ diff --git a/server/extension/test/0.5.0/expected/20_street_test.out b/server/extension/test/0.5.0/expected/20_street_test.out deleted file mode 100644 index 8fbce7a..0000000 --- a/server/extension/test/0.5.0/expected/20_street_test.out +++ /dev/null @@ -1,12 +0,0 @@ --- Check for namedplaces signatures -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = 'cdb_geocode_street_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, text'); - exists --------- - t -(1 row) - diff --git a/server/extension/test/0.5.0/expected/20_street_test.out b/server/extension/test/0.5.0/expected/20_street_test.out new file mode 120000 index 0000000..6081021 --- /dev/null +++ b/server/extension/test/0.5.0/expected/20_street_test.out @@ -0,0 +1 @@ +../../0.4.0/expected/20_street_test.out \ No newline at end of file diff --git a/server/extension/test/0.5.0/expected/30_admin0_test.out b/server/extension/test/0.5.0/expected/30_admin0_test.out deleted file mode 100644 index 55c7639..0000000 --- a/server/extension/test/0.5.0/expected/30_admin0_test.out +++ /dev/null @@ -1,47 +0,0 @@ --- Check that the public function is callable, even with no data --- It should return NULL -SELECT cdb_dataservices_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'Spain'); - cdb_geocode_admin0_polygon ----------------------------- - -(1 row) - --- Insert some dummy synonym -INSERT INTO admin0_synonyms (name, adm0_a3) VALUES ('Spain', 'ESP'); --- Insert some dummy geometry to return -INSERT INTO ne_admin0_v3 (adm0_a3, the_geom) VALUES('ESP', ST_GeomFromText( - 'POLYGON((-71.1031880899493 42.3152774590236, - -71.1031627617667 42.3152960829043, - -71.102923838298 42.3149156848307, - -71.1031880899493 42.3152774590236))',4326) -); --- This should return the polygon inserted above -SELECT cdb_dataservices_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'Spain'); - cdb_geocode_admin0_polygon --------------------------------------------------------------------------------------------------------------------------------------------------------------------- - 0103000020E61000000100000004000000D0EA37A29AC651C00FD603035B284540FEFCFB379AC651C0C0503E9F5B284540FFDDDD4D96C651C033AC3B284F284540D0EA37A29AC651C00FD603035B284540 -(1 row) - --- Check for admin0 signatures -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = 'cdb_geocode_admin0_polygon' - AND oidvectortypes(p.proargtypes) = 'text, text, text'); - exists --------- - t -(1 row) - -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = '_cdb_geocode_admin0_polygon' - AND oidvectortypes(p.proargtypes) = 'text'); - exists --------- - t -(1 row) - diff --git a/server/extension/test/0.5.0/expected/30_admin0_test.out b/server/extension/test/0.5.0/expected/30_admin0_test.out new file mode 120000 index 0000000..9cfed39 --- /dev/null +++ b/server/extension/test/0.5.0/expected/30_admin0_test.out @@ -0,0 +1 @@ +../../0.4.0/expected/30_admin0_test.out \ No newline at end of file diff --git a/server/extension/test/0.5.0/expected/40_admin1_test.out b/server/extension/test/0.5.0/expected/40_admin1_test.out deleted file mode 100644 index 4894b0c..0000000 --- a/server/extension/test/0.5.0/expected/40_admin1_test.out +++ /dev/null @@ -1,81 +0,0 @@ --- Check that the public function is callable, even with no data --- It should return NULL -SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California'); - cdb_geocode_admin1_polygon ----------------------------- - -(1 row) - -SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California', 'United States'); - cdb_geocode_admin1_polygon ----------------------------- - -(1 row) - --- Insert dummy data into country decoder table -INSERT INTO country_decoder (synonyms, iso3) VALUES (Array['united states'], 'USA'); --- Insert some dummy data and geometry to return -INSERT INTO global_province_polygons (synonyms, iso3, the_geom) VALUES (Array['california'], 'USA', ST_GeomFromText( - 'POLYGON((-71.1031880899493 42.3152774590236, - -71.1031627617667 42.3152960829043, - -71.102923838298 42.3149156848307, - -71.1031880899493 42.3152774590236))',4326) -); --- This should return the polygon inserted above -SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California'); - cdb_geocode_admin1_polygon --------------------------------------------------------------------------------------------------------------------------------------------------------------------- - 0103000020E61000000100000004000000D0EA37A29AC651C00FD603035B284540FEFCFB379AC651C0C0503E9F5B284540FFDDDD4D96C651C033AC3B284F284540D0EA37A29AC651C00FD603035B284540 -(1 row) - -SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California', 'United States'); - cdb_geocode_admin1_polygon --------------------------------------------------------------------------------------------------------------------------------------------------------------------- - 0103000020E61000000100000004000000D0EA37A29AC651C00FD603035B284540FEFCFB379AC651C0C0503E9F5B284540FFDDDD4D96C651C033AC3B284F284540D0EA37A29AC651C00FD603035B284540 -(1 row) - --- Check for admin1 signatures -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = 'cdb_geocode_admin1_polygon' - AND oidvectortypes(p.proargtypes) = 'text, text, text'); - exists --------- - t -(1 row) - -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = 'cdb_geocode_admin1_polygon' - AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); - exists --------- - t -(1 row) - -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = '_cdb_geocode_admin1_polygon' - AND oidvectortypes(p.proargtypes) = 'text'); - exists --------- - t -(1 row) - -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = '_cdb_geocode_admin1_polygon' - AND oidvectortypes(p.proargtypes) = 'text, text'); - exists --------- - t -(1 row) - diff --git a/server/extension/test/0.5.0/expected/40_admin1_test.out b/server/extension/test/0.5.0/expected/40_admin1_test.out new file mode 120000 index 0000000..32b2ff2 --- /dev/null +++ b/server/extension/test/0.5.0/expected/40_admin1_test.out @@ -0,0 +1 @@ +../../0.4.0/expected/40_admin1_test.out \ No newline at end of file diff --git a/server/extension/test/0.5.0/expected/50_namedplaces_test.out b/server/extension/test/0.5.0/expected/50_namedplaces_test.out deleted file mode 100644 index 7755bf4..0000000 --- a/server/extension/test/0.5.0/expected/50_namedplaces_test.out +++ /dev/null @@ -1,136 +0,0 @@ --- Check that the public function is callable, even with no data --- It should return NULL -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx'); - cdb_geocode_namedplace_point ------------------------------- - -(1 row) - -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain'); - cdb_geocode_namedplace_point ------------------------------- - -(1 row) - -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain'); - cdb_geocode_namedplace_point ------------------------------- - -(1 row) - --- Insert dummy data into points table -INSERT INTO global_cities_points_limited (geoname_id, name, iso2, admin1, admin2, population, lowername, the_geom) VALUES (3128760, 'Elche', 'ES', 'Valencia', 'AL', 34534, 'elche', ST_GeomFromText( - 'POINT(0.6983 39.26787)',4326) -); --- Insert dummy data into alternates table -INSERT INTO global_cities_alternates_limited (geoname_id, name, preferred, lowername, admin1_geonameid, iso2, admin1, the_geom) VALUES (3128760, 'Elx', true, 'elx', '000000', 'ES', 'Valencia', ST_GeomFromText( - 'POINT(0.6983 39.26787)',4326) -); --- Insert dummy data into country decoder table -INSERT INTO country_decoder (synonyms, iso2) VALUES (Array['spain'], 'ES'); --- Insert dummy data into admin1 decoder table -INSERT INTO admin1_decoder (admin1, synonyms, iso2) VALUES ('Valencia', Array['valencia', 'Valencia'], 'ES'); --- This should return the point inserted above -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx'); - cdb_geocode_namedplace_point ----------------------------------------------------- - 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 -(1 row) - -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche'); - cdb_geocode_namedplace_point ----------------------------------------------------- - 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 -(1 row) - -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain'); - cdb_geocode_namedplace_point ----------------------------------------------------- - 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 -(1 row) - -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'Spain'); - cdb_geocode_namedplace_point ----------------------------------------------------- - 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 -(1 row) - -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain'); - cdb_geocode_namedplace_point ----------------------------------------------------- - 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 -(1 row) - -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'valencia', 'Spain'); - cdb_geocode_namedplace_point ----------------------------------------------------- - 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 -(1 row) - --- Check for namedplaces signatures -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = 'cdb_geocode_namedplace_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text'); - exists --------- - t -(1 row) - -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = 'cdb_geocode_namedplace_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); - exists --------- - t -(1 row) - -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = 'cdb_geocode_namedplace_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); - exists --------- - t -(1 row) - -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = '_cdb_geocode_namedplace_point' - AND oidvectortypes(p.proargtypes) = 'text'); - exists --------- - t -(1 row) - -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = '_cdb_geocode_namedplace_point' - AND oidvectortypes(p.proargtypes) = 'text, text'); - exists --------- - t -(1 row) - -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = '_cdb_geocode_namedplace_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text'); - exists --------- - t -(1 row) - diff --git a/server/extension/test/0.5.0/expected/50_namedplaces_test.out b/server/extension/test/0.5.0/expected/50_namedplaces_test.out new file mode 120000 index 0000000..ebc8d68 --- /dev/null +++ b/server/extension/test/0.5.0/expected/50_namedplaces_test.out @@ -0,0 +1 @@ +../../0.4.0/expected/50_namedplaces_test.out \ No newline at end of file diff --git a/server/extension/test/0.5.0/expected/60_postalcodes_test.out b/server/extension/test/0.5.0/expected/60_postalcodes_test.out deleted file mode 100644 index b2a5c91..0000000 --- a/server/extension/test/0.5.0/expected/60_postalcodes_test.out +++ /dev/null @@ -1,163 +0,0 @@ --- Make sure dbs are clean -DELETE FROM global_postal_code_points; -DELETE FROM global_postal_code_polygons; -DELETE FROM country_decoder; -DELETE FROM available_services; -DELETE FROM admin0_synonyms; --- Check that the public function is callable, even with no data --- It should return NULL -SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204'); - cdb_geocode_postalcode_point ------------------------------- - -(1 row) - --- Insert dummy data into ip_address_locations -INSERT INTO global_postal_code_points (the_geom, iso3, postal_code, postal_code_num) VALUES ( - '0101000020E61000000000000000E040408036B47414764840', - 'ESP', - '03204', - 3204 -); -INSERT INTO global_postal_code_polygons (the_geom, iso3, postal_code, postal_code_num) VALUES ( - '0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040', - 'ESP', - '03204', - 3204 -); -INSERT INTO country_decoder (iso3, synonyms) VALUES ( - 'ESP', - Array['spain', 'Spain', 'ESP'] -); -INSERT INTO available_services (adm0_a3, admin0, postal_code_points, postal_code_polygons) VALUES ( - 'ESP', - 't', - 't', - 't' -); -INSERT INTO admin0_synonyms (adm0_a3, name, name_, rank) VALUES ( - 'ESP', - 'Spain', - 'spain', - 3 -); --- This should return the polygon inserted above -SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204'); - cdb_geocode_postalcode_point ----------------------------------------------------- - 0101000020E61000000000000000E040408036B47414764840 -(1 row) - -SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204', 'spain'); - cdb_geocode_postalcode_point ----------------------------------------------------- - 0101000020E61000000000000000E040408036B47414764840 -(1 row) - -SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', '03204'); - cdb_geocode_postalcode_polygon --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - 0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040 -(1 row) - -SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', '03204', 'spain'); - cdb_geocode_postalcode_polygon --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - 0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040 -(1 row) - --- Clean dbs -DELETE FROM global_postal_code_points; -DELETE FROM global_postal_code_polygons; -DELETE FROM country_decoder; -DELETE FROM available_services; -DELETE FROM admin0_synonyms; --- Check for namedplaces signatures (point and polygon) -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = 'cdb_geocode_postalcode_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text'); - exists --------- - t -(1 row) - -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = 'cdb_geocode_postalcode_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); - exists --------- - t -(1 row) - -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = 'cdb_geocode_postalcode_polygon' - AND oidvectortypes(p.proargtypes) = 'text, text, text'); - exists --------- - t -(1 row) - -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = 'cdb_geocode_postalcode_polygon' - AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); - exists --------- - t -(1 row) - -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = '_cdb_geocode_postalcode_point' - AND oidvectortypes(p.proargtypes) = 'text'); - exists --------- - t -(1 row) - -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = '_cdb_geocode_postalcode_point' - AND oidvectortypes(p.proargtypes) = 'text, text'); - exists --------- - t -(1 row) - -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = '_cdb_geocode_postalcode_polygon' - AND oidvectortypes(p.proargtypes) = 'text'); - exists --------- - t -(1 row) - -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = '_cdb_geocode_postalcode_polygon' - AND oidvectortypes(p.proargtypes) = 'text, text'); - exists --------- - t -(1 row) - diff --git a/server/extension/test/0.5.0/expected/60_postalcodes_test.out b/server/extension/test/0.5.0/expected/60_postalcodes_test.out new file mode 120000 index 0000000..147ebf6 --- /dev/null +++ b/server/extension/test/0.5.0/expected/60_postalcodes_test.out @@ -0,0 +1 @@ +../../0.4.0/expected/60_postalcodes_test.out \ No newline at end of file diff --git a/server/extension/test/0.5.0/expected/70_ips_test.out b/server/extension/test/0.5.0/expected/70_ips_test.out deleted file mode 100644 index 48012f7..0000000 --- a/server/extension/test/0.5.0/expected/70_ips_test.out +++ /dev/null @@ -1,40 +0,0 @@ --- Check that the public function is callable, even with no data --- It should return NULL -SELECT cdb_dataservices_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', '0.0.0.0'); - cdb_geocode_ipaddress_point ------------------------------ - -(1 row) - --- Insert dummy data into ip_address_locations -INSERT INTO ip_address_locations VALUES ('::ffff:0.0.0.0'::inet, (ST_SetSRID(ST_MakePoint('40.40', '3.71'), 4326))); --- This should return the polygon inserted above -SELECT cdb_dataservices_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', '0.0.0.0'); - cdb_geocode_ipaddress_point ----------------------------------------------------- - 0101000020E61000003333333333334440AE47E17A14AE0D40 -(1 row) - --- Check for namedplaces signatures (point and polygon) -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = 'cdb_geocode_ipaddress_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text'); - exists --------- - t -(1 row) - -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = '_cdb_geocode_ipaddress_point' - AND oidvectortypes(p.proargtypes) = 'text'); - exists --------- - t -(1 row) - diff --git a/server/extension/test/0.5.0/expected/70_ips_test.out b/server/extension/test/0.5.0/expected/70_ips_test.out new file mode 120000 index 0000000..9b61caa --- /dev/null +++ b/server/extension/test/0.5.0/expected/70_ips_test.out @@ -0,0 +1 @@ +../../0.4.0/expected/70_ips_test.out \ No newline at end of file diff --git a/server/extension/test/0.5.0/expected/85_isodistance_test.out b/server/extension/test/0.5.0/expected/85_isodistance_test.out deleted file mode 100644 index 5705284..0000000 --- a/server/extension/test/0.5.0/expected/85_isodistance_test.out +++ /dev/null @@ -1,12 +0,0 @@ --- Check for isodistance signatures -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = 'cdb_isodistance' - AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, integer[], text[]'); - exists --------- - t -(1 row) - diff --git a/server/extension/test/0.5.0/expected/85_isodistance_test.out b/server/extension/test/0.5.0/expected/85_isodistance_test.out new file mode 120000 index 0000000..3c81fbf --- /dev/null +++ b/server/extension/test/0.5.0/expected/85_isodistance_test.out @@ -0,0 +1 @@ +../../0.4.0/expected/85_isodistance_test.out \ No newline at end of file diff --git a/server/extension/test/0.5.0/expected/90_isochrone_test.out b/server/extension/test/0.5.0/expected/90_isochrone_test.out deleted file mode 100644 index 7ee0f04..0000000 --- a/server/extension/test/0.5.0/expected/90_isochrone_test.out +++ /dev/null @@ -1,12 +0,0 @@ --- Check for isochrone signatures -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = 'cdb_isochrone' - AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, integer[], text[]'); - exists --------- - t -(1 row) - diff --git a/server/extension/test/0.5.0/expected/90_isochrone_test.out b/server/extension/test/0.5.0/expected/90_isochrone_test.out new file mode 120000 index 0000000..596e8ab --- /dev/null +++ b/server/extension/test/0.5.0/expected/90_isochrone_test.out @@ -0,0 +1 @@ +../../0.4.0/expected/90_isochrone_test.out \ No newline at end of file diff --git a/server/extension/test/0.5.0/expected/95_route_point_to_point_test.out b/server/extension/test/0.5.0/expected/95_route_point_to_point_test.out new file mode 100644 index 0000000..1a1acb7 --- /dev/null +++ b/server/extension/test/0.5.0/expected/95_route_point_to_point_test.out @@ -0,0 +1,12 @@ +-- Check for routing point to point signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_route_point_to_point' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, geometry, text, text[], text'); + exists +-------- + t +(1 row) + diff --git a/server/extension/test/0.5.0/expected/99_remove_geocoder_api_user_test.out b/server/extension/test/0.5.0/expected/999_remove_geocoder_api_user_test.out similarity index 100% rename from server/extension/test/0.5.0/expected/99_remove_geocoder_api_user_test.out rename to server/extension/test/0.5.0/expected/999_remove_geocoder_api_user_test.out diff --git a/server/extension/test/0.5.0/sql/00_install_test.sql b/server/extension/test/0.5.0/sql/00_install_test.sql index ad69ffb..09503cc 100644 --- a/server/extension/test/0.5.0/sql/00_install_test.sql +++ b/server/extension/test/0.5.0/sql/00_install_test.sql @@ -11,6 +11,7 @@ CREATE EXTENSION cdb_dataservices_server; -- Mock the redis server connection to point to this very test db SELECT cartodb.cdb_conf_setconf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localhost", "redis_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); +SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"app_key": "dummy_key"}'); -- Mock the varnish invalidation function -- (used by cdb_geocoder tests) diff --git a/server/extension/test/0.5.0/sql/20_street_test.sql b/server/extension/test/0.5.0/sql/20_street_test.sql deleted file mode 100644 index 86e0368..0000000 --- a/server/extension/test/0.5.0/sql/20_street_test.sql +++ /dev/null @@ -1,7 +0,0 @@ --- Check for namedplaces signatures -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = 'cdb_geocode_street_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, text'); diff --git a/server/extension/test/0.5.0/sql/20_street_test.sql b/server/extension/test/0.5.0/sql/20_street_test.sql new file mode 120000 index 0000000..ca39762 --- /dev/null +++ b/server/extension/test/0.5.0/sql/20_street_test.sql @@ -0,0 +1 @@ +../../0.4.0/sql/20_street_test.sql \ No newline at end of file diff --git a/server/extension/test/0.5.0/sql/30_admin0_test.sql b/server/extension/test/0.5.0/sql/30_admin0_test.sql deleted file mode 100644 index 3249c60..0000000 --- a/server/extension/test/0.5.0/sql/30_admin0_test.sql +++ /dev/null @@ -1,32 +0,0 @@ --- Check that the public function is callable, even with no data --- It should return NULL -SELECT cdb_dataservices_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'Spain'); - --- Insert some dummy synonym -INSERT INTO admin0_synonyms (name, adm0_a3) VALUES ('Spain', 'ESP'); - --- Insert some dummy geometry to return -INSERT INTO ne_admin0_v3 (adm0_a3, the_geom) VALUES('ESP', ST_GeomFromText( - 'POLYGON((-71.1031880899493 42.3152774590236, - -71.1031627617667 42.3152960829043, - -71.102923838298 42.3149156848307, - -71.1031880899493 42.3152774590236))',4326) -); - --- This should return the polygon inserted above -SELECT cdb_dataservices_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'Spain'); - --- Check for admin0 signatures -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = 'cdb_geocode_admin0_polygon' - AND oidvectortypes(p.proargtypes) = 'text, text, text'); - -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = '_cdb_geocode_admin0_polygon' - AND oidvectortypes(p.proargtypes) = 'text'); diff --git a/server/extension/test/0.5.0/sql/30_admin0_test.sql b/server/extension/test/0.5.0/sql/30_admin0_test.sql new file mode 120000 index 0000000..f25c0a1 --- /dev/null +++ b/server/extension/test/0.5.0/sql/30_admin0_test.sql @@ -0,0 +1 @@ +../../0.4.0/sql/30_admin0_test.sql \ No newline at end of file diff --git a/server/extension/test/0.5.0/sql/40_admin1_test.sql b/server/extension/test/0.5.0/sql/40_admin1_test.sql deleted file mode 100644 index 7b3748c..0000000 --- a/server/extension/test/0.5.0/sql/40_admin1_test.sql +++ /dev/null @@ -1,48 +0,0 @@ --- Check that the public function is callable, even with no data --- It should return NULL -SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California'); -SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California', 'United States'); - --- Insert dummy data into country decoder table -INSERT INTO country_decoder (synonyms, iso3) VALUES (Array['united states'], 'USA'); - --- Insert some dummy data and geometry to return -INSERT INTO global_province_polygons (synonyms, iso3, the_geom) VALUES (Array['california'], 'USA', ST_GeomFromText( - 'POLYGON((-71.1031880899493 42.3152774590236, - -71.1031627617667 42.3152960829043, - -71.102923838298 42.3149156848307, - -71.1031880899493 42.3152774590236))',4326) -); - --- This should return the polygon inserted above -SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California'); -SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California', 'United States'); - --- Check for admin1 signatures -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = 'cdb_geocode_admin1_polygon' - AND oidvectortypes(p.proargtypes) = 'text, text, text'); - -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = 'cdb_geocode_admin1_polygon' - AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); - -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = '_cdb_geocode_admin1_polygon' - AND oidvectortypes(p.proargtypes) = 'text'); - -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = '_cdb_geocode_admin1_polygon' - AND oidvectortypes(p.proargtypes) = 'text, text'); diff --git a/server/extension/test/0.5.0/sql/40_admin1_test.sql b/server/extension/test/0.5.0/sql/40_admin1_test.sql new file mode 120000 index 0000000..ada2acf --- /dev/null +++ b/server/extension/test/0.5.0/sql/40_admin1_test.sql @@ -0,0 +1 @@ +../../0.4.0/sql/40_admin1_test.sql \ No newline at end of file diff --git a/server/extension/test/0.5.0/sql/50_namedplaces_test.sql b/server/extension/test/0.5.0/sql/50_namedplaces_test.sql deleted file mode 100644 index 306a682..0000000 --- a/server/extension/test/0.5.0/sql/50_namedplaces_test.sql +++ /dev/null @@ -1,72 +0,0 @@ --- Check that the public function is callable, even with no data --- It should return NULL -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx'); -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain'); -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain'); - --- Insert dummy data into points table -INSERT INTO global_cities_points_limited (geoname_id, name, iso2, admin1, admin2, population, lowername, the_geom) VALUES (3128760, 'Elche', 'ES', 'Valencia', 'AL', 34534, 'elche', ST_GeomFromText( - 'POINT(0.6983 39.26787)',4326) -); - --- Insert dummy data into alternates table -INSERT INTO global_cities_alternates_limited (geoname_id, name, preferred, lowername, admin1_geonameid, iso2, admin1, the_geom) VALUES (3128760, 'Elx', true, 'elx', '000000', 'ES', 'Valencia', ST_GeomFromText( - 'POINT(0.6983 39.26787)',4326) -); - --- Insert dummy data into country decoder table -INSERT INTO country_decoder (synonyms, iso2) VALUES (Array['spain'], 'ES'); - --- Insert dummy data into admin1 decoder table -INSERT INTO admin1_decoder (admin1, synonyms, iso2) VALUES ('Valencia', Array['valencia', 'Valencia'], 'ES'); - --- This should return the point inserted above -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx'); -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche'); -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain'); -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'Spain'); -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain'); -SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'valencia', 'Spain'); - --- Check for namedplaces signatures -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = 'cdb_geocode_namedplace_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text'); - -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = 'cdb_geocode_namedplace_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); - -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = 'cdb_geocode_namedplace_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); - -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = '_cdb_geocode_namedplace_point' - AND oidvectortypes(p.proargtypes) = 'text'); - -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = '_cdb_geocode_namedplace_point' - AND oidvectortypes(p.proargtypes) = 'text, text'); - -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = '_cdb_geocode_namedplace_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text'); diff --git a/server/extension/test/0.5.0/sql/50_namedplaces_test.sql b/server/extension/test/0.5.0/sql/50_namedplaces_test.sql new file mode 120000 index 0000000..deb2105 --- /dev/null +++ b/server/extension/test/0.5.0/sql/50_namedplaces_test.sql @@ -0,0 +1 @@ +../../0.4.0/sql/50_namedplaces_test.sql \ No newline at end of file diff --git a/server/extension/test/0.5.0/sql/60_postalcodes_test.sql b/server/extension/test/0.5.0/sql/60_postalcodes_test.sql deleted file mode 100644 index 55d1c85..0000000 --- a/server/extension/test/0.5.0/sql/60_postalcodes_test.sql +++ /dev/null @@ -1,117 +0,0 @@ --- Make sure dbs are clean -DELETE FROM global_postal_code_points; -DELETE FROM global_postal_code_polygons; -DELETE FROM country_decoder; -DELETE FROM available_services; -DELETE FROM admin0_synonyms; - --- Check that the public function is callable, even with no data --- It should return NULL -SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204'); - --- Insert dummy data into ip_address_locations -INSERT INTO global_postal_code_points (the_geom, iso3, postal_code, postal_code_num) VALUES ( - '0101000020E61000000000000000E040408036B47414764840', - 'ESP', - '03204', - 3204 -); - -INSERT INTO global_postal_code_polygons (the_geom, iso3, postal_code, postal_code_num) VALUES ( - '0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040', - 'ESP', - '03204', - 3204 -); - -INSERT INTO country_decoder (iso3, synonyms) VALUES ( - 'ESP', - Array['spain', 'Spain', 'ESP'] -); - -INSERT INTO available_services (adm0_a3, admin0, postal_code_points, postal_code_polygons) VALUES ( - 'ESP', - 't', - 't', - 't' -); - -INSERT INTO admin0_synonyms (adm0_a3, name, name_, rank) VALUES ( - 'ESP', - 'Spain', - 'spain', - 3 -); - --- This should return the polygon inserted above -SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204'); - -SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204', 'spain'); - -SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', '03204'); - -SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', '03204', 'spain'); - --- Clean dbs -DELETE FROM global_postal_code_points; -DELETE FROM global_postal_code_polygons; -DELETE FROM country_decoder; -DELETE FROM available_services; -DELETE FROM admin0_synonyms; - --- Check for namedplaces signatures (point and polygon) -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = 'cdb_geocode_postalcode_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text'); - -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = 'cdb_geocode_postalcode_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); - -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = 'cdb_geocode_postalcode_polygon' - AND oidvectortypes(p.proargtypes) = 'text, text, text'); - -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = 'cdb_geocode_postalcode_polygon' - AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); - -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = '_cdb_geocode_postalcode_point' - AND oidvectortypes(p.proargtypes) = 'text'); - -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = '_cdb_geocode_postalcode_point' - AND oidvectortypes(p.proargtypes) = 'text, text'); - -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = '_cdb_geocode_postalcode_polygon' - AND oidvectortypes(p.proargtypes) = 'text'); - -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = '_cdb_geocode_postalcode_polygon' - AND oidvectortypes(p.proargtypes) = 'text, text'); diff --git a/server/extension/test/0.5.0/sql/60_postalcodes_test.sql b/server/extension/test/0.5.0/sql/60_postalcodes_test.sql new file mode 120000 index 0000000..19eff09 --- /dev/null +++ b/server/extension/test/0.5.0/sql/60_postalcodes_test.sql @@ -0,0 +1 @@ +../../0.4.0/sql/60_postalcodes_test.sql \ No newline at end of file diff --git a/server/extension/test/0.5.0/sql/70_ips_test.sql b/server/extension/test/0.5.0/sql/70_ips_test.sql deleted file mode 100644 index 6111bd0..0000000 --- a/server/extension/test/0.5.0/sql/70_ips_test.sql +++ /dev/null @@ -1,24 +0,0 @@ --- Check that the public function is callable, even with no data --- It should return NULL -SELECT cdb_dataservices_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', '0.0.0.0'); - --- Insert dummy data into ip_address_locations -INSERT INTO ip_address_locations VALUES ('::ffff:0.0.0.0'::inet, (ST_SetSRID(ST_MakePoint('40.40', '3.71'), 4326))); - --- This should return the polygon inserted above -SELECT cdb_dataservices_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', '0.0.0.0'); - --- Check for namedplaces signatures (point and polygon) -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = 'cdb_geocode_ipaddress_point' - AND oidvectortypes(p.proargtypes) = 'text, text, text'); - -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = '_cdb_geocode_ipaddress_point' - AND oidvectortypes(p.proargtypes) = 'text'); diff --git a/server/extension/test/0.5.0/sql/70_ips_test.sql b/server/extension/test/0.5.0/sql/70_ips_test.sql new file mode 120000 index 0000000..7af108a --- /dev/null +++ b/server/extension/test/0.5.0/sql/70_ips_test.sql @@ -0,0 +1 @@ +../../0.4.0/sql/70_ips_test.sql \ No newline at end of file diff --git a/server/extension/test/0.5.0/sql/85_isodistance_test.sql b/server/extension/test/0.5.0/sql/85_isodistance_test.sql deleted file mode 100644 index f66b68a..0000000 --- a/server/extension/test/0.5.0/sql/85_isodistance_test.sql +++ /dev/null @@ -1,7 +0,0 @@ --- Check for isodistance signatures -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = 'cdb_isodistance' - AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, integer[], text[]'); diff --git a/server/extension/test/0.5.0/sql/85_isodistance_test.sql b/server/extension/test/0.5.0/sql/85_isodistance_test.sql new file mode 120000 index 0000000..d819b66 --- /dev/null +++ b/server/extension/test/0.5.0/sql/85_isodistance_test.sql @@ -0,0 +1 @@ +../../0.4.0/sql/85_isodistance_test.sql \ No newline at end of file diff --git a/server/extension/test/0.5.0/sql/90_isochrone_test.sql b/server/extension/test/0.5.0/sql/90_isochrone_test.sql deleted file mode 100644 index 113fb28..0000000 --- a/server/extension/test/0.5.0/sql/90_isochrone_test.sql +++ /dev/null @@ -1,7 +0,0 @@ --- Check for isochrone signatures -SELECT exists(SELECT * - FROM pg_proc p - INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) - WHERE ns.nspname = 'cdb_dataservices_server' - AND proname = 'cdb_isochrone' - AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, integer[], text[]'); diff --git a/server/extension/test/0.5.0/sql/90_isochrone_test.sql b/server/extension/test/0.5.0/sql/90_isochrone_test.sql new file mode 120000 index 0000000..33feb9b --- /dev/null +++ b/server/extension/test/0.5.0/sql/90_isochrone_test.sql @@ -0,0 +1 @@ +../../0.4.0/sql/90_isochrone_test.sql \ No newline at end of file diff --git a/server/extension/test/0.5.0/sql/95_route_point_to_point_test.sql b/server/extension/test/0.5.0/sql/95_route_point_to_point_test.sql new file mode 100644 index 0000000..409e0d2 --- /dev/null +++ b/server/extension/test/0.5.0/sql/95_route_point_to_point_test.sql @@ -0,0 +1,7 @@ +-- Check for routing point to point signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_route_point_to_point' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, geometry, text, text[], text'); diff --git a/server/extension/test/0.5.0/sql/99_remove_geocoder_api_user_test.sql b/server/extension/test/0.5.0/sql/999_remove_geocoder_api_user_test.sql similarity index 100% rename from server/extension/test/0.5.0/sql/99_remove_geocoder_api_user_test.sql rename to server/extension/test/0.5.0/sql/999_remove_geocoder_api_user_test.sql From df195a95399c9b522ed4d1ec512d33efb4652a36 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 24 Feb 2016 16:58:14 +0100 Subject: [PATCH 10/12] Client part of the mapzen integration --- client/Makefile | 5 +- .../cdb_dataservices_client--0.2.0--0.3.0.sql | 38 ++ .../cdb_dataservices_client--0.3.0--0.2.0.sql | 3 + client/cdb_dataservices_client--0.3.0.sql | 556 ++++++++++++++++++ client/cdb_dataservices_client.control | 2 +- .../renderer/interfaces/interface_0.3.0.yaml | 91 +++ client/renderer/sql-template-renderer | 4 + .../templates/20_public_functions.erb | 3 + .../templates/30_plproxy_functions.erb | 2 +- client/sql/0.3.0/00_header.sql | 1 + client/sql/0.3.0/10_geocoder_server_conn.sql | 1 + client/sql/0.3.0/15_config_management.sql | 1 + client/sql/0.3.0/16_custom_types.sql | 11 + client/sql/0.3.0/80_permissions.sql | 1 + .../0.3.0/expected/00_installation_test.out | 1 + client/test/0.3.0/expected/10_admin0_test.out | 1 + client/test/0.3.0/expected/20_admin1_test.out | 1 + .../0.3.0/expected/30_namedplaces_test.out | 1 + .../0.3.0/expected/40_postalcodes_test.out | 1 + .../0.3.0/expected/50_ipaddresses_test.out | 1 + client/test/0.3.0/expected/60_street_test.out | 1 + .../expected/80_route_point_to_point_test.out | 41 ++ .../0.3.0/expected/90_permissions_test.out | 128 ++++ .../test/0.3.0/sql/00_installation_test.sql | 1 + client/test/0.3.0/sql/10_admin0_test.sql | 1 + client/test/0.3.0/sql/20_admin1_test.sql | 1 + client/test/0.3.0/sql/30_namedplaces_test.sql | 1 + client/test/0.3.0/sql/40_postalcodes_test.sql | 1 + client/test/0.3.0/sql/50_ipaddresses_test.sql | 1 + client/test/0.3.0/sql/60_street_test.sql | 1 + .../sql/80_route_point_to_point_test.sql | 21 + client/test/0.3.0/sql/90_permissions_test.sql | 32 + 32 files changed, 952 insertions(+), 3 deletions(-) create mode 100644 client/cdb_dataservices_client--0.2.0--0.3.0.sql create mode 100644 client/cdb_dataservices_client--0.3.0--0.2.0.sql create mode 100644 client/cdb_dataservices_client--0.3.0.sql create mode 100644 client/renderer/interfaces/interface_0.3.0.yaml create mode 120000 client/sql/0.3.0/00_header.sql create mode 120000 client/sql/0.3.0/10_geocoder_server_conn.sql create mode 120000 client/sql/0.3.0/15_config_management.sql create mode 100644 client/sql/0.3.0/16_custom_types.sql create mode 120000 client/sql/0.3.0/80_permissions.sql create mode 120000 client/test/0.3.0/expected/00_installation_test.out create mode 120000 client/test/0.3.0/expected/10_admin0_test.out create mode 120000 client/test/0.3.0/expected/20_admin1_test.out create mode 120000 client/test/0.3.0/expected/30_namedplaces_test.out create mode 120000 client/test/0.3.0/expected/40_postalcodes_test.out create mode 120000 client/test/0.3.0/expected/50_ipaddresses_test.out create mode 120000 client/test/0.3.0/expected/60_street_test.out create mode 100644 client/test/0.3.0/expected/80_route_point_to_point_test.out create mode 100644 client/test/0.3.0/expected/90_permissions_test.out create mode 120000 client/test/0.3.0/sql/00_installation_test.sql create mode 120000 client/test/0.3.0/sql/10_admin0_test.sql create mode 120000 client/test/0.3.0/sql/20_admin1_test.sql create mode 120000 client/test/0.3.0/sql/30_namedplaces_test.sql create mode 120000 client/test/0.3.0/sql/40_postalcodes_test.sql create mode 120000 client/test/0.3.0/sql/50_ipaddresses_test.sql create mode 120000 client/test/0.3.0/sql/60_street_test.sql create mode 100644 client/test/0.3.0/sql/80_route_point_to_point_test.sql create mode 100644 client/test/0.3.0/sql/90_permissions_test.sql diff --git a/client/Makefile b/client/Makefile index 46572a5..a15da74 100644 --- a/client/Makefile +++ b/client/Makefile @@ -13,10 +13,13 @@ NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql DATA = $(NEW_EXTENSION_ARTIFACT) \ cdb_dataservices_client--0.0.1.sql \ cdb_dataservices_client--0.1.0.sql \ + cdb_dataservices_client--0.2.0.sql \ cdb_dataservices_client--0.1.0--0.0.1.sql \ cdb_dataservices_client--0.0.1--0.1.0.sql \ cdb_dataservices_client--0.2.0--0.1.0.sql \ - cdb_dataservices_client--0.1.0--0.2.0.sql + cdb_dataservices_client--0.1.0--0.2.0.sql \ + cdb_dataservices_client--0.2.0--0.3.0.sql \ + cdb_dataservices_client--0.3.0--0.2.0.sql REGRESS = $(notdir $(basename $(wildcard test/$(EXTVERSION)/sql/*test.sql))) diff --git a/client/cdb_dataservices_client--0.2.0--0.3.0.sql b/client/cdb_dataservices_client--0.2.0--0.3.0.sql new file mode 100644 index 0000000..5405508 --- /dev/null +++ b/client/cdb_dataservices_client--0.2.0--0.3.0.sql @@ -0,0 +1,38 @@ +CREATE TYPE cdb_dataservices_client.simple_route AS ( + shape geometry(LineString,4326), + length real, + duration integer +); + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_route_point_to_point (origin geometry(Point, 4326), destination geometry(Point, 4326), mode text, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ +DECLARE + ret cdb_dataservices_client.simple_route; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT * FROM cdb_dataservices_client._cdb_route_point_to_point(username, orgname, origin, destination, mode, options, units) INTO ret; + RETURN ret; + +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_route_point_to_point (username text, organization_name text, origin geometry(Point, 4326), destination geometry(Point, 4326), mode text, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_route_point_to_point (username, organization_name, origin, destination, mode, options, units); + +$$ LANGUAGE plproxy; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_route_point_to_point(origin geometry(Point, 4326), destination geometry(Point, 4326), mode text, options text[], units text) TO publicuser; diff --git a/client/cdb_dataservices_client--0.3.0--0.2.0.sql b/client/cdb_dataservices_client--0.3.0--0.2.0.sql new file mode 100644 index 0000000..36eb20e --- /dev/null +++ b/client/cdb_dataservices_client--0.3.0--0.2.0.sql @@ -0,0 +1,3 @@ +DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_route_point_to_point (geometry(Point, 4326), geometry(Point, 4326), text, text[], text); +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_route_point_to_point (text, text, geometry(Point, 4326), geometry(Point, 4326), text, text[], text); +DROP TYPE IF EXISTS cdb_dataservices_client.simple_route; \ No newline at end of file diff --git a/client/cdb_dataservices_client--0.3.0.sql b/client/cdb_dataservices_client--0.3.0.sql new file mode 100644 index 0000000..89f81bb --- /dev/null +++ b/client/cdb_dataservices_client--0.3.0.sql @@ -0,0 +1,556 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_dataservices_client" to load this file. \quit +-- +-- Geocoder server connection config +-- +-- The purpose of this function is provide to the PL/Proxy functions +-- the connection string needed to connect with the server + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._server_conn_str() +RETURNS text AS $$ +DECLARE + db_connection_str text; +BEGIN + SELECT cartodb.cdb_conf_getconf('geocoder_server_config')->'connection_str' INTO db_connection_str; + SELECT trim(both '"' FROM db_connection_str) INTO db_connection_str; + RETURN db_connection_str; +END; +$$ LANGUAGE 'plpgsql'; +CREATE TYPE cdb_dataservices_client._entity_config AS ( + username text, + organization_name text +); + +-- +-- Get entity config function +-- +-- The purpose of this function is to retrieve the username and organization name from +-- a) schema where he/her is the owner in case is an organization user +-- b) entity_name from the cdb_conf database in case is a non organization user +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_entity_config() +RETURNS record AS $$ +DECLARE + result cdb_dataservices_client._entity_config; + is_organization boolean; + username text; + organization_name text; +BEGIN + SELECT cartodb.cdb_conf_getconf('user_config')->'is_organization' INTO is_organization; + IF is_organization IS NULL THEN + RAISE EXCEPTION 'User must have user configuration in the config table'; + ELSIF is_organization = TRUE THEN + SELECT nspname + FROM pg_namespace s + LEFT JOIN pg_roles r ON s.nspowner = r.oid + WHERE r.rolname = session_user INTO username; + SELECT cartodb.cdb_conf_getconf('user_config')->>'entity_name' INTO organization_name; + ELSE + SELECT cartodb.cdb_conf_getconf('user_config')->>'entity_name' INTO username; + organization_name = NULL; + END IF; + result.username = username; + result.organization_name = organization_name; + RETURN result; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; +CREATE TYPE cdb_dataservices_client.isoline AS ( + center geometry(Geometry,4326), + data_range integer, + the_geom geometry(Multipolygon,4326) +); + +CREATE TYPE cdb_dataservices_client.simple_route AS ( + shape geometry(LineString,4326), + length real, + duration integer +);-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_admin0_polygon (country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_admin0_polygon(username, orgname, country_name) INTO ret; + RETURN ret; + +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_admin1_polygon (admin1_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name) INTO ret; + RETURN ret; + +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_admin1_polygon (admin1_name text, country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name, country_name) INTO ret; + RETURN ret; + +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point (city_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name) INTO ret; + RETURN ret; + +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point (city_name text, country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name, country_name) INTO ret; + RETURN ret; + +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point (city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name, admin1_name, country_name) INTO ret; + RETURN ret; + +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_polygon (postal_code text, country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name) INTO ret; + RETURN ret; + +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_point (postal_code text, country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name) INTO ret; + RETURN ret; + +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_ipaddress_point (ip_address text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_ipaddress_point(username, orgname, ip_address) INTO ret; + RETURN ret; + +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_street_point (searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; + RETURN ret; + +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_isodistance (source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY + SELECT * FROM cdb_dataservices_client._cdb_isodistance(username, orgname, source, mode, range, options); + +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_isochrone (source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY + SELECT * FROM cdb_dataservices_client._cdb_isochrone(username, orgname, source, mode, range, options); + +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_route_point_to_point (origin geometry(Point, 4326), destination geometry(Point, 4326), mode text, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ +DECLARE + ret cdb_dataservices_client.simple_route; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT * FROM cdb_dataservices_client._cdb_route_point_to_point(username, orgname, origin, destination, mode, options, units) INTO ret; + RETURN ret; + +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin0_polygon (username text, organization_name text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_admin0_polygon (username, organization_name, country_name); + +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin1_polygon (username text, organization_name text, admin1_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon (username, organization_name, admin1_name); + +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin1_polygon (username text, organization_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon (username, organization_name, admin1_name, country_name); + +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point (username text, organization_name text, city_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_namedplace_point (username, organization_name, city_name); + +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point (username text, organization_name text, city_name text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_namedplace_point (username, organization_name, city_name, country_name); + +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point (username text, organization_name text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_namedplace_point (username, organization_name, city_name, admin1_name, country_name); + +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_polygon (username text, organization_name text, postal_code text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon (username, organization_name, postal_code, country_name); + +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_point (username text, organization_name text, postal_code text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point (username, organization_name, postal_code, country_name); + +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_ipaddress_point (username text, organization_name text, ip_address text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_ipaddress_point (username, organization_name, ip_address); + +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_street_point (username text, organization_name text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_street_point (username, organization_name, searchtext, city, state_province, country); + +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_isodistance (username text, organization_name text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_isodistance (username, organization_name, source, mode, range, options); + +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_isochrone (username text, organization_name text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_isochrone (username, organization_name, source, mode, range, options); + +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_route_point_to_point (username text, organization_name text, origin geometry(Point, 4326), destination geometry(Point, 4326), mode text, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_route_point_to_point (username, organization_name, origin, destination, mode, options, units); + +$$ LANGUAGE plproxy; + +-- Make sure by default there are no permissions for publicuser +-- NOTE: this happens at extension creation time, as part of an implicit transaction. +REVOKE ALL PRIVILEGES ON SCHEMA cdb_dataservices_client FROM PUBLIC, publicuser CASCADE; + +-- Grant permissions on the schema to publicuser (but just the schema) +GRANT USAGE ON SCHEMA cdb_dataservices_client TO publicuser; + +-- Revoke execute permissions on all functions in the schema by default +REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_dataservices_client FROM PUBLIC, publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_admin0_polygon(country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_admin1_polygon(admin1_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_admin1_polygon(admin1_name text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point(city_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point(city_name text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_polygon(postal_code text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_point(postal_code text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_ipaddress_point(ip_address text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_isodistance(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_isochrone(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_route_point_to_point(origin geometry(Point, 4326), destination geometry(Point, 4326), mode text, options text[], units text) TO publicuser; diff --git a/client/cdb_dataservices_client.control b/client/cdb_dataservices_client.control index 65e5a22..805f145 100644 --- a/client/cdb_dataservices_client.control +++ b/client/cdb_dataservices_client.control @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices client API extension' -default_version = '0.2.0' +default_version = '0.3.0' requires = 'plproxy, cartodb' superuser = true schema = cdb_dataservices_client diff --git a/client/renderer/interfaces/interface_0.3.0.yaml b/client/renderer/interfaces/interface_0.3.0.yaml new file mode 100644 index 0000000..5fcf26b --- /dev/null +++ b/client/renderer/interfaces/interface_0.3.0.yaml @@ -0,0 +1,91 @@ +--- +- name: cdb_geocode_admin0_polygon + return_type: Geometry + params: + - { name: country_name, type: text } + +- name: cdb_geocode_admin1_polygon + return_type: Geometry + params: + - { name: admin1_name, type: text } + +- name: cdb_geocode_admin1_polygon + return_type: Geometry + params: + - { name: admin1_name, type: text } + - { name: country_name, type: text } + +- name: cdb_geocode_namedplace_point + return_type: Geometry + params: + - { name: city_name, type: text} + +- name: cdb_geocode_namedplace_point + return_type: Geometry + params: + - { name: city_name, type: text} + - { name: country_name, type: text} + +- name: cdb_geocode_namedplace_point + return_type: Geometry + params: + - { name: city_name, type: text} + - { name: admin1_name, type: text} + - { name: country_name, type: text} + + +- name: cdb_geocode_postalcode_polygon + return_type: Geometry + params: + - { name: postal_code, type: text} + - { name: country_name, type: text} + +- name: cdb_geocode_postalcode_point + return_type: Geometry + params: + - { name: postal_code, type: text} + - { name: country_name, type: text} + +- name: cdb_geocode_ipaddress_point + return_type: Geometry + params: + - { name: ip_address, type: text} + +- name: cdb_geocode_street_point + return_type: Geometry + params: + - { name: searchtext, type: text} + - { name: city, type: text, default: 'NULL'} + - { name: state_province, type: text, default: 'NULL'} + - { name: country, type: text, default: 'NULL'} + +- name: cdb_isodistance + return_type: SETOF cdb_dataservices_client.isoline + multi_row: true + multi_field: true + params: + - { name: source, type: "geometry(Geometry, 4326)" } + - { name: mode, type: text } + - { name: range, type: "integer[]" } + - { name: options, type: "text[]", default: 'ARRAY[]::text[]' } + +- name: cdb_isochrone + return_type: SETOF cdb_dataservices_client.isoline + multi_row: true + multi_field: true + params: + - { name: source, type: "geometry(Geometry, 4326)" } + - { name: mode, type: text } + - { name: range, type: "integer[]" } + - { name: options, type: "text[]", default: 'ARRAY[]::text[]' } + +- name: cdb_route_point_to_point + return_type: cdb_dataservices_client.simple_route + multi_field: true + params: + - { name: origin, type: "geometry(Point, 4326)" } + - { name: destination, type: "geometry(Point, 4326)" } + - { name: mode, type: text } + - { name: options, type: "text[]", default: 'ARRAY[]::text[]' } + - { name: units, type: "text", default: "'kilometers'"} + diff --git a/client/renderer/sql-template-renderer b/client/renderer/sql-template-renderer index 3e2dccc..80a7744 100755 --- a/client/renderer/sql-template-renderer +++ b/client/renderer/sql-template-renderer @@ -28,6 +28,10 @@ class SqlTemplateRenderer @function_signature['return_type'] end + def multi_field + @function_signature['multi_field'] + end + def multi_row @function_signature['multi_row'] end diff --git a/client/renderer/templates/20_public_functions.erb b/client/renderer/templates/20_public_functions.erb index 10755ea..5832e3f 100644 --- a/client/renderer/templates/20_public_functions.erb +++ b/client/renderer/templates/20_public_functions.erb @@ -22,6 +22,9 @@ BEGIN <% if multi_row %> RETURN QUERY SELECT * FROM <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(username, orgname, <%= params %>); + <% elsif multi_field %> + SELECT * FROM <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(username, orgname, <%= params %>) INTO ret; + RETURN ret; <% else %> SELECT <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(username, orgname, <%= params %>) INTO ret; RETURN ret; diff --git a/client/renderer/templates/30_plproxy_functions.erb b/client/renderer/templates/30_plproxy_functions.erb index 6e2909a..b5e117a 100644 --- a/client/renderer/templates/30_plproxy_functions.erb +++ b/client/renderer/templates/30_plproxy_functions.erb @@ -1,7 +1,7 @@ CREATE OR REPLACE FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %> (username text, organization_name text, <%= params_with_type_and_default %>) RETURNS <%= return_type %> AS $$ CONNECT <%= DATASERVICES_CLIENT_SCHEMA %>._server_conn_str(); - <% if multi_row %> + <% if multi_field %> SELECT * FROM <%= DATASERVICES_SERVER_SCHEMA %>.<%= name %> (username, organization_name, <%= params %>); <% else %> SELECT <%= DATASERVICES_SERVER_SCHEMA %>.<%= name %> (username, organization_name, <%= params %>); diff --git a/client/sql/0.3.0/00_header.sql b/client/sql/0.3.0/00_header.sql new file mode 120000 index 0000000..2277a52 --- /dev/null +++ b/client/sql/0.3.0/00_header.sql @@ -0,0 +1 @@ +../0.2.0/00_header.sql \ No newline at end of file diff --git a/client/sql/0.3.0/10_geocoder_server_conn.sql b/client/sql/0.3.0/10_geocoder_server_conn.sql new file mode 120000 index 0000000..dcd6ff0 --- /dev/null +++ b/client/sql/0.3.0/10_geocoder_server_conn.sql @@ -0,0 +1 @@ +../0.2.0/10_geocoder_server_conn.sql \ No newline at end of file diff --git a/client/sql/0.3.0/15_config_management.sql b/client/sql/0.3.0/15_config_management.sql new file mode 120000 index 0000000..c517070 --- /dev/null +++ b/client/sql/0.3.0/15_config_management.sql @@ -0,0 +1 @@ +../0.2.0/15_config_management.sql \ No newline at end of file diff --git a/client/sql/0.3.0/16_custom_types.sql b/client/sql/0.3.0/16_custom_types.sql new file mode 100644 index 0000000..97d728c --- /dev/null +++ b/client/sql/0.3.0/16_custom_types.sql @@ -0,0 +1,11 @@ +CREATE TYPE cdb_dataservices_client.isoline AS ( + center geometry(Geometry,4326), + data_range integer, + the_geom geometry(Multipolygon,4326) +); + +CREATE TYPE cdb_dataservices_client.simple_route AS ( + shape geometry(LineString,4326), + length real, + duration integer +); \ No newline at end of file diff --git a/client/sql/0.3.0/80_permissions.sql b/client/sql/0.3.0/80_permissions.sql new file mode 120000 index 0000000..359eb6a --- /dev/null +++ b/client/sql/0.3.0/80_permissions.sql @@ -0,0 +1 @@ +../0.2.0/80_permissions.sql \ No newline at end of file diff --git a/client/test/0.3.0/expected/00_installation_test.out b/client/test/0.3.0/expected/00_installation_test.out new file mode 120000 index 0000000..c971a1a --- /dev/null +++ b/client/test/0.3.0/expected/00_installation_test.out @@ -0,0 +1 @@ +../../0.2.0/expected/00_installation_test.out \ No newline at end of file diff --git a/client/test/0.3.0/expected/10_admin0_test.out b/client/test/0.3.0/expected/10_admin0_test.out new file mode 120000 index 0000000..f4d8bb4 --- /dev/null +++ b/client/test/0.3.0/expected/10_admin0_test.out @@ -0,0 +1 @@ +../../0.2.0/expected/10_admin0_test.out \ No newline at end of file diff --git a/client/test/0.3.0/expected/20_admin1_test.out b/client/test/0.3.0/expected/20_admin1_test.out new file mode 120000 index 0000000..9e2dc41 --- /dev/null +++ b/client/test/0.3.0/expected/20_admin1_test.out @@ -0,0 +1 @@ +../../0.2.0/expected/20_admin1_test.out \ No newline at end of file diff --git a/client/test/0.3.0/expected/30_namedplaces_test.out b/client/test/0.3.0/expected/30_namedplaces_test.out new file mode 120000 index 0000000..1cc8e82 --- /dev/null +++ b/client/test/0.3.0/expected/30_namedplaces_test.out @@ -0,0 +1 @@ +../../0.2.0/expected/30_namedplaces_test.out \ No newline at end of file diff --git a/client/test/0.3.0/expected/40_postalcodes_test.out b/client/test/0.3.0/expected/40_postalcodes_test.out new file mode 120000 index 0000000..b5eca98 --- /dev/null +++ b/client/test/0.3.0/expected/40_postalcodes_test.out @@ -0,0 +1 @@ +../../0.2.0/expected/40_postalcodes_test.out \ No newline at end of file diff --git a/client/test/0.3.0/expected/50_ipaddresses_test.out b/client/test/0.3.0/expected/50_ipaddresses_test.out new file mode 120000 index 0000000..4e6cf45 --- /dev/null +++ b/client/test/0.3.0/expected/50_ipaddresses_test.out @@ -0,0 +1 @@ +../../0.2.0/expected/50_ipaddresses_test.out \ No newline at end of file diff --git a/client/test/0.3.0/expected/60_street_test.out b/client/test/0.3.0/expected/60_street_test.out new file mode 120000 index 0000000..a89a234 --- /dev/null +++ b/client/test/0.3.0/expected/60_street_test.out @@ -0,0 +1 @@ +../../0.2.0/expected/60_street_test.out \ No newline at end of file diff --git a/client/test/0.3.0/expected/80_route_point_to_point_test.out b/client/test/0.3.0/expected/80_route_point_to_point_test.out new file mode 100644 index 0000000..a054326 --- /dev/null +++ b/client/test/0.3.0/expected/80_route_point_to_point_test.out @@ -0,0 +1,41 @@ +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_dataservices_client; +-- Mock the server functions +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point (username text, orgname text, origin geometry(Point, 4326), destination geometry(Point, 4326), mode TEXT, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ +DECLARE + ret cdb_dataservices_client.simple_route; +BEGIN + RAISE NOTICE 'cdb_dataservices_server.cdb_route_point_to_point invoked with params (%, %, %, %, %, %, %)', username, orgname, origin, destination, mode, options, units; + SELECT NULL, 5.33, 100 INTO ret; + RETURN ret; +END; +$$ LANGUAGE 'plpgsql'; +-- Exercise the public and the proxied function +SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); +NOTICE: cdb_dataservices_client._cdb_route_point_to_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_point_to_point invoked with params (test_user, , 0101000000D53E1D8F19F455C0185B087250F24440, 0101000000465F419AB1F255C0D8B628B341EE4440, car, {}, kilometers) +CONTEXT: SQL statement "SELECT * FROM cdb_dataservices_client._cdb_route_point_to_point(username, orgname, origin, destination, mode, options, units)" +PL/pgSQL function cdb_route_point_to_point(geometry,geometry,text,text[],text) line 16 at SQL statement + cdb_route_point_to_point +-------------------------- + (,5.33,100) +(1 row) + +SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]); +NOTICE: cdb_dataservices_client._cdb_route_point_to_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_point_to_point invoked with params (test_user, , 0101000000D53E1D8F19F455C0185B087250F24440, 0101000000465F419AB1F255C0D8B628B341EE4440, car, {mode_type=shortest}, kilometers) +CONTEXT: SQL statement "SELECT * FROM cdb_dataservices_client._cdb_route_point_to_point(username, orgname, origin, destination, mode, options, units)" +PL/pgSQL function cdb_route_point_to_point(geometry,geometry,text,text[],text) line 16 at SQL statement + cdb_route_point_to_point +-------------------------- + (,5.33,100) +(1 row) + +SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY[]::text[], 'miles'); +NOTICE: cdb_dataservices_client._cdb_route_point_to_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_point_to_point invoked with params (test_user, , 0101000000D53E1D8F19F455C0185B087250F24440, 0101000000465F419AB1F255C0D8B628B341EE4440, car, {}, miles) +CONTEXT: SQL statement "SELECT * FROM cdb_dataservices_client._cdb_route_point_to_point(username, orgname, origin, destination, mode, options, units)" +PL/pgSQL function cdb_route_point_to_point(geometry,geometry,text,text[],text) line 16 at SQL statement + cdb_route_point_to_point +-------------------------- + (,5.33,100) +(1 row) + diff --git a/client/test/0.3.0/expected/90_permissions_test.out b/client/test/0.3.0/expected/90_permissions_test.out new file mode 100644 index 0000000..d07923c --- /dev/null +++ b/client/test/0.3.0/expected/90_permissions_test.out @@ -0,0 +1,128 @@ +-- Use regular user role +SET ROLE test_regular_user; +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_dataservices_client; +-- Exercise the public function +-- it is public, it shall work +SELECT cdb_geocode_admin0_polygon('Spain'); +NOTICE: cdb_dataservices_client._cdb_geocode_admin0_polygon(3): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin0_polygon invoked with params (test_user, , Spain) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._cdb_geocode_admin0_polygon(username, orgname, country_name)" +PL/pgSQL function cdb_geocode_admin0_polygon(text) line 16 at SQL statement + cdb_geocode_admin0_polygon +---------------------------- + +(1 row) + +SELECT cdb_geocode_admin1_polygon('California'); +NOTICE: cdb_dataservices_client._cdb_geocode_admin1_polygon(3): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (test_user, , California) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name)" +PL/pgSQL function cdb_geocode_admin1_polygon(text) line 16 at SQL statement + cdb_geocode_admin1_polygon +---------------------------- + +(1 row) + +SELECT cdb_geocode_admin1_polygon('California', 'United States'); +NOTICE: cdb_dataservices_client._cdb_geocode_admin1_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (test_user, , California, United States) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name, country_name)" +PL/pgSQL function cdb_geocode_admin1_polygon(text,text) line 16 at SQL statement + cdb_geocode_admin1_polygon +---------------------------- + +(1 row) + +SELECT cdb_geocode_namedplace_point('Elx'); +NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(3): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , Elx) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name)" +PL/pgSQL function cdb_geocode_namedplace_point(text) line 16 at SQL statement + cdb_geocode_namedplace_point +------------------------------ + +(1 row) + +SELECT cdb_geocode_namedplace_point('Elx', 'Valencia'); +NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , Elx, Valencia) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name, country_name)" +PL/pgSQL function cdb_geocode_namedplace_point(text,text) line 16 at SQL statement + cdb_geocode_namedplace_point +------------------------------ + +(1 row) + +SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); +NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , Elx, Valencia, Spain) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name, admin1_name, country_name)" +PL/pgSQL function cdb_geocode_namedplace_point(text,text,text) line 16 at SQL statement + cdb_geocode_namedplace_point +------------------------------ + +(1 row) + +SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); +NOTICE: cdb_dataservices_client._cdb_geocode_postalcode_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_postalcode_polygon invoked with params (test_user, , 03204, Spain) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name)" +PL/pgSQL function cdb_geocode_postalcode_polygon(text,text) line 16 at SQL statement + cdb_geocode_postalcode_polygon +-------------------------------- + +(1 row) + +SELECT cdb_geocode_postalcode_point('03204', 'Spain'); +NOTICE: cdb_dataservices_client._cdb_geocode_postalcode_point(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_postalcode_point invoked with params (test_user, , 03204, Spain) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name)" +PL/pgSQL function cdb_geocode_postalcode_point(text,text) line 16 at SQL statement + cdb_geocode_postalcode_point +------------------------------ + +(1 row) + +SELECT cdb_geocode_ipaddress_point('8.8.8.8'); +NOTICE: cdb_dataservices_client._cdb_geocode_ipaddress_point(3): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_ipaddress_point invoked with params (test_user, , 8.8.8.8) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._cdb_geocode_ipaddress_point(username, orgname, ip_address)" +PL/pgSQL function cdb_geocode_ipaddress_point(text) line 16 at SQL statement + cdb_geocode_ipaddress_point +----------------------------- + +(1 row) + +SELECT cdb_geocode_street_point('one street, 1'); +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , one street, 1, , , ) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country)" +PL/pgSQL function cdb_geocode_street_point(text,text,text,text) line 16 at SQL statement + cdb_geocode_street_point +-------------------------- + +(1 row) + +SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); +NOTICE: cdb_dataservices_client._cdb_route_point_to_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_point_to_point invoked with params (test_user, , 0101000000D53E1D8F19F455C0185B087250F24440, 0101000000465F419AB1F255C0D8B628B341EE4440, car, {}, kilometers) +CONTEXT: SQL statement "SELECT * FROM cdb_dataservices_client._cdb_route_point_to_point(username, orgname, origin, destination, mode, options, units)" +PL/pgSQL function cdb_route_point_to_point(geometry,geometry,text,text[],text) line 16 at SQL statement + cdb_route_point_to_point +-------------------------- + (,5.33,100) +(1 row) + +-- Check the regular user has no permissions on private functions +SELECT _cdb_geocode_admin0_polygon('evil_user', 'evil_orgname', 'Hell'); +ERROR: permission denied for function _cdb_geocode_admin0_polygon +SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'Hell'); +ERROR: permission denied for function _cdb_geocode_admin1_polygon +SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'Sheol', 'Hell'); +ERROR: permission denied for function _cdb_geocode_admin1_polygon +SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol'); +ERROR: permission denied for function _cdb_geocode_namedplace_point +SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol', 'Hell'); +ERROR: permission denied for function _cdb_geocode_namedplace_point +SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol', 'Hell', 'Ugly world'); +ERROR: permission denied for function _cdb_geocode_namedplace_point +SELECT _cdb_geocode_postalcode_polygon('evil_user', 'evil_orgname', '66666', 'Hell'); +ERROR: permission denied for function _cdb_geocode_postalcode_polygon +SELECT _cdb_geocode_postalcode_point('evil_user', 'evil_orgname', '66666', 'Hell'); +ERROR: permission denied for function _cdb_geocode_postalcode_point +SELECT _cdb_geocode_ipaddress_point('evil_user', 'evil_orgname', '8.8.8.8'); +ERROR: permission denied for function _cdb_geocode_ipaddress_point +SELECT _cdb_geocode_street_point('evil_user', 'evil_orgname', 'one street, 1'); +ERROR: permission denied for function _cdb_geocode_street_point +SELECT _cdb_route_point_to_point('evil_user', 'evil_orgname', 'POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); +ERROR: permission denied for function _cdb_route_point_to_point diff --git a/client/test/0.3.0/sql/00_installation_test.sql b/client/test/0.3.0/sql/00_installation_test.sql new file mode 120000 index 0000000..57b2753 --- /dev/null +++ b/client/test/0.3.0/sql/00_installation_test.sql @@ -0,0 +1 @@ +../../0.2.0/sql/00_installation_test.sql \ No newline at end of file diff --git a/client/test/0.3.0/sql/10_admin0_test.sql b/client/test/0.3.0/sql/10_admin0_test.sql new file mode 120000 index 0000000..42ea0af --- /dev/null +++ b/client/test/0.3.0/sql/10_admin0_test.sql @@ -0,0 +1 @@ +../../0.2.0/sql/10_admin0_test.sql \ No newline at end of file diff --git a/client/test/0.3.0/sql/20_admin1_test.sql b/client/test/0.3.0/sql/20_admin1_test.sql new file mode 120000 index 0000000..eff33be --- /dev/null +++ b/client/test/0.3.0/sql/20_admin1_test.sql @@ -0,0 +1 @@ +../../0.2.0/sql/20_admin1_test.sql \ No newline at end of file diff --git a/client/test/0.3.0/sql/30_namedplaces_test.sql b/client/test/0.3.0/sql/30_namedplaces_test.sql new file mode 120000 index 0000000..efcb31a --- /dev/null +++ b/client/test/0.3.0/sql/30_namedplaces_test.sql @@ -0,0 +1 @@ +../../0.2.0/sql/30_namedplaces_test.sql \ No newline at end of file diff --git a/client/test/0.3.0/sql/40_postalcodes_test.sql b/client/test/0.3.0/sql/40_postalcodes_test.sql new file mode 120000 index 0000000..fc5a4c7 --- /dev/null +++ b/client/test/0.3.0/sql/40_postalcodes_test.sql @@ -0,0 +1 @@ +../../0.2.0/sql/40_postalcodes_test.sql \ No newline at end of file diff --git a/client/test/0.3.0/sql/50_ipaddresses_test.sql b/client/test/0.3.0/sql/50_ipaddresses_test.sql new file mode 120000 index 0000000..f94e6ad --- /dev/null +++ b/client/test/0.3.0/sql/50_ipaddresses_test.sql @@ -0,0 +1 @@ +../../0.2.0/sql/50_ipaddresses_test.sql \ No newline at end of file diff --git a/client/test/0.3.0/sql/60_street_test.sql b/client/test/0.3.0/sql/60_street_test.sql new file mode 120000 index 0000000..643e51d --- /dev/null +++ b/client/test/0.3.0/sql/60_street_test.sql @@ -0,0 +1 @@ +../../0.2.0/sql/60_street_test.sql \ No newline at end of file diff --git a/client/test/0.3.0/sql/80_route_point_to_point_test.sql b/client/test/0.3.0/sql/80_route_point_to_point_test.sql new file mode 100644 index 0000000..4e15114 --- /dev/null +++ b/client/test/0.3.0/sql/80_route_point_to_point_test.sql @@ -0,0 +1,21 @@ +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_dataservices_client; + +-- Mock the server functions + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point (username text, orgname text, origin geometry(Point, 4326), destination geometry(Point, 4326), mode TEXT, options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') +RETURNS cdb_dataservices_client.simple_route AS $$ +DECLARE + ret cdb_dataservices_client.simple_route; +BEGIN + RAISE NOTICE 'cdb_dataservices_server.cdb_route_point_to_point invoked with params (%, %, %, %, %, %, %)', username, orgname, origin, destination, mode, options, units; + SELECT NULL, 5.33, 100 INTO ret; + RETURN ret; +END; +$$ LANGUAGE 'plpgsql'; + + +-- Exercise the public and the proxied function +SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); +SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]); +SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY[]::text[], 'miles'); diff --git a/client/test/0.3.0/sql/90_permissions_test.sql b/client/test/0.3.0/sql/90_permissions_test.sql new file mode 100644 index 0000000..adc0be2 --- /dev/null +++ b/client/test/0.3.0/sql/90_permissions_test.sql @@ -0,0 +1,32 @@ +-- Use regular user role +SET ROLE test_regular_user; + +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_dataservices_client; + +-- Exercise the public function +-- it is public, it shall work +SELECT cdb_geocode_admin0_polygon('Spain'); +SELECT cdb_geocode_admin1_polygon('California'); +SELECT cdb_geocode_admin1_polygon('California', 'United States'); +SELECT cdb_geocode_namedplace_point('Elx'); +SELECT cdb_geocode_namedplace_point('Elx', 'Valencia'); +SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); +SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); +SELECT cdb_geocode_postalcode_point('03204', 'Spain'); +SELECT cdb_geocode_ipaddress_point('8.8.8.8'); +SELECT cdb_geocode_street_point('one street, 1'); +SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); + +-- Check the regular user has no permissions on private functions +SELECT _cdb_geocode_admin0_polygon('evil_user', 'evil_orgname', 'Hell'); +SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'Hell'); +SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'Sheol', 'Hell'); +SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol'); +SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol', 'Hell'); +SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol', 'Hell', 'Ugly world'); +SELECT _cdb_geocode_postalcode_polygon('evil_user', 'evil_orgname', '66666', 'Hell'); +SELECT _cdb_geocode_postalcode_point('evil_user', 'evil_orgname', '66666', 'Hell'); +SELECT _cdb_geocode_ipaddress_point('evil_user', 'evil_orgname', '8.8.8.8'); +SELECT _cdb_geocode_street_point('evil_user', 'evil_orgname', 'one street, 1'); +SELECT _cdb_route_point_to_point('evil_user', 'evil_orgname', 'POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car'); From 9a364456610add3e91e030f3440e408e1afb7872 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 24 Feb 2016 17:22:42 +0100 Subject: [PATCH 11/12] Sentinel id could be removed safely from the config Removing the sentinel_master_id property from the cdb_conf redis config, the extension is going to use a single redis instance instead of Sentinel --- .../extension/cdb_dataservices_server--0.4.0--0.5.0.sql | 9 +++++++-- server/extension/cdb_dataservices_server--0.5.0.sql | 9 +++++++-- server/extension/sql/0.5.0/10_redis_helper.sql | 9 +++++++-- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/server/extension/cdb_dataservices_server--0.4.0--0.5.0.sql b/server/extension/cdb_dataservices_server--0.4.0--0.5.0.sql index 06abdb3..fedc7df 100644 --- a/server/extension/cdb_dataservices_server--0.4.0--0.5.0.sql +++ b/server/extension/cdb_dataservices_server--0.4.0--0.5.0.sql @@ -13,13 +13,18 @@ RETURNS cdb_dataservices_server._redis_conf_params AS $$ else: import json params = json.loads(conf) - return { - "sentinel_master_id": params['sentinel_master_id'], + redis_conf_params = { "redis_host": params['redis_host'], "redis_port": params['redis_port'], "timeout": params['timeout'], "redis_db": params['redis_db'] } + if "sentinel_master_id" in params: + redis_conf_params["sentinel_master_id"] = params["sentinel_master_id"] + else: + redis_conf_params["sentinel_master_id"] = None + + return redis_conf_params $$ LANGUAGE plpythonu; -- Get the connection to redis from cache or create a new one diff --git a/server/extension/cdb_dataservices_server--0.5.0.sql b/server/extension/cdb_dataservices_server--0.5.0.sql index 9d1d8f6..ccbc582 100644 --- a/server/extension/cdb_dataservices_server--0.5.0.sql +++ b/server/extension/cdb_dataservices_server--0.5.0.sql @@ -92,13 +92,18 @@ RETURNS cdb_dataservices_server._redis_conf_params AS $$ else: import json params = json.loads(conf) - return { - "sentinel_master_id": params['sentinel_master_id'], + redis_conf_params = { "redis_host": params['redis_host'], "redis_port": params['redis_port'], "timeout": params['timeout'], "redis_db": params['redis_db'] } + if "sentinel_master_id" in params: + redis_conf_params["sentinel_master_id"] = params["sentinel_master_id"] + else: + redis_conf_params["sentinel_master_id"] = None + + return redis_conf_params $$ LANGUAGE plpythonu; -- Get the connection to redis from cache or create a new one diff --git a/server/extension/sql/0.5.0/10_redis_helper.sql b/server/extension/sql/0.5.0/10_redis_helper.sql index 97223f4..77f3b01 100644 --- a/server/extension/sql/0.5.0/10_redis_helper.sql +++ b/server/extension/sql/0.5.0/10_redis_helper.sql @@ -16,13 +16,18 @@ RETURNS cdb_dataservices_server._redis_conf_params AS $$ else: import json params = json.loads(conf) - return { - "sentinel_master_id": params['sentinel_master_id'], + redis_conf_params = { "redis_host": params['redis_host'], "redis_port": params['redis_port'], "timeout": params['timeout'], "redis_db": params['redis_db'] } + if "sentinel_master_id" in params: + redis_conf_params["sentinel_master_id"] = params["sentinel_master_id"] + else: + redis_conf_params["sentinel_master_id"] = None + + return redis_conf_params $$ LANGUAGE plpythonu; -- Get the connection to redis from cache or create a new one From 7ddd2dfc8a309f326eaad1a071a2bd83cc51cc53 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 1 Mar 2016 12:15:05 +0100 Subject: [PATCH 12/12] Update reference.md --- doc/reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/reference.md b/doc/reference.md index 503decf..93c487f 100644 --- a/doc/reference.md +++ b/doc/reference.md @@ -269,7 +269,7 @@ UPDATE {tablename} SET the_geom = cdb_geocode_ipaddress_point('102.23.34.1') ``` ## Street-level geocoder -This function provides a street-level geocoding service. This service uses the street level geocoder defined for the user (currently, only the Here geocoder is available). +This function provides a street-level geocoding service. This service uses the street level geocoder defined for the user. **This service is subject to quota limitations, and extra fees may apply.** Please view our [terms and conditions](https://cartodb.com/terms/)