diff --git a/client/.gitignore b/client/.gitignore index cf79682..dece5cc 100644 --- a/client/.gitignore +++ b/client/.gitignore @@ -6,3 +6,4 @@ regression.out 90_grant_execute.sql cdb_geocoder_client--0.0.1.sql cdb_geocoder_client--0.1.0.sql +cdb_geocoder_client--0.2.0.sql diff --git a/client/Makefile b/client/Makefile index dd32709..46572a5 100644 --- a/client/Makefile +++ b/client/Makefile @@ -12,8 +12,11 @@ NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql # @see http://www.postgresql.org/docs/current/static/extend-pgxs.html DATA = $(NEW_EXTENSION_ARTIFACT) \ cdb_dataservices_client--0.0.1.sql \ + cdb_dataservices_client--0.1.0.sql \ cdb_dataservices_client--0.1.0--0.0.1.sql \ - cdb_dataservices_client--0.0.1--0.1.0.sql + cdb_dataservices_client--0.0.1--0.1.0.sql \ + cdb_dataservices_client--0.2.0--0.1.0.sql \ + cdb_dataservices_client--0.1.0--0.2.0.sql REGRESS = $(notdir $(basename $(wildcard test/$(EXTVERSION)/sql/*test.sql))) diff --git a/client/cdb_dataservices_client--0.1.0--0.2.0.sql b/client/cdb_dataservices_client--0.1.0--0.2.0.sql new file mode 100644 index 0000000..173859a --- /dev/null +++ b/client/cdb_dataservices_client--0.1.0--0.2.0.sql @@ -0,0 +1,60 @@ +CREATE TYPE cdb_dataservices_client.isoline AS ( + center geometry(Geometry,4326), + data_range integer, + the_geom geometry(Multipolygon,4326) +); + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_isodistance (source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + RETURN QUERY + SELECT * FROM cdb_dataservices_client._cdb_isodistance(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_isodistance (username text, organization_name text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + SELECT * FROM cdb_dataservices_server.cdb_isodistance (username, organization_name, source, mode, range, options); +$$ LANGUAGE plproxy; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_isodistance(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_isochrone (source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY + SELECT * FROM cdb_dataservices_client._cdb_isochrone(username, orgname, source, mode, range, options); +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_isochrone (username text, organization_name text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + SELECT * FROM cdb_dataservices_server.cdb_isochrone (username, organization_name, source, mode, range, options); +$$ LANGUAGE plproxy; + +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_isochrone(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; diff --git a/client/cdb_dataservices_client--0.2.0--0.1.0.sql b/client/cdb_dataservices_client--0.2.0--0.1.0.sql new file mode 100644 index 0000000..3fe2211 --- /dev/null +++ b/client/cdb_dataservices_client--0.2.0--0.1.0.sql @@ -0,0 +1,7 @@ +DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_isochrone (geometry(Geometry, 4326), text, integer[], text[]); +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_isochrone (text, text, geometry(Geometry, 4326), text, integer[], text[]); + +DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_isodistance (geometry(Geometry, 4326), text, integer[], text[]); +DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_isodistance (text, text, geometry(Geometry, 4326), text, integer[], text[]); + +DROP TYPE IF EXISTS cdb_dataservices_client.isoline; \ No newline at end of file diff --git a/client/cdb_dataservices_client--0.2.0.sql b/client/cdb_dataservices_client--0.2.0.sql new file mode 100644 index 0000000..20336ac --- /dev/null +++ b/client/cdb_dataservices_client--0.2.0.sql @@ -0,0 +1,513 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_dataservices_client" to load this file. \quit +-- +-- Geocoder server connection config +-- +-- The purpose of this function is provide to the PL/Proxy functions +-- the connection string needed to connect with the server + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._server_conn_str() +RETURNS text AS $$ +DECLARE + db_connection_str text; +BEGIN + SELECT cartodb.cdb_conf_getconf('geocoder_server_config')->'connection_str' INTO db_connection_str; + SELECT trim(both '"' FROM db_connection_str) INTO db_connection_str; + RETURN db_connection_str; +END; +$$ LANGUAGE 'plpgsql'; +CREATE TYPE cdb_dataservices_client._entity_config AS ( + username text, + organization_name text +); + +-- +-- Get entity config function +-- +-- The purpose of this function is to retrieve the username and organization name from +-- a) schema where he/her is the owner in case is an organization user +-- b) entity_name from the cdb_conf database in case is a non organization user +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_entity_config() +RETURNS record AS $$ +DECLARE + result cdb_dataservices_client._entity_config; + is_organization boolean; + username text; + organization_name text; +BEGIN + SELECT cartodb.cdb_conf_getconf('user_config')->'is_organization' INTO is_organization; + IF is_organization IS NULL THEN + RAISE EXCEPTION 'User must have user configuration in the config table'; + ELSIF is_organization = TRUE THEN + SELECT nspname + FROM pg_namespace s + LEFT JOIN pg_roles r ON s.nspowner = r.oid + WHERE r.rolname = session_user INTO username; + SELECT cartodb.cdb_conf_getconf('user_config')->>'entity_name' INTO organization_name; + ELSE + SELECT cartodb.cdb_conf_getconf('user_config')->>'entity_name' INTO username; + organization_name = NULL; + END IF; + result.username = username; + result.organization_name = organization_name; + RETURN result; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; +CREATE TYPE cdb_dataservices_client.isoline AS ( + center geometry(Geometry,4326), + data_range integer, + the_geom geometry(Multipolygon,4326) +);-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_admin0_polygon (country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_admin0_polygon(username, orgname, country_name) INTO ret; + RETURN ret; + +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_admin1_polygon (admin1_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name) INTO ret; + RETURN ret; + +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_admin1_polygon (admin1_name text, country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name, country_name) INTO ret; + RETURN ret; + +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point (city_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name) INTO ret; + RETURN ret; + +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point (city_name text, country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name, country_name) INTO ret; + RETURN ret; + +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point (city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name, admin1_name, country_name) INTO ret; + RETURN ret; + +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_polygon (postal_code text, country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name) INTO ret; + RETURN ret; + +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_point (postal_code text, country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name) INTO ret; + RETURN ret; + +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_ipaddress_point (ip_address text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_ipaddress_point(username, orgname, ip_address) INTO ret; + RETURN ret; + +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocode_street_point (searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + SELECT cdb_dataservices_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; + RETURN ret; + +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_isodistance (source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY + SELECT * FROM cdb_dataservices_client._cdb_isodistance(username, orgname, source, mode, range, options); + +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_isochrone (source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ +DECLARE + + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + + RETURN QUERY + SELECT * FROM cdb_dataservices_client._cdb_isochrone(username, orgname, source, mode, range, options); + +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin0_polygon (username text, organization_name text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_admin0_polygon (username, organization_name, country_name); + +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin1_polygon (username text, organization_name text, admin1_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon (username, organization_name, admin1_name); + +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_admin1_polygon (username text, organization_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon (username, organization_name, admin1_name, country_name); + +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point (username text, organization_name text, city_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_namedplace_point (username, organization_name, city_name); + +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point (username text, organization_name text, city_name text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_namedplace_point (username, organization_name, city_name, country_name); + +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_namedplace_point (username text, organization_name text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_namedplace_point (username, organization_name, city_name, admin1_name, country_name); + +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_polygon (username text, organization_name text, postal_code text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon (username, organization_name, postal_code, country_name); + +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_postalcode_point (username text, organization_name text, postal_code text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_postalcode_point (username, organization_name, postal_code, country_name); + +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_ipaddress_point (username text, organization_name text, ip_address text) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_ipaddress_point (username, organization_name, ip_address); + +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocode_street_point (username text, organization_name text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT cdb_dataservices_server.cdb_geocode_street_point (username, organization_name, searchtext, city, state_province, country); + +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_isodistance (username text, organization_name text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_isodistance (username, organization_name, source, mode, range, options); + +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_isochrone (username text, organization_name text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[]) +RETURNS SETOF cdb_dataservices_client.isoline AS $$ + CONNECT cdb_dataservices_client._server_conn_str(); + + SELECT * FROM cdb_dataservices_server.cdb_isochrone (username, organization_name, source, mode, range, options); + +$$ LANGUAGE plproxy; + +-- Make sure by default there are no permissions for publicuser +-- NOTE: this happens at extension creation time, as part of an implicit transaction. +REVOKE ALL PRIVILEGES ON SCHEMA cdb_dataservices_client FROM PUBLIC, publicuser CASCADE; + +-- Grant permissions on the schema to publicuser (but just the schema) +GRANT USAGE ON SCHEMA cdb_dataservices_client TO publicuser; + +-- Revoke execute permissions on all functions in the schema by default +REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_dataservices_client FROM PUBLIC, publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_admin0_polygon(country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_admin1_polygon(admin1_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_admin1_polygon(admin1_name text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point(city_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point(city_name text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_polygon(postal_code text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_postalcode_point(postal_code text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_ipaddress_point(ip_address text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_isodistance(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_isochrone(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser; diff --git a/client/cdb_dataservices_client.control b/client/cdb_dataservices_client.control index f060043..65e5a22 100644 --- a/client/cdb_dataservices_client.control +++ b/client/cdb_dataservices_client.control @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices client API extension' -default_version = '0.1.0' +default_version = '0.2.0' requires = 'plproxy, cartodb' superuser = true schema = cdb_dataservices_client diff --git a/client/renderer/interfaces/interface_0.2.0.yaml b/client/renderer/interfaces/interface_0.2.0.yaml new file mode 100644 index 0000000..71fe731 --- /dev/null +++ b/client/renderer/interfaces/interface_0.2.0.yaml @@ -0,0 +1,79 @@ +--- +- name: cdb_geocode_admin0_polygon + return_type: Geometry + params: + - { name: country_name, type: text } + +- name: cdb_geocode_admin1_polygon + return_type: Geometry + params: + - { name: admin1_name, type: text } + +- name: cdb_geocode_admin1_polygon + return_type: Geometry + params: + - { name: admin1_name, type: text } + - { name: country_name, type: text } + +- name: cdb_geocode_namedplace_point + return_type: Geometry + params: + - { name: city_name, type: text} + +- name: cdb_geocode_namedplace_point + return_type: Geometry + params: + - { name: city_name, type: text} + - { name: country_name, type: text} + +- name: cdb_geocode_namedplace_point + return_type: Geometry + params: + - { name: city_name, type: text} + - { name: admin1_name, type: text} + - { name: country_name, type: text} + + +- name: cdb_geocode_postalcode_polygon + return_type: Geometry + params: + - { name: postal_code, type: text} + - { name: country_name, type: text} + +- name: cdb_geocode_postalcode_point + return_type: Geometry + params: + - { name: postal_code, type: text} + - { name: country_name, type: text} + +- name: cdb_geocode_ipaddress_point + return_type: Geometry + params: + - { name: ip_address, type: text} + +- name: cdb_geocode_street_point + return_type: Geometry + params: + - { name: searchtext, type: text} + - { name: city, type: text, default: 'NULL'} + - { name: state_province, type: text, default: 'NULL'} + - { name: country, type: text, default: 'NULL'} + +- name: cdb_isodistance + return_type: SETOF cdb_dataservices_client.isoline + multi_row: true + params: + - { name: source, type: "geometry(Geometry, 4326)" } + - { name: mode, type: text } + - { name: range, type: "integer[]" } + - { name: options, type: "text[]", default: 'ARRAY[]::text[]' } + +- name: cdb_isochrone + return_type: SETOF cdb_dataservices_client.isoline + multi_row: true + params: + - { name: source, type: "geometry(Geometry, 4326)" } + - { name: mode, type: text } + - { name: range, type: "integer[]" } + - { name: options, type: "text[]", default: 'ARRAY[]::text[]' } + diff --git a/client/renderer/sql-template-renderer b/client/renderer/sql-template-renderer index e9d5be2..3e2dccc 100755 --- a/client/renderer/sql-template-renderer +++ b/client/renderer/sql-template-renderer @@ -28,6 +28,10 @@ class SqlTemplateRenderer @function_signature['return_type'] end + def multi_row + @function_signature['multi_row'] + end + def user_config_key @function_signature['user_config_key'] end diff --git a/client/renderer/templates/20_public_functions.erb b/client/renderer/templates/20_public_functions.erb index da1b81c..10755ea 100644 --- a/client/renderer/templates/20_public_functions.erb +++ b/client/renderer/templates/20_public_functions.erb @@ -7,7 +7,7 @@ CREATE OR REPLACE FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>.<%= name %> (<%= params_with_type_and_default %>) RETURNS <%= return_type %> AS $$ DECLARE - ret <%= return_type %>; + <% if not multi_row %>ret <%= return_type %>;<% end %> username text; orgname text; BEGIN @@ -19,8 +19,13 @@ BEGIN IF username IS NULL OR username = '' OR username = '""' THEN RAISE EXCEPTION 'Username is a mandatory argument, check it out'; END IF; - SELECT <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(username, orgname, <%= params %>) INTO ret; - RETURN ret; + <% if multi_row %> + RETURN QUERY + SELECT * FROM <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(username, orgname, <%= params %>); + <% else %> + SELECT <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(username, orgname, <%= params %>) INTO ret; + RETURN ret; + <% end %> END; $$ LANGUAGE 'plpgsql' SECURITY DEFINER; diff --git a/client/renderer/templates/30_plproxy_functions.erb b/client/renderer/templates/30_plproxy_functions.erb index 617547d..6e2909a 100644 --- a/client/renderer/templates/30_plproxy_functions.erb +++ b/client/renderer/templates/30_plproxy_functions.erb @@ -1,6 +1,10 @@ CREATE OR REPLACE FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %> (username text, organization_name text, <%= params_with_type_and_default %>) RETURNS <%= return_type %> AS $$ CONNECT <%= DATASERVICES_CLIENT_SCHEMA %>._server_conn_str(); + <% if multi_row %> + SELECT * FROM <%= DATASERVICES_SERVER_SCHEMA %>.<%= name %> (username, organization_name, <%= params %>); + <% else %> SELECT <%= DATASERVICES_SERVER_SCHEMA %>.<%= name %> (username, organization_name, <%= params %>); + <% end %> $$ LANGUAGE plproxy; diff --git a/client/sql/0.2.0/00_header.sql b/client/sql/0.2.0/00_header.sql new file mode 100644 index 0000000..65eb4bf --- /dev/null +++ b/client/sql/0.2.0/00_header.sql @@ -0,0 +1,3 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_dataservices_client" to load this file. \quit diff --git a/client/sql/0.2.0/10_geocoder_server_conn.sql b/client/sql/0.2.0/10_geocoder_server_conn.sql new file mode 100644 index 0000000..89cbcbd --- /dev/null +++ b/client/sql/0.2.0/10_geocoder_server_conn.sql @@ -0,0 +1,16 @@ +-- +-- Geocoder server connection config +-- +-- The purpose of this function is provide to the PL/Proxy functions +-- the connection string needed to connect with the server + +CREATE OR REPLACE FUNCTION cdb_dataservices_client._server_conn_str() +RETURNS text AS $$ +DECLARE + db_connection_str text; +BEGIN + SELECT cartodb.cdb_conf_getconf('geocoder_server_config')->'connection_str' INTO db_connection_str; + SELECT trim(both '"' FROM db_connection_str) INTO db_connection_str; + RETURN db_connection_str; +END; +$$ LANGUAGE 'plpgsql'; diff --git a/client/sql/0.2.0/15_config_management.sql b/client/sql/0.2.0/15_config_management.sql new file mode 100644 index 0000000..7680cfd --- /dev/null +++ b/client/sql/0.2.0/15_config_management.sql @@ -0,0 +1,37 @@ +CREATE TYPE cdb_dataservices_client._entity_config AS ( + username text, + organization_name text +); + +-- +-- Get entity config function +-- +-- The purpose of this function is to retrieve the username and organization name from +-- a) schema where he/her is the owner in case is an organization user +-- b) entity_name from the cdb_conf database in case is a non organization user +CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_entity_config() +RETURNS record AS $$ +DECLARE + result cdb_dataservices_client._entity_config; + is_organization boolean; + username text; + organization_name text; +BEGIN + SELECT cartodb.cdb_conf_getconf('user_config')->'is_organization' INTO is_organization; + IF is_organization IS NULL THEN + RAISE EXCEPTION 'User must have user configuration in the config table'; + ELSIF is_organization = TRUE THEN + SELECT nspname + FROM pg_namespace s + LEFT JOIN pg_roles r ON s.nspowner = r.oid + WHERE r.rolname = session_user INTO username; + SELECT cartodb.cdb_conf_getconf('user_config')->>'entity_name' INTO organization_name; + ELSE + SELECT cartodb.cdb_conf_getconf('user_config')->>'entity_name' INTO username; + organization_name = NULL; + END IF; + result.username = username; + result.organization_name = organization_name; + RETURN result; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; diff --git a/client/sql/0.2.0/16_custom_types.sql b/client/sql/0.2.0/16_custom_types.sql new file mode 100644 index 0000000..5b37dde --- /dev/null +++ b/client/sql/0.2.0/16_custom_types.sql @@ -0,0 +1,5 @@ +CREATE TYPE cdb_dataservices_client.isoline AS ( + center geometry(Geometry,4326), + data_range integer, + the_geom geometry(Multipolygon,4326) +); \ No newline at end of file diff --git a/client/sql/0.2.0/80_permissions.sql b/client/sql/0.2.0/80_permissions.sql new file mode 100644 index 0000000..4af1393 --- /dev/null +++ b/client/sql/0.2.0/80_permissions.sql @@ -0,0 +1,9 @@ +-- Make sure by default there are no permissions for publicuser +-- NOTE: this happens at extension creation time, as part of an implicit transaction. +REVOKE ALL PRIVILEGES ON SCHEMA cdb_dataservices_client FROM PUBLIC, publicuser CASCADE; + +-- Grant permissions on the schema to publicuser (but just the schema) +GRANT USAGE ON SCHEMA cdb_dataservices_client TO publicuser; + +-- Revoke execute permissions on all functions in the schema by default +REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_dataservices_client FROM PUBLIC, publicuser; diff --git a/client/test/0.2.0/expected/00_installation_test.out b/client/test/0.2.0/expected/00_installation_test.out new file mode 100644 index 0000000..8c6f98d --- /dev/null +++ b/client/test/0.2.0/expected/00_installation_test.out @@ -0,0 +1,29 @@ +-- Install dependencies +CREATE EXTENSION postgis; +CREATE EXTENSION schema_triggers; +CREATE EXTENSION plpythonu; +CREATE EXTENSION cartodb; +CREATE EXTENSION plproxy; +-- Install the extension +CREATE EXTENSION cdb_dataservices_client; +-- Mock the server connection to point to this very test db +SELECT cartodb.cdb_conf_setconf('geocoder_server_config', '{"connection_str": "dbname=contrib_regression host=127.0.0.1 user=postgres"}'); + cdb_conf_setconf +------------------ + +(1 row) + +-- Mock the user configuration +SELECT cartodb.cdb_conf_setconf('user_config', '{"is_organization": false, "entity_name": "test_user"}'); + cdb_conf_setconf +------------------ + +(1 row) + +-- Mock the server schema +CREATE SCHEMA cdb_dataservices_server; +-- Create a test user to check permissions +DROP ROLE IF EXISTS test_regular_user; +CREATE ROLE test_regular_user; +GRANT publicuser TO test_regular_user; +ALTER ROLE test_regular_user SET search_path TO public,cartodb,cdb_dataservices_client; diff --git a/client/test/0.2.0/expected/10_admin0_test.out b/client/test/0.2.0/expected/10_admin0_test.out new file mode 100644 index 0000000..8d70ce3 --- /dev/null +++ b/client/test/0.2.0/expected/10_admin0_test.out @@ -0,0 +1,20 @@ +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_dataservices_client; +-- Mock the server function +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_admin0_polygon invoked with params (%, %, %)', username, orgname, country_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; +-- Exercise the public and the proxied function +SELECT cdb_geocode_admin0_polygon('Spain'); +NOTICE: cdb_dataservices_client._cdb_geocode_admin0_polygon(3): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin0_polygon invoked with params (test_user, , Spain) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._cdb_geocode_admin0_polygon(username, orgname, country_name)" +PL/pgSQL function cdb_geocode_admin0_polygon(text) line 16 at SQL statement + cdb_geocode_admin0_polygon +---------------------------- + +(1 row) + diff --git a/client/test/0.2.0/expected/20_admin1_test.out b/client/test/0.2.0/expected/20_admin1_test.out new file mode 100644 index 0000000..a703109 --- /dev/null +++ b/client/test/0.2.0/expected/20_admin1_test.out @@ -0,0 +1,36 @@ +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_dataservices_client; +-- Mock the server functions +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (%, %, %)', username, orgname, admin1_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (%, %, %, %)', username, orgname, admin1_name, country_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; +-- Exercise the public and the proxied function +SELECT cdb_geocode_admin1_polygon('California'); +NOTICE: cdb_dataservices_client._cdb_geocode_admin1_polygon(3): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (test_user, , California) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name)" +PL/pgSQL function cdb_geocode_admin1_polygon(text) line 16 at SQL statement + cdb_geocode_admin1_polygon +---------------------------- + +(1 row) + +SELECT cdb_geocode_admin1_polygon('California', 'United States'); +NOTICE: cdb_dataservices_client._cdb_geocode_admin1_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (test_user, , California, United States) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name, country_name)" +PL/pgSQL function cdb_geocode_admin1_polygon(text,text) line 16 at SQL statement + cdb_geocode_admin1_polygon +---------------------------- + +(1 row) + diff --git a/client/test/0.2.0/expected/30_namedplaces_test.out b/client/test/0.2.0/expected/30_namedplaces_test.out new file mode 100644 index 0000000..458ec3a --- /dev/null +++ b/client/test/0.2.0/expected/30_namedplaces_test.out @@ -0,0 +1,52 @@ +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_dataservices_client; +-- Mock the server functions +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (%, %, %)', username, orgname, city_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (%, %, %, %)', username, orgname, city_name, country_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (%, %, %, %, %)', username, orgname, city_name, admin1_name, country_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; +-- Exercise the public and the proxied function +SELECT cdb_geocode_namedplace_point('Elx'); +NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(3): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , Elx) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name)" +PL/pgSQL function cdb_geocode_namedplace_point(text) line 16 at SQL statement + cdb_geocode_namedplace_point +------------------------------ + +(1 row) + +SELECT cdb_geocode_namedplace_point('Elx', 'Spain'); +NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , Elx, Spain) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name, country_name)" +PL/pgSQL function cdb_geocode_namedplace_point(text,text) line 16 at SQL statement + cdb_geocode_namedplace_point +------------------------------ + +(1 row) + +SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); +NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , Elx, Valencia, Spain) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name, admin1_name, country_name)" +PL/pgSQL function cdb_geocode_namedplace_point(text,text,text) line 16 at SQL statement + cdb_geocode_namedplace_point +------------------------------ + +(1 row) + diff --git a/client/test/0.2.0/expected/40_postalcodes_test.out b/client/test/0.2.0/expected/40_postalcodes_test.out new file mode 100644 index 0000000..3be51df --- /dev/null +++ b/client/test/0.2.0/expected/40_postalcodes_test.out @@ -0,0 +1,36 @@ +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_dataservices_client; +-- Mock the server functions +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, postal_code text, country_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_postalcode_polygon invoked with params (%, %, %, %)', username, orgname, postal_code, country_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, postal_code text, country_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_postalcode_point invoked with params (%, %, %, %)', username, orgname, postal_code, country_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; +-- Exercise the public and the proxied function +SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); +NOTICE: cdb_dataservices_client._cdb_geocode_postalcode_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_postalcode_polygon invoked with params (test_user, , 03204, Spain) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name)" +PL/pgSQL function cdb_geocode_postalcode_polygon(text,text) line 16 at SQL statement + cdb_geocode_postalcode_polygon +-------------------------------- + +(1 row) + +SELECT cdb_geocode_postalcode_point('03204', 'Spain'); +NOTICE: cdb_dataservices_client._cdb_geocode_postalcode_point(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_postalcode_point invoked with params (test_user, , 03204, Spain) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name)" +PL/pgSQL function cdb_geocode_postalcode_point(text,text) line 16 at SQL statement + cdb_geocode_postalcode_point +------------------------------ + +(1 row) + diff --git a/client/test/0.2.0/expected/50_ipaddresses_test.out b/client/test/0.2.0/expected/50_ipaddresses_test.out new file mode 100644 index 0000000..917b2b1 --- /dev/null +++ b/client/test/0.2.0/expected/50_ipaddresses_test.out @@ -0,0 +1,20 @@ +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_dataservices_client; +-- Mock the server functions +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip_address text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_ipaddress_point invoked with params (%, %, %)', username, orgname, ip_address; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; +-- Exercise the public and the proxied function +SELECT cdb_geocode_ipaddress_point('8.8.8.8'); +NOTICE: cdb_dataservices_client._cdb_geocode_ipaddress_point(3): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_ipaddress_point invoked with params (test_user, , 8.8.8.8) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._cdb_geocode_ipaddress_point(username, orgname, ip_address)" +PL/pgSQL function cdb_geocode_ipaddress_point(text) line 16 at SQL statement + cdb_geocode_ipaddress_point +----------------------------- + +(1 row) + diff --git a/client/test/0.2.0/expected/60_street_test.out b/client/test/0.2.0/expected/60_street_test.out new file mode 100644 index 0000000..9ebce2d --- /dev/null +++ b/client/test/0.2.0/expected/60_street_test.out @@ -0,0 +1,101 @@ +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_dataservices_client; +-- Mock the server functions +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (%, %, %, %, %, %)', username, orgname, searchtext, city, state_province, country; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; +-- Exercise the public and the proxied function +SELECT cdb_geocode_street_point('One street, 1'); +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , One street, 1, , , ) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country)" +PL/pgSQL function cdb_geocode_street_point(text,text,text,text) line 16 at SQL statement + cdb_geocode_street_point +-------------------------- + +(1 row) + +SELECT cdb_geocode_street_point('One street', 'city'); +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , One street, city, , ) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country)" +PL/pgSQL function cdb_geocode_street_point(text,text,text,text) line 16 at SQL statement + cdb_geocode_street_point +-------------------------- + +(1 row) + +SELECT cdb_geocode_street_point('One street', 'city', 'state'); +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , One street, city, state, ) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country)" +PL/pgSQL function cdb_geocode_street_point(text,text,text,text) line 16 at SQL statement + cdb_geocode_street_point +-------------------------- + +(1 row) + +SELECT cdb_geocode_street_point('One street', 'city', 'state', 'country'); +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , One street, city, state, country) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country)" +PL/pgSQL function cdb_geocode_street_point(text,text,text,text) line 16 at SQL statement + cdb_geocode_street_point +-------------------------- + +(1 row) + +SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country'); +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , One street, city, , country) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country)" +PL/pgSQL function cdb_geocode_street_point(text,text,text,text) line 16 at SQL statement + cdb_geocode_street_point +-------------------------- + +(1 row) + +SELECT cdb_geocode_street_point('One street, 1'); +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , One street, 1, , , ) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country)" +PL/pgSQL function cdb_geocode_street_point(text,text,text,text) line 16 at SQL statement + cdb_geocode_street_point +-------------------------- + +(1 row) + +SELECT cdb_geocode_street_point('One street', 'city'); +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , One street, city, , ) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country)" +PL/pgSQL function cdb_geocode_street_point(text,text,text,text) line 16 at SQL statement + cdb_geocode_street_point +-------------------------- + +(1 row) + +SELECT cdb_geocode_street_point('One street', 'city', 'state'); +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , One street, city, state, ) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country)" +PL/pgSQL function cdb_geocode_street_point(text,text,text,text) line 16 at SQL statement + cdb_geocode_street_point +-------------------------- + +(1 row) + +SELECT cdb_geocode_street_point('One street', 'city', 'state', 'country'); +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , One street, city, state, country) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country)" +PL/pgSQL function cdb_geocode_street_point(text,text,text,text) line 16 at SQL statement + cdb_geocode_street_point +-------------------------- + +(1 row) + +SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country'); +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , One street, city, , country) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country)" +PL/pgSQL function cdb_geocode_street_point(text,text,text,text) line 16 at SQL statement + cdb_geocode_street_point +-------------------------- + +(1 row) + diff --git a/client/test/0.2.0/expected/90_permissions_test.out b/client/test/0.2.0/expected/90_permissions_test.out new file mode 100644 index 0000000..3c12c06 --- /dev/null +++ b/client/test/0.2.0/expected/90_permissions_test.out @@ -0,0 +1,117 @@ +-- Use regular user role +SET ROLE test_regular_user; +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_dataservices_client; +-- Exercise the public function +-- it is public, it shall work +SELECT cdb_geocode_admin0_polygon('Spain'); +NOTICE: cdb_dataservices_client._cdb_geocode_admin0_polygon(3): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin0_polygon invoked with params (test_user, , Spain) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._cdb_geocode_admin0_polygon(username, orgname, country_name)" +PL/pgSQL function cdb_geocode_admin0_polygon(text) line 16 at SQL statement + cdb_geocode_admin0_polygon +---------------------------- + +(1 row) + +SELECT cdb_geocode_admin1_polygon('California'); +NOTICE: cdb_dataservices_client._cdb_geocode_admin1_polygon(3): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (test_user, , California) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name)" +PL/pgSQL function cdb_geocode_admin1_polygon(text) line 16 at SQL statement + cdb_geocode_admin1_polygon +---------------------------- + +(1 row) + +SELECT cdb_geocode_admin1_polygon('California', 'United States'); +NOTICE: cdb_dataservices_client._cdb_geocode_admin1_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (test_user, , California, United States) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name, country_name)" +PL/pgSQL function cdb_geocode_admin1_polygon(text,text) line 16 at SQL statement + cdb_geocode_admin1_polygon +---------------------------- + +(1 row) + +SELECT cdb_geocode_namedplace_point('Elx'); +NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(3): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , Elx) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name)" +PL/pgSQL function cdb_geocode_namedplace_point(text) line 16 at SQL statement + cdb_geocode_namedplace_point +------------------------------ + +(1 row) + +SELECT cdb_geocode_namedplace_point('Elx', 'Valencia'); +NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , Elx, Valencia) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name, country_name)" +PL/pgSQL function cdb_geocode_namedplace_point(text,text) line 16 at SQL statement + cdb_geocode_namedplace_point +------------------------------ + +(1 row) + +SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); +NOTICE: cdb_dataservices_client._cdb_geocode_namedplace_point(5): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, , Elx, Valencia, Spain) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._cdb_geocode_namedplace_point(username, orgname, city_name, admin1_name, country_name)" +PL/pgSQL function cdb_geocode_namedplace_point(text,text,text) line 16 at SQL statement + cdb_geocode_namedplace_point +------------------------------ + +(1 row) + +SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); +NOTICE: cdb_dataservices_client._cdb_geocode_postalcode_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_postalcode_polygon invoked with params (test_user, , 03204, Spain) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name)" +PL/pgSQL function cdb_geocode_postalcode_polygon(text,text) line 16 at SQL statement + cdb_geocode_postalcode_polygon +-------------------------------- + +(1 row) + +SELECT cdb_geocode_postalcode_point('03204', 'Spain'); +NOTICE: cdb_dataservices_client._cdb_geocode_postalcode_point(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_postalcode_point invoked with params (test_user, , 03204, Spain) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name)" +PL/pgSQL function cdb_geocode_postalcode_point(text,text) line 16 at SQL statement + cdb_geocode_postalcode_point +------------------------------ + +(1 row) + +SELECT cdb_geocode_ipaddress_point('8.8.8.8'); +NOTICE: cdb_dataservices_client._cdb_geocode_ipaddress_point(3): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_ipaddress_point invoked with params (test_user, , 8.8.8.8) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._cdb_geocode_ipaddress_point(username, orgname, ip_address)" +PL/pgSQL function cdb_geocode_ipaddress_point(text) line 16 at SQL statement + cdb_geocode_ipaddress_point +----------------------------- + +(1 row) + +SELECT cdb_geocode_street_point('one street, 1'); +NOTICE: cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, , one street, 1, , , ) +CONTEXT: SQL statement "SELECT cdb_dataservices_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country)" +PL/pgSQL function cdb_geocode_street_point(text,text,text,text) line 16 at SQL statement + cdb_geocode_street_point +-------------------------- + +(1 row) + +-- Check the regular user has no permissions on private functions +SELECT _cdb_geocode_admin0_polygon('evil_user', 'evil_orgname', 'Hell'); +ERROR: permission denied for function _cdb_geocode_admin0_polygon +SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'Hell'); +ERROR: permission denied for function _cdb_geocode_admin1_polygon +SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'Sheol', 'Hell'); +ERROR: permission denied for function _cdb_geocode_admin1_polygon +SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol'); +ERROR: permission denied for function _cdb_geocode_namedplace_point +SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol', 'Hell'); +ERROR: permission denied for function _cdb_geocode_namedplace_point +SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol', 'Hell', 'Ugly world'); +ERROR: permission denied for function _cdb_geocode_namedplace_point +SELECT _cdb_geocode_postalcode_polygon('evil_user', 'evil_orgname', '66666', 'Hell'); +ERROR: permission denied for function _cdb_geocode_postalcode_polygon +SELECT _cdb_geocode_postalcode_point('evil_user', 'evil_orgname', '66666', 'Hell'); +ERROR: permission denied for function _cdb_geocode_postalcode_point +SELECT _cdb_geocode_ipaddress_point('evil_user', 'evil_orgname', '8.8.8.8'); +ERROR: permission denied for function _cdb_geocode_ipaddress_point +SELECT _cdb_geocode_street_point('evil_user', 'evil_orgname', 'one street, 1'); +ERROR: permission denied for function _cdb_geocode_street_point diff --git a/client/test/0.2.0/sql/00_installation_test.sql b/client/test/0.2.0/sql/00_installation_test.sql new file mode 100644 index 0000000..fc3395f --- /dev/null +++ b/client/test/0.2.0/sql/00_installation_test.sql @@ -0,0 +1,23 @@ +-- Install dependencies +CREATE EXTENSION postgis; +CREATE EXTENSION schema_triggers; +CREATE EXTENSION plpythonu; +CREATE EXTENSION cartodb; +CREATE EXTENSION plproxy; + +-- Install the extension +CREATE EXTENSION cdb_dataservices_client; + +-- Mock the server connection to point to this very test db +SELECT cartodb.cdb_conf_setconf('geocoder_server_config', '{"connection_str": "dbname=contrib_regression host=127.0.0.1 user=postgres"}'); +-- Mock the user configuration +SELECT cartodb.cdb_conf_setconf('user_config', '{"is_organization": false, "entity_name": "test_user"}'); + +-- Mock the server schema +CREATE SCHEMA cdb_dataservices_server; + +-- Create a test user to check permissions +DROP ROLE IF EXISTS test_regular_user; +CREATE ROLE test_regular_user; +GRANT publicuser TO test_regular_user; +ALTER ROLE test_regular_user SET search_path TO public,cartodb,cdb_dataservices_client; diff --git a/client/test/0.2.0/sql/10_admin0_test.sql b/client/test/0.2.0/sql/10_admin0_test.sql new file mode 100644 index 0000000..e6750f1 --- /dev/null +++ b/client/test/0.2.0/sql/10_admin0_test.sql @@ -0,0 +1,15 @@ +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_dataservices_client; + +-- Mock the server function +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_admin0_polygon invoked with params (%, %, %)', username, orgname, country_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; + + +-- Exercise the public and the proxied function +SELECT cdb_geocode_admin0_polygon('Spain'); diff --git a/client/test/0.2.0/sql/20_admin1_test.sql b/client/test/0.2.0/sql/20_admin1_test.sql new file mode 100644 index 0000000..ecd2275 --- /dev/null +++ b/client/test/0.2.0/sql/20_admin1_test.sql @@ -0,0 +1,24 @@ +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_dataservices_client; + +-- Mock the server functions +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (%, %, %)', username, orgname, admin1_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (%, %, %, %)', username, orgname, admin1_name, country_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; + + +-- Exercise the public and the proxied function +SELECT cdb_geocode_admin1_polygon('California'); +SELECT cdb_geocode_admin1_polygon('California', 'United States'); diff --git a/client/test/0.2.0/sql/30_namedplaces_test.sql b/client/test/0.2.0/sql/30_namedplaces_test.sql new file mode 100644 index 0000000..bdfba65 --- /dev/null +++ b/client/test/0.2.0/sql/30_namedplaces_test.sql @@ -0,0 +1,33 @@ +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_dataservices_client; + +-- Mock the server functions +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (%, %, %)', username, orgname, city_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (%, %, %, %)', username, orgname, city_name, country_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (%, %, %, %, %)', username, orgname, city_name, admin1_name, country_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; + +-- Exercise the public and the proxied function +SELECT cdb_geocode_namedplace_point('Elx'); +SELECT cdb_geocode_namedplace_point('Elx', 'Spain'); +SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); + diff --git a/client/test/0.2.0/sql/40_postalcodes_test.sql b/client/test/0.2.0/sql/40_postalcodes_test.sql new file mode 100644 index 0000000..c1c2191 --- /dev/null +++ b/client/test/0.2.0/sql/40_postalcodes_test.sql @@ -0,0 +1,23 @@ +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_dataservices_client; + +-- Mock the server functions +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, postal_code text, country_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_postalcode_polygon invoked with params (%, %, %, %)', username, orgname, postal_code, country_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, postal_code text, country_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_postalcode_point invoked with params (%, %, %, %)', username, orgname, postal_code, country_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; + +-- Exercise the public and the proxied function +SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); +SELECT cdb_geocode_postalcode_point('03204', 'Spain'); diff --git a/client/test/0.2.0/sql/50_ipaddresses_test.sql b/client/test/0.2.0/sql/50_ipaddresses_test.sql new file mode 100644 index 0000000..c1fd8f7 --- /dev/null +++ b/client/test/0.2.0/sql/50_ipaddresses_test.sql @@ -0,0 +1,15 @@ +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_dataservices_client; + +-- Mock the server functions +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip_address text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_ipaddress_point invoked with params (%, %, %)', username, orgname, ip_address; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; + + +-- Exercise the public and the proxied function +SELECT cdb_geocode_ipaddress_point('8.8.8.8'); diff --git a/client/test/0.2.0/sql/60_street_test.sql b/client/test/0.2.0/sql/60_street_test.sql new file mode 100644 index 0000000..cd22b5e --- /dev/null +++ b/client/test/0.2.0/sql/60_street_test.sql @@ -0,0 +1,24 @@ +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_dataservices_client; + +-- Mock the server functions +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (%, %, %, %, %, %)', username, orgname, searchtext, city, state_province, country; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; + + +-- Exercise the public and the proxied function +SELECT cdb_geocode_street_point('One street, 1'); +SELECT cdb_geocode_street_point('One street', 'city'); +SELECT cdb_geocode_street_point('One street', 'city', 'state'); +SELECT cdb_geocode_street_point('One street', 'city', 'state', 'country'); +SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country'); +SELECT cdb_geocode_street_point('One street, 1'); +SELECT cdb_geocode_street_point('One street', 'city'); +SELECT cdb_geocode_street_point('One street', 'city', 'state'); +SELECT cdb_geocode_street_point('One street', 'city', 'state', 'country'); +SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country'); diff --git a/client/test/0.2.0/sql/90_permissions_test.sql b/client/test/0.2.0/sql/90_permissions_test.sql new file mode 100644 index 0000000..88ec14c --- /dev/null +++ b/client/test/0.2.0/sql/90_permissions_test.sql @@ -0,0 +1,30 @@ +-- Use regular user role +SET ROLE test_regular_user; + +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_dataservices_client; + +-- Exercise the public function +-- it is public, it shall work +SELECT cdb_geocode_admin0_polygon('Spain'); +SELECT cdb_geocode_admin1_polygon('California'); +SELECT cdb_geocode_admin1_polygon('California', 'United States'); +SELECT cdb_geocode_namedplace_point('Elx'); +SELECT cdb_geocode_namedplace_point('Elx', 'Valencia'); +SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); +SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); +SELECT cdb_geocode_postalcode_point('03204', 'Spain'); +SELECT cdb_geocode_ipaddress_point('8.8.8.8'); +SELECT cdb_geocode_street_point('one street, 1'); + +-- Check the regular user has no permissions on private functions +SELECT _cdb_geocode_admin0_polygon('evil_user', 'evil_orgname', 'Hell'); +SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'Hell'); +SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'Sheol', 'Hell'); +SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol'); +SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol', 'Hell'); +SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol', 'Hell', 'Ugly world'); +SELECT _cdb_geocode_postalcode_polygon('evil_user', 'evil_orgname', '66666', 'Hell'); +SELECT _cdb_geocode_postalcode_point('evil_user', 'evil_orgname', '66666', 'Hell'); +SELECT _cdb_geocode_ipaddress_point('evil_user', 'evil_orgname', '8.8.8.8'); +SELECT _cdb_geocode_street_point('evil_user', 'evil_orgname', 'one street, 1'); diff --git a/doc/reference.md b/doc/reference.md index 503decf..8b5c707 100644 --- a/doc/reference.md +++ b/doc/reference.md @@ -310,3 +310,65 @@ SELECT cdb_geocode_street_point('651 Lombard Street', 'San Francisco', 'Californ ```bash UPDATE {tablename} SET the_geom = cdb_geocode_street_point({street_name_column}) ``` +# Routing functions + +The following rouging functions are available, grouped by categories. + +## Isolines + +This function provides an isolines generator sirves based on time or distance. + +### cdb_isodistance(_source geometry, mode text, range integer[], options text[]_) + +#### Arguments + +Name | Type | Description | Accepted values +--- | --- | --- | --- +`source` | `geometry` | Source point, in 4326 projection, taken as the start point +`mode` | `geometry` | Type of transport used to calculate the isolines. | `car` and `walk` +`range` | `integer[]` | Range of the isoline in meters +`options` | `text[]` | Multiple options to add more capabilities to the analysis. See the options section to know more. + +#### Options + +The options values must be pass in the following way: `option=value`. + +Name | Type | Description | Accepted values +--- | --- | --- | --- +`is_destination` | `boolean` | If is true the source point is the destination instead of the starting one. +`mode_type` | `text` | Type of route calculation | `shortest` or `fastest`. By default is `shortest` +`mode_traffic` | `text` | Use the traffic data to calculate the route. | `enabled` or `disabled`. By default is `disabled` +`singlecomponent` | `boolean` | If set to true the isoline service will always return single polygon, instead of creating a separate polygon for each ferry separated island. | `true` or `false`. Default value is false. +`resolution` | `text` | Allows to specify level of detail needed for the isoline polygon. Unit is meters per pixel. Higher resolution may cause increased response time from the service. +`maxpoints` | `text` | Allows to limit amount of points in the returned isoline. If isoline consists of multiple components, sum of points from all components is considered. Each component will have at least 2 points, so it is possible that more points than maxpoints value will be returned. This is in case when 2 * number of components is higher than maxpoints. Enlarging number of maxpoints may cause increased response time from the service. +`quality` | `text` | Allows to reduce the quality of the isoline in favor of the response time. | `1`, `2`, `3`. Default value is 1 and it is the best quality. + +#### Returns + +Name | Type | Description +--- | --- | --- +`center` | `geometry` | Source point, in 4326 projection, taken as the start point +`data_range` | `integer` | The range that belongs to the generated isoline. +`the_geom` | `geometry (multipolygon)` | Geometry of the generated isoline in 4326 projection. + +#### Example + +##### Select + +```bash +SELECT cdb_isodistance('010100000000000000008006C00DEB9D3C72F44340', 'car', ARRAY[1000,2000]::integer[]); +SELECT cdb_isodistance('010100000000000000008006C00DEB9D3C72F44340', 'walk', ARRAY[1000]::integer[], ARRAY['mode_traffic=enabled','quality=3']::text[]); +``` + +### cdb_isochrone(_source geometry, mode text, range integer[], options text[]_) + +This function uses the same parameters and info as the `cdb_isodistance` function with the difference that the range is measured in seconds instead of meters + +#### Example + +##### Select + +```bash +SELECT cdb_isochrone('010100000000000000008006C00DEB9D3C72F44340', 'car', ARRAY[300,900,12000]::integer[]); +SELECT cdb_isodistance('010100000000000000008006C00DEB9D3C72F44340', 'walk', ARRAY[300,900]::integer[], ARRAY['mode_traffic=enabled','quality=3']::text[]); +``` diff --git a/server/extension/Makefile b/server/extension/Makefile index a7b01b4..337931b 100644 --- a/server/extension/Makefile +++ b/server/extension/Makefile @@ -14,12 +14,15 @@ DATA = $(NEW_EXTENSION_ARTIFACT) \ cdb_dataservices_server--0.0.1.sql \ cdb_dataservices_server--0.1.0.sql \ cdb_dataservices_server--0.2.0.sql \ + cdb_dataservices_server--0.3.0.sql \ cdb_dataservices_server--0.1.0--0.0.1.sql \ cdb_dataservices_server--0.0.1--0.1.0.sql \ cdb_dataservices_server--0.2.0--0.1.0.sql \ cdb_dataservices_server--0.1.0--0.2.0.sql \ cdb_dataservices_server--0.2.0--0.3.0.sql \ - cdb_dataservices_server--0.3.0--0.2.0.sql + cdb_dataservices_server--0.3.0--0.2.0.sql \ + cdb_dataservices_server--0.3.0--0.4.0.sql \ + cdb_dataservices_server--0.4.0--0.3.0.sql REGRESS = $(notdir $(basename $(wildcard test/$(EXTVERSION)/sql/*test.sql))) TEST_DIR = test/$(EXTVERSION) diff --git a/server/extension/cdb_dataservices_server--0.3.0--0.4.0.sql b/server/extension/cdb_dataservices_server--0.3.0--0.4.0.sql new file mode 100644 index 0000000..b1b85a1 --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.3.0--0.4.0.sql @@ -0,0 +1,115 @@ +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + import json + from cartodb_services.metrics import RoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf'] + if not heremaps_conf_json: + heremaps_app_id = None + heremaps_app_code = None + else: + heremaps_conf = json.loads(heremaps_conf_json) + heremaps_app_id = heremaps_conf['app_id'] + heremaps_app_code = heremaps_conf['app_code'] + routing_config = RoutingConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code) + # --Think about the security concerns with this kind of global cache, it should be only available + # --for this user session but... + GD[cache_key] = routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE TYPE cdb_dataservices_server.isoline AS (center geometry(Geometry,4326), data_range integer, the_geom geometry(Multipolygon,4326)); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_routing_isolines(username TEXT, orgname TEXT, type TEXT, source geometry(Geometry, 4326), mode TEXT, data_range integer[], options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.here import HereMapsRoutingIsoline + from cartodb_services.metrics import QuotaService + from cartodb_services.here.types import geo_polyline_to_multipolygon + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_routing_config = GD["user_routing_config_{0}".format(username)] + + quota_service = QuotaService(user_routing_config, redis_conn) + + try: + client = HereMapsRoutingIsoline(user_routing_config.heremaps_app_id, user_routing_config.heremaps_app_code, base_url = HereMapsRoutingIsoline.STAGING_ROUTING_BASE_URL ) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + source_str = 'geo!%f,%f' % (lat, lon) + else: + source_str = None + + if type == 'isodistance': + resp = client.calculate_isodistance(source_str, mode, data_range, options) + elif type == 'isochrone': + resp = client.calculate_isochrone(source_str, mode, data_range, options) + + if resp: + result = [] + for isoline in resp: + data_range_n = isoline['range'] + polyline = isoline['geom'] + multipolygon = geo_polyline_to_multipolygon(polyline) + result.append([source, data_range_n, multipolygon]) + quota_service.increment_success_geocoder_use() + quota_service.increment_isolines_service_use(len(resp)) + return result + else: + quota_service.increment_empty_geocoder_use() + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to obtain isodistances: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_routing_config_{0}".format(username)] + type = 'isodistance' + + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + isolines = [] + for element in result: + isoline = element['isoline'] + isoline = isoline.translate(None, "()").split(',') + isolines.append(isoline) + + return isolines +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_routing_config_{0}".format(username)] + type = 'isochrone' + + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + isolines = [] + for element in result: + isoline = element['isoline'] + isoline = isoline.translate(None, "()").split(',') + isolines.append(isoline) + + return isolines +$$ LANGUAGE plpythonu; diff --git a/server/extension/cdb_dataservices_server--0.4.0--0.3.0.sql b/server/extension/cdb_dataservices_server--0.4.0--0.3.0.sql new file mode 100644 index 0000000..bb764a1 --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.4.0--0.3.0.sql @@ -0,0 +1,5 @@ +DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_isochrone(TEXT, TEXT, geometry(Geometry, 4326), TEXT, integer[], text[]); +DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_isodistance(TEXT, TEXT, geometry(Geometry, 4326), TEXT, integer[], text[]); +DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_here_routing_isolines(TEXT, TEXT, TEXT, geometry(Geometry, 4326), TEXT, integer[], text[]); +DROP FUNCTION IF EXISTS cdb_dataservices_server._get_routing_config(text, text); +DROP TYPE IF EXISTS cdb_dataservices_server.isoline; \ No newline at end of file diff --git a/server/extension/cdb_dataservices_server--0.4.0.sql b/server/extension/cdb_dataservices_server--0.4.0.sql new file mode 100644 index 0000000..df953c3 --- /dev/null +++ b/server/extension/cdb_dataservices_server--0.4.0.sql @@ -0,0 +1,908 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_dataservices_server" to load this file. \quit +CREATE TYPE cdb_dataservices_server._redis_conf_params AS ( + sentinel_host text, + sentinel_port int, + sentinel_master_id text, + redis_db text, + timeout float +); + +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_redis_conf_v2(config_key text) +RETURNS cdb_dataservices_server._redis_conf_params AS $$ + conf_query = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(config_key) + conf = plpy.execute(conf_query)[0]['conf'] + if conf is None: + plpy.error("There is no redis configuration defined") + else: + import json + params = json.loads(conf) + return { + "sentinel_host": params['sentinel_host'], + "sentinel_port": params['sentinel_port'], + "sentinel_master_id": params['sentinel_master_id'], + "timeout": params['timeout'], + "redis_db": params['redis_db'] + } +$$ LANGUAGE plpythonu; + +-- Get the connection to redis from cache or create a new one +CREATE OR REPLACE FUNCTION cdb_dataservices_server._connect_to_redis(user_id text) +RETURNS boolean AS $$ + cache_key = "redis_connection_{0}".format(user_id) + if cache_key in GD: + return False + else: + from cartodb_services.tools import RedisConnection + metadata_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, + c.sentinel_master_id, c.timeout, c.redis_db + from cdb_dataservices_server._get_redis_conf_v2('redis_metadata_config') c;""")[0] + metrics_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, + c.sentinel_master_id, c.timeout, c.redis_db + from cdb_dataservices_server._get_redis_conf_v2('redis_metrics_config') c;""")[0] + redis_metadata_connection = RedisConnection(metadata_config_params['sentinel_host'], + metadata_config_params['sentinel_port'], + metadata_config_params['sentinel_master_id'], + timeout=metadata_config_params['timeout'], + redis_db=metadata_config_params['redis_db']).redis_connection() + redis_metrics_connection = RedisConnection(metrics_config_params['sentinel_host'], + metrics_config_params['sentinel_port'], + metrics_config_params['sentinel_master_id'], + timeout=metrics_config_params['timeout'], + redis_db=metrics_config_params['redis_db']).redis_connection() + GD[cache_key] = { + 'redis_metadata_connection': redis_metadata_connection, + 'redis_metrics_connection': redis_metrics_connection, + } + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + import json + from cartodb_services.metrics import GeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf'] + if not heremaps_conf_json: + heremaps_app_id = None + heremaps_app_code = None + else: + heremaps_conf = json.loads(heremaps_conf_json) + heremaps_app_id = heremaps_conf['app_id'] + heremaps_app_code = heremaps_conf['app_code'] + geocoder_config = GeocoderConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code) + # --Think about the security concerns with this kind of global cache, it should be only available + # --for this user session but... + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + import json + from cartodb_services.metrics import RoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf'] + if not heremaps_conf_json: + heremaps_app_id = None + heremaps_app_code = None + else: + heremaps_conf = json.loads(heremaps_conf_json) + heremaps_app_id = heremaps_conf['app_id'] + heremaps_app_code = heremaps_conf['app_code'] + routing_config = RoutingConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code) + # --Think about the security concerns with this kind of global cache, it should be only available + # --for this user session but... + GD[cache_key] = routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; +-- Geocodes a street address given a searchtext and a state and/or country +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + plpy.error('Requested geocoder is not available') + +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.here import HereMapsGeocoder + from cartodb_services.metrics import QuotaService + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + # -- Check the quota + quota_service = QuotaService(user_geocoder_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reach the limit of your quota') + + try: + geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + quota_service.increment_success_geocoder_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using here maps geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.google import GoogleMapsGeocoder + from cartodb_services.metrics import QuotaService + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + quota_service = QuotaService(user_geocoder_config, redis_conn) + + try: + geocoder = GoogleMapsGeocoder(user_geocoder_config.google_client_id, user_geocoder_config.google_api_key) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + quota_service.increment_success_geocoder_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using google maps geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin0_polygon(country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT n.the_geom as geom INTO ret + FROM (SELECT q, lower(regexp_replace(q, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text x + FROM (SELECT country_name q) g) d + LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x + LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.the_geom, s.adm0_a3; + + RETURN ret; + END +$$ LANGUAGE plpgsql; +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [admin1_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"]) + rv = plpy.execute(plan, [admin1_name, country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT q, ( + SELECT the_geom + FROM global_province_polygons + WHERE d.c = ANY (synonyms) + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM ( + SELECT + trim(replace(lower(admin1_name),'.',' ')) c, admin1_name q + ) d + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + WITH p AS (SELECT r.c, r.q, (SELECT iso3 FROM country_decoder WHERE lower(country_name) = ANY (synonyms)) i FROM (SELECT trim(replace(lower(admin1_name),'.',' ')) c, country_name q) r) + SELECT + geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_province_polygons + WHERE p.c = ANY (synonyms) + AND iso3 = p.i + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM p) n; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [city_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"]) + rv = plpy.execute(plan, [city_name, country_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"]) + rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH best AS (SELECT s AS q, (SELECT the_geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT city_name as s) p), + next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT city_name as s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT q, geom, TRUE AS success FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH p AS (SELECT r.s, r.c, (SELECT iso2 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT city_name AS s, country_name::text AS c) r), + best AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE c = p.c AND geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH inputcountry AS ( + SELECT iso2 as isoTwo FROM country_decoder WHERE lower(country_name) = ANY (synonyms) LIMIT 1 + ), + p AS ( + SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder, inputcountry WHERE lower(r.a1) = ANY (synonyms) AND admin1_decoder.iso2 = inputcountry.isoTwo LIMIT 1) i FROM (SELECT city_name AS s, admin1_name::text AS a1) r), + best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"]) + rv = plpy.execute(plan, [ip], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_ipaddress_point(ip text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + + new_ip INET; + BEGIN + BEGIN + IF family(ip::inet) = 6 THEN + new_ip := ip::inet; + ELSE + new_ip := ('::ffff:' || ip)::inet; + END IF; + EXCEPTION WHEN OTHERS THEN + SELECT NULL as geom INTO ret; + RETURN ret; + END; + + WITH + ips AS (SELECT ip s, new_ip net), + matches AS (SELECT s, (SELECT the_geom FROM ip_address_locations WHERE network_start_ip <= ips.net ORDER BY network_start_ip DESC LIMIT 1) geom FROM ips) + SELECT geom INTO ret + FROM matches; + RETURN ret; +END +$$ LANGUAGE plpgsql; +CREATE TYPE cdb_dataservices_server.isoline AS (center geometry(Geometry,4326), data_range integer, the_geom geometry(Multipolygon,4326)); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_routing_isolines(username TEXT, orgname TEXT, type TEXT, source geometry(Geometry, 4326), mode TEXT, data_range integer[], options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.here import HereMapsRoutingIsoline + from cartodb_services.metrics import QuotaService + from cartodb_services.here.types import geo_polyline_to_multipolygon + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_routing_config = GD["user_routing_config_{0}".format(username)] + + quota_service = QuotaService(user_routing_config, redis_conn) + + try: + client = HereMapsRoutingIsoline(user_routing_config.heremaps_app_id, user_routing_config.heremaps_app_code, base_url = HereMapsRoutingIsoline.STAGING_ROUTING_BASE_URL ) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + source_str = 'geo!%f,%f' % (lat, lon) + else: + source_str = None + + if type == 'isodistance': + resp = client.calculate_isodistance(source_str, mode, data_range, options) + elif type == 'isochrone': + resp = client.calculate_isochrone(source_str, mode, data_range, options) + + if resp: + result = [] + for isoline in resp: + data_range_n = isoline['range'] + polyline = isoline['geom'] + multipolygon = geo_polyline_to_multipolygon(polyline) + result.append([source, data_range_n, multipolygon]) + quota_service.increment_success_geocoder_use() + quota_service.increment_isolines_service_use(len(resp)) + return result + else: + quota_service.increment_empty_geocoder_use() + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to obtain isodistances using here maps geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu SECURITY DEFINER; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_routing_config_{0}".format(username)] + type = 'isodistance' + + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + isolines = [] + for element in result: + isoline = element['isoline'] + isoline = isoline.translate(None, "()").split(',') + isolines.append(isoline) + + return isolines +$$ LANGUAGE plpythonu; +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_routing_config_{0}".format(username)] + type = 'isochrone' + + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + isolines = [] + for element in result: + isoline = element['isoline'] + isoline = isoline.translate(None, "()").split(',') + isolines.append(isoline) + + return isolines +$$ LANGUAGE plpythonu; +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT * + FROM pg_catalog.pg_user + WHERE usename = 'geocoder_api') THEN + + CREATE USER geocoder_api; + END IF; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO geocoder_api; + GRANT USAGE ON SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT USAGE ON SCHEMA public TO geocoder_api; + GRANT SELECT ON ALL TABLES IN SCHEMA public TO geocoder_api; +END$$; diff --git a/server/extension/cdb_dataservices_server.control b/server/extension/cdb_dataservices_server.control index 8c7d04c..895ee6d 100644 --- a/server/extension/cdb_dataservices_server.control +++ b/server/extension/cdb_dataservices_server.control @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices server extension' -default_version = '0.3.0' +default_version = '0.4.0' requires = 'plpythonu, postgis, cdb_geocoder' superuser = true schema = cdb_dataservices_server diff --git a/server/extension/sql/0.4.0/00_header.sql b/server/extension/sql/0.4.0/00_header.sql new file mode 100644 index 0000000..2c8035c --- /dev/null +++ b/server/extension/sql/0.4.0/00_header.sql @@ -0,0 +1,3 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_dataservices_server" to load this file. \quit diff --git a/server/extension/sql/0.4.0/10_redis_helper.sql b/server/extension/sql/0.4.0/10_redis_helper.sql new file mode 100644 index 0000000..371ba32 --- /dev/null +++ b/server/extension/sql/0.4.0/10_redis_helper.sql @@ -0,0 +1,57 @@ +CREATE TYPE cdb_dataservices_server._redis_conf_params AS ( + sentinel_host text, + sentinel_port int, + sentinel_master_id text, + redis_db text, + timeout float +); + +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_redis_conf_v2(config_key text) +RETURNS cdb_dataservices_server._redis_conf_params AS $$ + conf_query = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(config_key) + conf = plpy.execute(conf_query)[0]['conf'] + if conf is None: + plpy.error("There is no redis configuration defined") + else: + import json + params = json.loads(conf) + return { + "sentinel_host": params['sentinel_host'], + "sentinel_port": params['sentinel_port'], + "sentinel_master_id": params['sentinel_master_id'], + "timeout": params['timeout'], + "redis_db": params['redis_db'] + } +$$ LANGUAGE plpythonu; + +-- Get the connection to redis from cache or create a new one +CREATE OR REPLACE FUNCTION cdb_dataservices_server._connect_to_redis(user_id text) +RETURNS boolean AS $$ + cache_key = "redis_connection_{0}".format(user_id) + if cache_key in GD: + return False + else: + from cartodb_services.tools import RedisConnection + metadata_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, + c.sentinel_master_id, c.timeout, c.redis_db + from cdb_dataservices_server._get_redis_conf_v2('redis_metadata_config') c;""")[0] + metrics_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, + c.sentinel_master_id, c.timeout, c.redis_db + from cdb_dataservices_server._get_redis_conf_v2('redis_metrics_config') c;""")[0] + redis_metadata_connection = RedisConnection(metadata_config_params['sentinel_host'], + metadata_config_params['sentinel_port'], + metadata_config_params['sentinel_master_id'], + timeout=metadata_config_params['timeout'], + redis_db=metadata_config_params['redis_db']).redis_connection() + redis_metrics_connection = RedisConnection(metrics_config_params['sentinel_host'], + metrics_config_params['sentinel_port'], + metrics_config_params['sentinel_master_id'], + timeout=metrics_config_params['timeout'], + redis_db=metrics_config_params['redis_db']).redis_connection() + GD[cache_key] = { + 'redis_metadata_connection': redis_metadata_connection, + 'redis_metrics_connection': redis_metrics_connection, + } + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; diff --git a/server/extension/sql/0.4.0/15_config_helper.sql b/server/extension/sql/0.4.0/15_config_helper.sql new file mode 100644 index 0000000..47aad42 --- /dev/null +++ b/server/extension/sql/0.4.0/15_config_helper.sql @@ -0,0 +1,51 @@ +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_geocoder_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + import json + from cartodb_services.metrics import GeocoderConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf'] + if not heremaps_conf_json: + heremaps_app_id = None + heremaps_app_code = None + else: + heremaps_conf = json.loads(heremaps_conf_json) + heremaps_app_id = heremaps_conf['app_id'] + heremaps_app_code = heremaps_conf['app_code'] + geocoder_config = GeocoderConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code) + # --Think about the security concerns with this kind of global cache, it should be only available + # --for this user session but... + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; + +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_routing_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_routing_config_{0}".format(username) + if cache_key in GD: + return False + else: + import json + from cartodb_services.metrics import RoutingConfig + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf'] + if not heremaps_conf_json: + heremaps_app_id = None + heremaps_app_code = None + else: + heremaps_conf = json.loads(heremaps_conf_json) + heremaps_app_id = heremaps_conf['app_id'] + heremaps_app_code = heremaps_conf['app_code'] + routing_config = RoutingConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code) + # --Think about the security concerns with this kind of global cache, it should be only available + # --for this user session but... + GD[cache_key] = routing_config + return True +$$ LANGUAGE plpythonu SECURITY DEFINER; diff --git a/server/extension/sql/0.4.0/20_geocode_street.sql b/server/extension/sql/0.4.0/20_geocode_street.sql new file mode 100644 index 0000000..de77f62 --- /dev/null +++ b/server/extension/sql/0.4.0/20_geocode_street.sql @@ -0,0 +1,84 @@ +-- Geocodes a street address given a searchtext and a state and/or country +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + plpy.error('Requested geocoder is not available') + +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.here import HereMapsGeocoder + from cartodb_services.metrics import QuotaService + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + # -- Check the quota + quota_service = QuotaService(user_geocoder_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reach the limit of your quota') + + try: + geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + quota_service.increment_success_geocoder_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using here maps geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from cartodb_services.google import GoogleMapsGeocoder + from cartodb_services.metrics import QuotaService + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + quota_service = QuotaService(user_geocoder_config, redis_conn) + + try: + geocoder = GoogleMapsGeocoder(user_geocoder_config.google_client_id, user_geocoder_config.google_api_key) + coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country) + if coordinates: + quota_service.increment_success_geocoder_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using google maps geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; diff --git a/server/extension/sql/0.4.0/30_admin0.sql b/server/extension/sql/0.4.0/30_admin0.sql new file mode 100644 index 0000000..ec8fe5c --- /dev/null +++ b/server/extension/sql/0.4.0/30_admin0.sql @@ -0,0 +1,50 @@ +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin0_polygon(country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT n.the_geom as geom INTO ret + FROM (SELECT q, lower(regexp_replace(q, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text x + FROM (SELECT country_name q) g) d + LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x + LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.the_geom, s.adm0_a3; + + RETURN ret; + END +$$ LANGUAGE plpgsql; diff --git a/server/extension/sql/0.4.0/40_admin1.sql b/server/extension/sql/0.4.0/40_admin1.sql new file mode 100644 index 0000000..dc8e037 --- /dev/null +++ b/server/extension/sql/0.4.0/40_admin1.sql @@ -0,0 +1,117 @@ +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [admin1_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"]) + rv = plpy.execute(plan, [admin1_name, country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT q, ( + SELECT the_geom + FROM global_province_polygons + WHERE d.c = ANY (synonyms) + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM ( + SELECT + trim(replace(lower(admin1_name),'.',' ')) c, admin1_name q + ) d + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_admin1_polygon(admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + WITH p AS (SELECT r.c, r.q, (SELECT iso3 FROM country_decoder WHERE lower(country_name) = ANY (synonyms)) i FROM (SELECT trim(replace(lower(admin1_name),'.',' ')) c, country_name q) r) + SELECT + geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_province_polygons + WHERE p.c = ANY (synonyms) + AND iso3 = p.i + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM p) n; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + diff --git a/server/extension/sql/0.4.0/50_namedplaces.sql b/server/extension/sql/0.4.0/50_namedplaces.sql new file mode 100644 index 0000000..80306cb --- /dev/null +++ b/server/extension/sql/0.4.0/50_namedplaces.sql @@ -0,0 +1,164 @@ +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [city_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"]) + rv = plpy.execute(plan, [city_name, country_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"]) + rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH best AS (SELECT s AS q, (SELECT the_geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT city_name as s) p), + next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT city_name as s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT q, geom, TRUE AS success FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH p AS (SELECT r.s, r.c, (SELECT iso2 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT city_name AS s, country_name::text AS c) r), + best AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE c = p.c AND geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH inputcountry AS ( + SELECT iso2 as isoTwo FROM country_decoder WHERE lower(country_name) = ANY (synonyms) LIMIT 1 + ), + p AS ( + SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder, inputcountry WHERE lower(r.a1) = ANY (synonyms) AND admin1_decoder.iso2 = inputcountry.isoTwo LIMIT 1) i FROM (SELECT city_name AS s, admin1_name::text AS a1) r), + best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + diff --git a/server/extension/sql/0.4.0/60_postalcodes.sql b/server/extension/sql/0.4.0/60_postalcodes.sql new file mode 100644 index 0000000..4ba321b --- /dev/null +++ b/server/extension/sql/0.4.0/60_postalcodes.sql @@ -0,0 +1,219 @@ +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_point(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_postalcode_polygon(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; diff --git a/server/extension/sql/0.4.0/70_ips.sql b/server/extension/sql/0.4.0/70_ips.sql new file mode 100644 index 0000000..f76190f --- /dev/null +++ b/server/extension/sql/0.4.0/70_ips.sql @@ -0,0 +1,61 @@ +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = InternalGeocoderConfig(redis_conn, username, orgname) + + quota_service = QuotaService(user_geocoder_config, redis_conn) + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"]) + rv = plpy.execute(plan, [ip], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_geocoder_use() + return result + else: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocode_ipaddress_point(ip text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + + new_ip INET; + BEGIN + BEGIN + IF family(ip::inet) = 6 THEN + new_ip := ip::inet; + ELSE + new_ip := ('::ffff:' || ip)::inet; + END IF; + EXCEPTION WHEN OTHERS THEN + SELECT NULL as geom INTO ret; + RETURN ret; + END; + + WITH + ips AS (SELECT ip s, new_ip net), + matches AS (SELECT s, (SELECT the_geom FROM ip_address_locations WHERE network_start_ip <= ips.net ORDER BY network_start_ip DESC LIMIT 1) geom FROM ips) + SELECT geom INTO ret + FROM matches; + RETURN ret; +END +$$ LANGUAGE plpgsql; diff --git a/server/extension/sql/0.4.0/80_routing_helper.sql b/server/extension/sql/0.4.0/80_routing_helper.sql new file mode 100644 index 0000000..adcdf4c --- /dev/null +++ b/server/extension/sql/0.4.0/80_routing_helper.sql @@ -0,0 +1,51 @@ +CREATE TYPE cdb_dataservices_server.isoline AS (center geometry(Geometry,4326), data_range integer, the_geom geometry(Multipolygon,4326)); + +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_routing_isolines(username TEXT, orgname TEXT, type TEXT, source geometry(Geometry, 4326), mode TEXT, data_range integer[], options text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + import json + from cartodb_services.here import HereMapsRoutingIsoline + from cartodb_services.metrics import QuotaService + from cartodb_services.here.types import geo_polyline_to_multipolygon + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_routing_config = GD["user_routing_config_{0}".format(username)] + + quota_service = QuotaService(user_routing_config, redis_conn) + + try: + client = HereMapsRoutingIsoline(user_routing_config.heremaps_app_id, user_routing_config.heremaps_app_code, base_url = HereMapsRoutingIsoline.STAGING_ROUTING_BASE_URL ) + + if source: + lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat'] + lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon'] + source_str = 'geo!%f,%f' % (lat, lon) + else: + source_str = None + + if type == 'isodistance': + resp = client.calculate_isodistance(source_str, mode, data_range, options) + elif type == 'isochrone': + resp = client.calculate_isochrone(source_str, mode, data_range, options) + + if resp: + result = [] + for isoline in resp: + data_range_n = isoline['range'] + polyline = isoline['geom'] + multipolygon = geo_polyline_to_multipolygon(polyline) + result.append([source, data_range_n, multipolygon]) + quota_service.increment_success_geocoder_use() + quota_service.increment_isolines_service_use(len(resp)) + return result + else: + quota_service.increment_empty_geocoder_use() + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to obtain isodistances using here maps geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + finally: + quota_service.increment_total_geocoder_use() +$$ LANGUAGE plpythonu SECURITY DEFINER; diff --git a/server/extension/sql/0.4.0/85_isodistance.sql b/server/extension/sql/0.4.0/85_isodistance.sql new file mode 100644 index 0000000..b5b553c --- /dev/null +++ b/server/extension/sql/0.4.0/85_isodistance.sql @@ -0,0 +1,18 @@ +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_routing_config_{0}".format(username)] + type = 'isodistance' + + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + isolines = [] + for element in result: + isoline = element['isoline'] + isoline = isoline.translate(None, "()").split(',') + isolines.append(isoline) + + return isolines +$$ LANGUAGE plpythonu; diff --git a/server/extension/sql/0.4.0/90_isochrone.sql b/server/extension/sql/0.4.0/90_isochrone.sql new file mode 100644 index 0000000..31f9d3a --- /dev/null +++ b/server/extension/sql/0.4.0/90_isochrone.sql @@ -0,0 +1,18 @@ +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) +RETURNS SETOF cdb_dataservices_server.isoline AS $$ + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_isolines_config = GD["user_routing_config_{0}".format(username)] + type = 'isochrone' + + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_routing_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"]) + result = plpy.execute(here_plan, [username, orgname, type, source, mode, range, options]) + isolines = [] + for element in result: + isoline = element['isoline'] + isoline = isoline.translate(None, "()").split(',') + isolines.append(isoline) + + return isolines +$$ LANGUAGE plpythonu; diff --git a/server/extension/sql/0.4.0/99_geocoder_server_user.sql b/server/extension/sql/0.4.0/99_geocoder_server_user.sql new file mode 100644 index 0000000..e011682 --- /dev/null +++ b/server/extension/sql/0.4.0/99_geocoder_server_user.sql @@ -0,0 +1,15 @@ +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT * + FROM pg_catalog.pg_user + WHERE usename = 'geocoder_api') THEN + + CREATE USER geocoder_api; + END IF; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO geocoder_api; + GRANT USAGE ON SCHEMA cdb_dataservices_server TO geocoder_api; + GRANT USAGE ON SCHEMA public TO geocoder_api; + GRANT SELECT ON ALL TABLES IN SCHEMA public TO geocoder_api; +END$$; diff --git a/server/extension/test/0.4.0/expected/00_install_test.out b/server/extension/test/0.4.0/expected/00_install_test.out new file mode 100644 index 0000000..db041d0 --- /dev/null +++ b/server/extension/test/0.4.0/expected/00_install_test.out @@ -0,0 +1,36 @@ +-- Install dependencies +CREATE EXTENSION postgis; +CREATE EXTENSION schema_triggers; +CREATE EXTENSION plpythonu; +CREATE EXTENSION cartodb; +CREATE EXTENSION cdb_geocoder; +-- Install the extension +CREATE EXTENSION cdb_dataservices_server; +-- Mock the redis server connection to point to this very test db +SELECT cartodb.cdb_conf_setconf('redis_metrics_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); + cdb_conf_setconf +------------------ + +(1 row) + +SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); + cdb_conf_setconf +------------------ + +(1 row) + +-- Mock the varnish invalidation function +-- (used by cdb_geocoder tests) +CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$ +BEGIN + RETURN; +END +$$ +LANGUAGE plpgsql; +-- Set user quota +SELECT cartodb.CDB_SetUserQuotaInBytes(0); + cdb_setuserquotainbytes +------------------------- + 0 +(1 row) + diff --git a/server/extension/test/0.4.0/expected/20_street_test.out b/server/extension/test/0.4.0/expected/20_street_test.out new file mode 100644 index 0000000..8fbce7a --- /dev/null +++ b/server/extension/test/0.4.0/expected/20_street_test.out @@ -0,0 +1,12 @@ +-- Check for namedplaces signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_street_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, text'); + exists +-------- + t +(1 row) + diff --git a/server/extension/test/0.4.0/expected/30_admin0_test.out b/server/extension/test/0.4.0/expected/30_admin0_test.out new file mode 100644 index 0000000..55c7639 --- /dev/null +++ b/server/extension/test/0.4.0/expected/30_admin0_test.out @@ -0,0 +1,47 @@ +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_dataservices_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'Spain'); + cdb_geocode_admin0_polygon +---------------------------- + +(1 row) + +-- Insert some dummy synonym +INSERT INTO admin0_synonyms (name, adm0_a3) VALUES ('Spain', 'ESP'); +-- Insert some dummy geometry to return +INSERT INTO ne_admin0_v3 (adm0_a3, the_geom) VALUES('ESP', ST_GeomFromText( + 'POLYGON((-71.1031880899493 42.3152774590236, + -71.1031627617667 42.3152960829043, + -71.102923838298 42.3149156848307, + -71.1031880899493 42.3152774590236))',4326) +); +-- This should return the polygon inserted above +SELECT cdb_dataservices_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'Spain'); + cdb_geocode_admin0_polygon +-------------------------------------------------------------------------------------------------------------------------------------------------------------------- + 0103000020E61000000100000004000000D0EA37A29AC651C00FD603035B284540FEFCFB379AC651C0C0503E9F5B284540FFDDDD4D96C651C033AC3B284F284540D0EA37A29AC651C00FD603035B284540 +(1 row) + +-- Check for admin0 signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_admin0_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_admin0_polygon' + AND oidvectortypes(p.proargtypes) = 'text'); + exists +-------- + t +(1 row) + diff --git a/server/extension/test/0.4.0/expected/40_admin1_test.out b/server/extension/test/0.4.0/expected/40_admin1_test.out new file mode 100644 index 0000000..4894b0c --- /dev/null +++ b/server/extension/test/0.4.0/expected/40_admin1_test.out @@ -0,0 +1,81 @@ +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California'); + cdb_geocode_admin1_polygon +---------------------------- + +(1 row) + +SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California', 'United States'); + cdb_geocode_admin1_polygon +---------------------------- + +(1 row) + +-- Insert dummy data into country decoder table +INSERT INTO country_decoder (synonyms, iso3) VALUES (Array['united states'], 'USA'); +-- Insert some dummy data and geometry to return +INSERT INTO global_province_polygons (synonyms, iso3, the_geom) VALUES (Array['california'], 'USA', ST_GeomFromText( + 'POLYGON((-71.1031880899493 42.3152774590236, + -71.1031627617667 42.3152960829043, + -71.102923838298 42.3149156848307, + -71.1031880899493 42.3152774590236))',4326) +); +-- This should return the polygon inserted above +SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California'); + cdb_geocode_admin1_polygon +-------------------------------------------------------------------------------------------------------------------------------------------------------------------- + 0103000020E61000000100000004000000D0EA37A29AC651C00FD603035B284540FEFCFB379AC651C0C0503E9F5B284540FFDDDD4D96C651C033AC3B284F284540D0EA37A29AC651C00FD603035B284540 +(1 row) + +SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California', 'United States'); + cdb_geocode_admin1_polygon +-------------------------------------------------------------------------------------------------------------------------------------------------------------------- + 0103000020E61000000100000004000000D0EA37A29AC651C00FD603035B284540FEFCFB379AC651C0C0503E9F5B284540FFDDDD4D96C651C033AC3B284F284540D0EA37A29AC651C00FD603035B284540 +(1 row) + +-- Check for admin1 signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_admin1_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_admin1_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_admin1_polygon' + AND oidvectortypes(p.proargtypes) = 'text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_admin1_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text'); + exists +-------- + t +(1 row) + diff --git a/server/extension/test/0.4.0/expected/50_namedplaces_test.out b/server/extension/test/0.4.0/expected/50_namedplaces_test.out new file mode 100644 index 0000000..7755bf4 --- /dev/null +++ b/server/extension/test/0.4.0/expected/50_namedplaces_test.out @@ -0,0 +1,136 @@ +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx'); + cdb_geocode_namedplace_point +------------------------------ + +(1 row) + +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain'); + cdb_geocode_namedplace_point +------------------------------ + +(1 row) + +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain'); + cdb_geocode_namedplace_point +------------------------------ + +(1 row) + +-- Insert dummy data into points table +INSERT INTO global_cities_points_limited (geoname_id, name, iso2, admin1, admin2, population, lowername, the_geom) VALUES (3128760, 'Elche', 'ES', 'Valencia', 'AL', 34534, 'elche', ST_GeomFromText( + 'POINT(0.6983 39.26787)',4326) +); +-- Insert dummy data into alternates table +INSERT INTO global_cities_alternates_limited (geoname_id, name, preferred, lowername, admin1_geonameid, iso2, admin1, the_geom) VALUES (3128760, 'Elx', true, 'elx', '000000', 'ES', 'Valencia', ST_GeomFromText( + 'POINT(0.6983 39.26787)',4326) +); +-- Insert dummy data into country decoder table +INSERT INTO country_decoder (synonyms, iso2) VALUES (Array['spain'], 'ES'); +-- Insert dummy data into admin1 decoder table +INSERT INTO admin1_decoder (admin1, synonyms, iso2) VALUES ('Valencia', Array['valencia', 'Valencia'], 'ES'); +-- This should return the point inserted above +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx'); + cdb_geocode_namedplace_point +---------------------------------------------------- + 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 +(1 row) + +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche'); + cdb_geocode_namedplace_point +---------------------------------------------------- + 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 +(1 row) + +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain'); + cdb_geocode_namedplace_point +---------------------------------------------------- + 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 +(1 row) + +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'Spain'); + cdb_geocode_namedplace_point +---------------------------------------------------- + 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 +(1 row) + +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain'); + cdb_geocode_namedplace_point +---------------------------------------------------- + 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 +(1 row) + +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'valencia', 'Spain'); + cdb_geocode_namedplace_point +---------------------------------------------------- + 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 +(1 row) + +-- Check for namedplaces signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + exists +-------- + t +(1 row) + diff --git a/server/extension/test/0.4.0/expected/60_postalcodes_test.out b/server/extension/test/0.4.0/expected/60_postalcodes_test.out new file mode 100644 index 0000000..b2a5c91 --- /dev/null +++ b/server/extension/test/0.4.0/expected/60_postalcodes_test.out @@ -0,0 +1,163 @@ +-- Make sure dbs are clean +DELETE FROM global_postal_code_points; +DELETE FROM global_postal_code_polygons; +DELETE FROM country_decoder; +DELETE FROM available_services; +DELETE FROM admin0_synonyms; +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204'); + cdb_geocode_postalcode_point +------------------------------ + +(1 row) + +-- Insert dummy data into ip_address_locations +INSERT INTO global_postal_code_points (the_geom, iso3, postal_code, postal_code_num) VALUES ( + '0101000020E61000000000000000E040408036B47414764840', + 'ESP', + '03204', + 3204 +); +INSERT INTO global_postal_code_polygons (the_geom, iso3, postal_code, postal_code_num) VALUES ( + '0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040', + 'ESP', + '03204', + 3204 +); +INSERT INTO country_decoder (iso3, synonyms) VALUES ( + 'ESP', + Array['spain', 'Spain', 'ESP'] +); +INSERT INTO available_services (adm0_a3, admin0, postal_code_points, postal_code_polygons) VALUES ( + 'ESP', + 't', + 't', + 't' +); +INSERT INTO admin0_synonyms (adm0_a3, name, name_, rank) VALUES ( + 'ESP', + 'Spain', + 'spain', + 3 +); +-- This should return the polygon inserted above +SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204'); + cdb_geocode_postalcode_point +---------------------------------------------------- + 0101000020E61000000000000000E040408036B47414764840 +(1 row) + +SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204', 'spain'); + cdb_geocode_postalcode_point +---------------------------------------------------- + 0101000020E61000000000000000E040408036B47414764840 +(1 row) + +SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', '03204'); + cdb_geocode_postalcode_polygon +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + 0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040 +(1 row) + +SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', '03204', 'spain'); + cdb_geocode_postalcode_polygon +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + 0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040 +(1 row) + +-- Clean dbs +DELETE FROM global_postal_code_points; +DELETE FROM global_postal_code_polygons; +DELETE FROM country_decoder; +DELETE FROM available_services; +DELETE FROM admin0_synonyms; +-- Check for namedplaces signatures (point and polygon) +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_postalcode_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_postalcode_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_postalcode_point' + AND oidvectortypes(p.proargtypes) = 'text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_postalcode_point' + AND oidvectortypes(p.proargtypes) = 'text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text'); + exists +-------- + t +(1 row) + diff --git a/server/extension/test/0.4.0/expected/70_ips_test.out b/server/extension/test/0.4.0/expected/70_ips_test.out new file mode 100644 index 0000000..48012f7 --- /dev/null +++ b/server/extension/test/0.4.0/expected/70_ips_test.out @@ -0,0 +1,40 @@ +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_dataservices_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', '0.0.0.0'); + cdb_geocode_ipaddress_point +----------------------------- + +(1 row) + +-- Insert dummy data into ip_address_locations +INSERT INTO ip_address_locations VALUES ('::ffff:0.0.0.0'::inet, (ST_SetSRID(ST_MakePoint('40.40', '3.71'), 4326))); +-- This should return the polygon inserted above +SELECT cdb_dataservices_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', '0.0.0.0'); + cdb_geocode_ipaddress_point +---------------------------------------------------- + 0101000020E61000003333333333334440AE47E17A14AE0D40 +(1 row) + +-- Check for namedplaces signatures (point and polygon) +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_ipaddress_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_ipaddress_point' + AND oidvectortypes(p.proargtypes) = 'text'); + exists +-------- + t +(1 row) + diff --git a/server/extension/test/0.4.0/expected/85_isodistance_test.out b/server/extension/test/0.4.0/expected/85_isodistance_test.out new file mode 100644 index 0000000..5705284 --- /dev/null +++ b/server/extension/test/0.4.0/expected/85_isodistance_test.out @@ -0,0 +1,12 @@ +-- Check for isodistance signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_isodistance' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, integer[], text[]'); + exists +-------- + t +(1 row) + diff --git a/server/extension/test/0.4.0/expected/90_isochrone_test.out b/server/extension/test/0.4.0/expected/90_isochrone_test.out new file mode 100644 index 0000000..7ee0f04 --- /dev/null +++ b/server/extension/test/0.4.0/expected/90_isochrone_test.out @@ -0,0 +1,12 @@ +-- Check for isochrone signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_isochrone' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, integer[], text[]'); + exists +-------- + t +(1 row) + diff --git a/server/extension/test/0.4.0/expected/99_remove_geocoder_api_user_test.out b/server/extension/test/0.4.0/expected/99_remove_geocoder_api_user_test.out new file mode 100644 index 0000000..ad4fee7 --- /dev/null +++ b/server/extension/test/0.4.0/expected/99_remove_geocoder_api_user_test.out @@ -0,0 +1,5 @@ +REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_dataservices_server FROM geocoder_api; +REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA public FROM geocoder_api; +REVOKE USAGE ON SCHEMA cdb_dataservices_server FROM geocoder_api; +REVOKE USAGE ON SCHEMA public FROM geocoder_api; +REVOKE SELECT ON ALL TABLES IN SCHEMA public FROM geocoder_api; diff --git a/server/extension/test/0.4.0/sql/00_install_test.sql b/server/extension/test/0.4.0/sql/00_install_test.sql new file mode 100644 index 0000000..da91608 --- /dev/null +++ b/server/extension/test/0.4.0/sql/00_install_test.sql @@ -0,0 +1,25 @@ +-- Install dependencies +CREATE EXTENSION postgis; +CREATE EXTENSION schema_triggers; +CREATE EXTENSION plpythonu; +CREATE EXTENSION cartodb; +CREATE EXTENSION cdb_geocoder; + +-- Install the extension +CREATE EXTENSION cdb_dataservices_server; + +-- Mock the redis server connection to point to this very test db +SELECT cartodb.cdb_conf_setconf('redis_metrics_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); +SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); + +-- Mock the varnish invalidation function +-- (used by cdb_geocoder tests) +CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$ +BEGIN + RETURN; +END +$$ +LANGUAGE plpgsql; + +-- Set user quota +SELECT cartodb.CDB_SetUserQuotaInBytes(0); diff --git a/server/extension/test/0.4.0/sql/20_street_test.sql b/server/extension/test/0.4.0/sql/20_street_test.sql new file mode 100644 index 0000000..86e0368 --- /dev/null +++ b/server/extension/test/0.4.0/sql/20_street_test.sql @@ -0,0 +1,7 @@ +-- Check for namedplaces signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_street_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, text'); diff --git a/server/extension/test/0.4.0/sql/30_admin0_test.sql b/server/extension/test/0.4.0/sql/30_admin0_test.sql new file mode 100644 index 0000000..3249c60 --- /dev/null +++ b/server/extension/test/0.4.0/sql/30_admin0_test.sql @@ -0,0 +1,32 @@ +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_dataservices_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'Spain'); + +-- Insert some dummy synonym +INSERT INTO admin0_synonyms (name, adm0_a3) VALUES ('Spain', 'ESP'); + +-- Insert some dummy geometry to return +INSERT INTO ne_admin0_v3 (adm0_a3, the_geom) VALUES('ESP', ST_GeomFromText( + 'POLYGON((-71.1031880899493 42.3152774590236, + -71.1031627617667 42.3152960829043, + -71.102923838298 42.3149156848307, + -71.1031880899493 42.3152774590236))',4326) +); + +-- This should return the polygon inserted above +SELECT cdb_dataservices_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'Spain'); + +-- Check for admin0 signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_admin0_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_admin0_polygon' + AND oidvectortypes(p.proargtypes) = 'text'); diff --git a/server/extension/test/0.4.0/sql/40_admin1_test.sql b/server/extension/test/0.4.0/sql/40_admin1_test.sql new file mode 100644 index 0000000..7b3748c --- /dev/null +++ b/server/extension/test/0.4.0/sql/40_admin1_test.sql @@ -0,0 +1,48 @@ +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California'); +SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California', 'United States'); + +-- Insert dummy data into country decoder table +INSERT INTO country_decoder (synonyms, iso3) VALUES (Array['united states'], 'USA'); + +-- Insert some dummy data and geometry to return +INSERT INTO global_province_polygons (synonyms, iso3, the_geom) VALUES (Array['california'], 'USA', ST_GeomFromText( + 'POLYGON((-71.1031880899493 42.3152774590236, + -71.1031627617667 42.3152960829043, + -71.102923838298 42.3149156848307, + -71.1031880899493 42.3152774590236))',4326) +); + +-- This should return the polygon inserted above +SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California'); +SELECT cdb_dataservices_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California', 'United States'); + +-- Check for admin1 signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_admin1_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_admin1_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_admin1_polygon' + AND oidvectortypes(p.proargtypes) = 'text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_admin1_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text'); diff --git a/server/extension/test/0.4.0/sql/50_namedplaces_test.sql b/server/extension/test/0.4.0/sql/50_namedplaces_test.sql new file mode 100644 index 0000000..306a682 --- /dev/null +++ b/server/extension/test/0.4.0/sql/50_namedplaces_test.sql @@ -0,0 +1,72 @@ +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain'); + +-- Insert dummy data into points table +INSERT INTO global_cities_points_limited (geoname_id, name, iso2, admin1, admin2, population, lowername, the_geom) VALUES (3128760, 'Elche', 'ES', 'Valencia', 'AL', 34534, 'elche', ST_GeomFromText( + 'POINT(0.6983 39.26787)',4326) +); + +-- Insert dummy data into alternates table +INSERT INTO global_cities_alternates_limited (geoname_id, name, preferred, lowername, admin1_geonameid, iso2, admin1, the_geom) VALUES (3128760, 'Elx', true, 'elx', '000000', 'ES', 'Valencia', ST_GeomFromText( + 'POINT(0.6983 39.26787)',4326) +); + +-- Insert dummy data into country decoder table +INSERT INTO country_decoder (synonyms, iso2) VALUES (Array['spain'], 'ES'); + +-- Insert dummy data into admin1 decoder table +INSERT INTO admin1_decoder (admin1, synonyms, iso2) VALUES ('Valencia', Array['valencia', 'Valencia'], 'ES'); + +-- This should return the point inserted above +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'Spain'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain'); +SELECT cdb_dataservices_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'valencia', 'Spain'); + +-- Check for namedplaces signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); diff --git a/server/extension/test/0.4.0/sql/60_postalcodes_test.sql b/server/extension/test/0.4.0/sql/60_postalcodes_test.sql new file mode 100644 index 0000000..55d1c85 --- /dev/null +++ b/server/extension/test/0.4.0/sql/60_postalcodes_test.sql @@ -0,0 +1,117 @@ +-- Make sure dbs are clean +DELETE FROM global_postal_code_points; +DELETE FROM global_postal_code_polygons; +DELETE FROM country_decoder; +DELETE FROM available_services; +DELETE FROM admin0_synonyms; + +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204'); + +-- Insert dummy data into ip_address_locations +INSERT INTO global_postal_code_points (the_geom, iso3, postal_code, postal_code_num) VALUES ( + '0101000020E61000000000000000E040408036B47414764840', + 'ESP', + '03204', + 3204 +); + +INSERT INTO global_postal_code_polygons (the_geom, iso3, postal_code, postal_code_num) VALUES ( + '0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040', + 'ESP', + '03204', + 3204 +); + +INSERT INTO country_decoder (iso3, synonyms) VALUES ( + 'ESP', + Array['spain', 'Spain', 'ESP'] +); + +INSERT INTO available_services (adm0_a3, admin0, postal_code_points, postal_code_polygons) VALUES ( + 'ESP', + 't', + 't', + 't' +); + +INSERT INTO admin0_synonyms (adm0_a3, name, name_, rank) VALUES ( + 'ESP', + 'Spain', + 'spain', + 3 +); + +-- This should return the polygon inserted above +SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204'); + +SELECT cdb_dataservices_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204', 'spain'); + +SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', '03204'); + +SELECT cdb_dataservices_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', '03204', 'spain'); + +-- Clean dbs +DELETE FROM global_postal_code_points; +DELETE FROM global_postal_code_polygons; +DELETE FROM country_decoder; +DELETE FROM available_services; +DELETE FROM admin0_synonyms; + +-- Check for namedplaces signatures (point and polygon) +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_postalcode_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_postalcode_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_postalcode_point' + AND oidvectortypes(p.proargtypes) = 'text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_postalcode_point' + AND oidvectortypes(p.proargtypes) = 'text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text'); diff --git a/server/extension/test/0.4.0/sql/70_ips_test.sql b/server/extension/test/0.4.0/sql/70_ips_test.sql new file mode 100644 index 0000000..6111bd0 --- /dev/null +++ b/server/extension/test/0.4.0/sql/70_ips_test.sql @@ -0,0 +1,24 @@ +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_dataservices_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', '0.0.0.0'); + +-- Insert dummy data into ip_address_locations +INSERT INTO ip_address_locations VALUES ('::ffff:0.0.0.0'::inet, (ST_SetSRID(ST_MakePoint('40.40', '3.71'), 4326))); + +-- This should return the polygon inserted above +SELECT cdb_dataservices_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', '0.0.0.0'); + +-- Check for namedplaces signatures (point and polygon) +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_geocode_ipaddress_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = '_cdb_geocode_ipaddress_point' + AND oidvectortypes(p.proargtypes) = 'text'); diff --git a/server/extension/test/0.4.0/sql/85_isodistance_test.sql b/server/extension/test/0.4.0/sql/85_isodistance_test.sql new file mode 100644 index 0000000..f66b68a --- /dev/null +++ b/server/extension/test/0.4.0/sql/85_isodistance_test.sql @@ -0,0 +1,7 @@ +-- Check for isodistance signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_isodistance' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, integer[], text[]'); diff --git a/server/extension/test/0.4.0/sql/90_isochrone_test.sql b/server/extension/test/0.4.0/sql/90_isochrone_test.sql new file mode 100644 index 0000000..113fb28 --- /dev/null +++ b/server/extension/test/0.4.0/sql/90_isochrone_test.sql @@ -0,0 +1,7 @@ +-- Check for isochrone signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_dataservices_server' + AND proname = 'cdb_isochrone' + AND oidvectortypes(p.proargtypes) = 'text, text, geometry, text, integer[], text[]'); diff --git a/server/extension/test/0.4.0/sql/99_remove_geocoder_api_user_test.sql b/server/extension/test/0.4.0/sql/99_remove_geocoder_api_user_test.sql new file mode 100644 index 0000000..ad4fee7 --- /dev/null +++ b/server/extension/test/0.4.0/sql/99_remove_geocoder_api_user_test.sql @@ -0,0 +1,5 @@ +REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_dataservices_server FROM geocoder_api; +REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA public FROM geocoder_api; +REVOKE USAGE ON SCHEMA cdb_dataservices_server FROM geocoder_api; +REVOKE USAGE ON SCHEMA public FROM geocoder_api; +REVOKE SELECT ON ALL TABLES IN SCHEMA public FROM geocoder_api; diff --git a/server/lib/python/cartodb_services/cartodb_services/here/__init__.py b/server/lib/python/cartodb_services/cartodb_services/here/__init__.py index 5d910f3..7c20cc5 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/__init__.py @@ -1 +1,2 @@ from geocoder import HereMapsGeocoder +from routing import HereMapsRoutingIsoline diff --git a/server/lib/python/cartodb_services/cartodb_services/here/exceptions.py b/server/lib/python/cartodb_services/cartodb_services/here/exceptions.py index f4a3015..76fa5ac 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/exceptions.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/exceptions.py @@ -11,6 +11,14 @@ class BadGeocodingParams(Exception): return repr('Bad geocoding params: ' + json.dumps(self.value)) +class WrongParams(Exception): + def __init__(self, value): + self.value = value + + def __str__(self): + return repr('Wrong parameters passed: ' + json.dumps(self.value)) + + class NoGeocodingParams(Exception): def __str__(self): return repr('No params for geocoding specified') diff --git a/server/lib/python/cartodb_services/cartodb_services/here/routing.py b/server/lib/python/cartodb_services/cartodb_services/here/routing.py new file mode 100644 index 0000000..53fe101 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/here/routing.py @@ -0,0 +1,119 @@ +import requests +import json + +from exceptions import WrongParams + + +class HereMapsRoutingIsoline: + 'A Here Maps Routing wrapper for python' + + PRODUCTION_ROUTING_BASE_URL = 'https://isoline.route.api.here.com' + STAGING_ROUTING_BASE_URL = 'https://isoline.route.cit.api.here.com' + ISOLINE_PATH = '/routing/7.2/calculateisoline.json' + + ACCEPTED_MODES = { + "walk": "pedestrian", + "car": "car" + } + + OPTIONAL_PARAMS = [ + 'departure', + 'arrival', + 'singlecomponent', + 'resolution', + 'maxpoints', + 'quality' + ] + + def __init__(self, app_id, app_code, base_url=PRODUCTION_ROUTING_BASE_URL): + self._app_id = app_id + self._app_code = app_code + self._url = "{0}{1}".format(base_url, self.ISOLINE_PATH) + + def calculate_isodistance(self, source, mode, data_range, options=[]): + return self.__calculate_isolines(source, mode, data_range, 'distance', + options) + + def calculate_isochrone(self, source, mode, data_range, options=[]): + return self.__calculate_isolines(source, mode, data_range, 'time', + options) + + def __calculate_isolines(self, source, mode, data_range, range_type, + options=[]): + parsed_options = self.__parse_options(options) + source_param = self.__parse_source_param(source, parsed_options) + mode_param = self.__parse_mode_param(mode, parsed_options) + request_params = self.__parse_request_parameters(source_param, + mode_param, + data_range, + range_type, + parsed_options) + response = requests.get(self._url, params=request_params) + if response.status_code == requests.codes.ok: + return self.__parse_isolines_response(response.text) + else: + response.raise_for_status() + + def __parse_options(self, options): + return dict(option.split('=') for option in options) + + def __parse_request_parameters(self, source, mode, data_range, range_type, + options): + filtered_options = {k: v for k, v in options.iteritems() + if k.lower() in self.OPTIONAL_PARAMS} + filtered_options.update(source) + filtered_options.update(mode) + filtered_options.update({'range': ",".join(map(str, data_range))}) + filtered_options.update({'rangetype': range_type}) + filtered_options.update({'app_id': self._app_id}) + filtered_options.update({'app_code': self._app_code}) + + return filtered_options + + def __parse_isolines_response(self, response): + parsed_response = json.loads(response) + isolines_response = parsed_response['response']['isoline'] + isolines = [] + for isoline in isolines_response: + isolines.append({'range': isoline['range'], + 'geom': isoline['component'][0]['shape']}) + + return isolines + + def __parse_source_param(self, source, options): + key = 'start' + if 'is_destination' in options and options['is_destination']: + key = 'destination' + + return {key: source} + + def __parse_mode_param(self, mode, options): + if mode in self.ACCEPTED_MODES: + mode_source = self.ACCEPTED_MODES[mode] + else: + raise WrongParams("{0} is not an accepted mode type".format(mode)) + + if 'mode_type' in options: + mode_type = options['mode_type'] + else: + mode_type = 'shortest' + + if 'mode_traffic' in options: + mode_traffic = "traffic:{0}".format(options['mode_traffic']) + else: + mode_traffic = None + + if 'mode_feature' in options and 'mode_feature_weight' in options: + mode_feature = "{0}:{1}".format(options['mode_feature'], + options['mode_feature_weight']) + else: + mode_feature = None + + mode_param = "{0};{1}".format(mode_type, mode_source) + if mode_traffic: + mode_param = "{0};{1}".format(mode_param, mode_traffic) + + if mode_feature: + mode_param = "{0};{1}".format(mode_param, mode_feature) + + return {'mode': mode_param} diff --git a/server/lib/python/cartodb_services/cartodb_services/here/types.py b/server/lib/python/cartodb_services/cartodb_services/here/types.py new file mode 100644 index 0000000..90e1d9e --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/here/types.py @@ -0,0 +1,16 @@ +# Helper to deal with type conversion between HERE and PostGIS +import plpy + + +def geo_polyline_to_multipolygon(polyline): + """Convert a HERE polyline shape to a PostGIS multipolygon""" + coordinates = [] + for point in polyline: + lat, lon = point.split(',') + coordinates.append("%s %s" % (lon, lat)) + wkt_coordinates = ','.join(coordinates) + + sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates) + geometry = plpy.execute(sql, 1)[0]['geom'] + + return geometry diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py b/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py index 4ae8535..51147f8 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py @@ -1,3 +1,3 @@ -from config import GeocoderConfig, InternalGeocoderConfig, ConfigException +from config import GeocoderConfig, RoutingConfig, InternalGeocoderConfig, ConfigException from quota import QuotaService from user import UserMetricsService diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index 2d3ccc3..2b8c51b 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -28,6 +28,28 @@ class ServiceConfig(object): return self._orgname +class RoutingConfig(ServiceConfig): + + def __init__(self, redis_connection, username, orgname=None, + heremaps_app_id=None, heremaps_app_code=None): + super(RoutingConfig, self).__init__(redis_connection, + username, orgname) + self._heremaps_app_id = heremaps_app_id + self._heremaps_app_code = heremaps_app_code + + @property + def service_type(self): + return 'routing_here' + + @property + def heremaps_app_id(self): + return self._heremaps_app_id + + @property + def heremaps_app_code(self): + return self._heremaps_app_code + + class InternalGeocoderConfig(ServiceConfig): def __init__(self, redis_connection, username, orgname=None): diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py index 9a373cf..340642c 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py @@ -28,6 +28,12 @@ class QuotaService: else: return False + # TODO + # We are going to change this class to be the generic one and + # create specific for routing and geocoding services but because + # this implies change all the extension functions, we are going to + # make the change in a minor release + def increment_success_geocoder_use(self, amount=1): self._user_service.increment_service_use( self._user_geocoder_config.service_type, "success_responses", @@ -47,3 +53,8 @@ class QuotaService: self._user_service.increment_service_use( self._user_geocoder_config.service_type, "total_requests", amount=amount) + + def increment_isolines_service_use(self, amount=1): + self._user_service.increment_service_use( + self._user_geocoder_config.service_type, "isolines_generated", + amount=amount) diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index 3d2448e..3774847 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.1.0', + version='0.2.0', description='CartoDB Services API Python Library', diff --git a/server/lib/python/cartodb_services/test/test_heremapsgeocoder.py b/server/lib/python/cartodb_services/test/test_heremapsgeocoder.py index fc3a0d8..c6cd790 100644 --- a/server/lib/python/cartodb_services/test/test_heremapsgeocoder.py +++ b/server/lib/python/cartodb_services/test/test_heremapsgeocoder.py @@ -23,68 +23,81 @@ class HereMapsGeocoderTestCase(unittest.TestCase): } }""" - GOOD_RESPONSE = """{ - "Response": { - "MetaInfo": { - "Timestamp": "2015-11-04T16:30:32.187+0000" - }, - "View": [{ - "_type": "SearchResultsViewType", - "ViewId": 0, - "Result": { - "Relevance": 0.89, - "MatchLevel": "street", - "MatchQuality": { - "City": 1.0, - "Street": [1.0] - }, - "Location": { - "LocationId": "NT_yyKB4r3mCWAX4voWgxPcuA", - "LocationType": "address", - "DisplayPosition": { - "Latitude": 40.43433, - "Longitude": -3.70126 - }, - "NavigationPosition": [{ - "Latitude": 40.43433, - "Longitude": -3.70126 - }], - "MapView": { - "TopLeft": { - "Latitude": 40.43493, - "Longitude": -3.70404 - }, - "BottomRight": { - "Latitude": 40.43373, - "Longitude": -3.69873 - } - }, - "Address": { - "Label": "Calle de Eloy Gonzalo, Madrid, Espana", - "Country": "ESP", - "State": "Comunidad de Madrid", - "County": "Madrid", - "City": "Madrid", - "District": "Trafalgar", - "Street": "Calle de Eloy Gonzalo", - "AdditionalData": [{ - "value": "Espana", - "key": "CountryName" - }, - { - "value": "Comunidad de Madrid", - "key": "StateName" - }, - { - "value": "Madrid", - "key": "CountyName" - }] - } + GOOD_RESPONSE = unicode("""{ + "Response": { + "MetaInfo": { + "Timestamp": "2016-02-10T14:17:33.792+0000", + "NextPageInformation": "2" + }, + "View": [ + { + "_type": "SearchResultsViewType", + "ViewId": 0, + "Result": [ + { + "Relevance": 1, + "MatchLevel": "houseNumber", + "MatchQuality": { + "Street": [ + 1 + ], + "HouseNumber": 1 + }, + "MatchType": "pointAddress", + "Location": { + "LocationId": "NT_CKopMSB9JnBYAO11CMOrxB_zUD", + "LocationType": "address", + "DisplayPosition": { + "Latitude": 37.70246, + "Longitude": -5.2794 + }, + "NavigationPosition": [ + { + "Latitude": 37.7024199, + "Longitude": -5.27939 } + ], + "MapView": { + "TopLeft": { + "Latitude": 37.7035842, + "Longitude": -5.2808208 + }, + "BottomRight": { + "Latitude": 37.7013358, + "Longitude": -5.2779792 + } + }, + "Address": { + "Label": "Calle Amor de Dios, 35, 14700 Palma del Río (Córdoba), España", + "Country": "ESP", + "State": "Andalucía", + "County": "Córdoba", + "City": "Palma del Río", + "Street": "Calle Amor de Dios", + "HouseNumber": "35", + "PostalCode": "14700", + "AdditionalData": [ + { + "value": "España", + "key": "CountryName" + }, + { + "value": "Andalucía", + "key": "StateName" + }, + { + "value": "Córdoba", + "key": "CountyName" + } + ] + } } - }] - } - }""" + } + ] + } + ] + } + }""", 'utf-8') MALFORMED_RESPONSE = """{"manolo": "escobar"}""" @@ -95,12 +108,12 @@ class HereMapsGeocoderTestCase(unittest.TestCase): req_mock.register_uri('GET', HereMapsGeocoder.PRODUCTION_GEOCODE_JSON_URL, text=self.GOOD_RESPONSE) response = self.geocoder.geocode( - searchtext='Calle Eloy Gonzalo 27', - city='Madrid', + searchtext='Calle amor de dios', + city='Cordoba', country='España') - self.assertEqual(response[0], -3.70126) - self.assertEqual(response[1], 40.43433) + self.assertEqual(response[0], -5.2794) + self.assertEqual(response[1], 37.70246) def test_geocode_address_with_invalid_params(self, req_mock): req_mock.register_uri('GET', HereMapsGeocoder.PRODUCTION_GEOCODE_JSON_URL, @@ -127,6 +140,6 @@ class HereMapsGeocoderTestCase(unittest.TestCase): text=self.MALFORMED_RESPONSE) with self.assertRaises(MalformedResult): self.geocoder.geocode( - searchtext='Calle Eloy Gonzalo 27', - city='Madrid', + searchtext='Calle amor de dios', + city='Cordoba', country='España') diff --git a/server/lib/python/cartodb_services/test/test_heremapsrouting.py b/server/lib/python/cartodb_services/test/test_heremapsrouting.py new file mode 100644 index 0000000..1d27f30 --- /dev/null +++ b/server/lib/python/cartodb_services/test/test_heremapsrouting.py @@ -0,0 +1,214 @@ +#!/usr/local/bin/python +# -*- coding: utf-8 -*- + +import unittest +import requests_mock +from urlparse import urlparse, parse_qs + +from cartodb_services.here import HereMapsRoutingIsoline +from cartodb_services.here.exceptions import BadGeocodingParams +from cartodb_services.here.exceptions import NoGeocodingParams +from cartodb_services.here.exceptions import MalformedResult + +requests_mock.Mocker.TEST_PREFIX = 'test_' + + +@requests_mock.Mocker() +class HereMapsRoutingIsolineTestCase(unittest.TestCase): + EMPTY_RESPONSE = """{ + "response": { + "metaInfo": { + "timestamp": "2016-02-10T10:42:21Z", + "mapVersion": "8.30.61.107", + "moduleVersion": "7.2.65.0-1222", + "interfaceVersion": "2.6.20" + }, + "center": { + "latitude": 33, + "longitude": 0.9999999 + }, + "isoline": [ + { + "range": 1000, + "component": [ + { + "id": 0, + "shape": [] + } + ] + } + ], + "start": { + "linkId": "+1025046831", + "mappedPosition": { + "latitude": 32.968725, + "longitude": 0.9993629 + }, + "originalPosition": { + "latitude": 33, + "longitude": 0.9999999 + } + } + } + }""" + + ERROR_RESPONSE = """{ + "_type": "ns2:RoutingServiceErrorType", + "type": "ApplicationError", + "subtype": "InitIsolineSearchFailed", + "details": "Error is NGEO_ERROR_UNKNOWN", + "additionalData": [ + { + "key": "error_code", + "value": "NGEO_ERROR_UNKNOWN" + } + ], + "metaInfo": { + "timestamp": "2016-02-10T10:39:35Z", + "mapVersion": "8.30.61.107", + "moduleVersion": "7.2.65.0-1222", + "interfaceVersion": "2.6.20" + } + }""" + + GOOD_RESPONSE = """{ + "response": { + "metaInfo": { + "timestamp": "2016-02-10T10:42:21Z", + "mapVersion": "8.30.61.107", + "moduleVersion": "7.2.65.0-1222", + "interfaceVersion": "2.6.20" + }, + "center": { + "latitude": 33, + "longitude": 0.9999999 + }, + "isoline": [ + { + "range": 1000, + "component": [ + { + "id": 0, + "shape": [ + "32.9699707,0.9462833", + "32.9699707,0.9458542", + "32.9699707,0.9462833" + ] + } + ] + }, { + "range": 2000, + "component": [ + { + "id": 0, + "shape": [ + "32.9699707,0.9462833", + "32.9699707,0.9750366", + "32.9699707,0.9462833" + ] + } + ] + } + ], + "start": { + "linkId": "+1025046831", + "mappedPosition": { + "latitude": 32.968725, + "longitude": 0.9993629 + }, + "originalPosition": { + "latitude": 33, + "longitude": 0.9999999 + } + } + } + }""" + + MALFORMED_RESPONSE = """{"manolo": "escobar"}""" + + def setUp(self): + self.routing = HereMapsRoutingIsoline(None, None) + self.isoline_url = "{0}{1}".format(HereMapsRoutingIsoline.PRODUCTION_ROUTING_BASE_URL, + HereMapsRoutingIsoline.ISOLINE_PATH) + + def test_calculate_isodistance_with_valid_params(self, req_mock): + print self.isoline_url + url = "{0}?start=geo%2133.0%2C1.0&mode=shortest%3Bcar".format(self.isoline_url) + req_mock.register_uri('GET', url, text=self.GOOD_RESPONSE) + response = self.routing.calculate_isodistance('geo!33.0,1.0', 'car', + ['1000', '2000']) + self.assertEqual(len(response), 2) + self.assertEqual(response[0]['range'], 1000) + self.assertEqual(response[1]['range'], 2000) + self.assertEqual(response[0]['geom'], [u'32.9699707,0.9462833', + u'32.9699707,0.9458542', + u'32.9699707,0.9462833']) + self.assertEqual(response[1]['geom'], [u'32.9699707,0.9462833', + u'32.9699707,0.9750366', + u'32.9699707,0.9462833']) + + def test_calculate_isochrone_with_valid_params(self, req_mock): + print self.isoline_url + url = "{0}?start=geo%2133.0%2C1.0&mode=shortest%3Bcar".format(self.isoline_url) + req_mock.register_uri('GET', url, text=self.GOOD_RESPONSE) + response = self.routing.calculate_isochrone('geo!33.0,1.0', 'car', + ['1000', '2000']) + self.assertEqual(len(response), 2) + self.assertEqual(response[0]['range'], 1000) + self.assertEqual(response[1]['range'], 2000) + self.assertEqual(response[0]['geom'], [u'32.9699707,0.9462833', + u'32.9699707,0.9458542', + u'32.9699707,0.9462833']) + self.assertEqual(response[1]['geom'], [u'32.9699707,0.9462833', + u'32.9699707,0.9750366', + u'32.9699707,0.9462833']) + + def test_calculate_isolines_empty_response(self, req_mock): + url = "{0}?start=geo%2133.0%2C1.0&mode=shortest%3Bcar".format( + self.isoline_url) + req_mock.register_uri('GET', url, text=self.EMPTY_RESPONSE) + response = self.routing.calculate_isochrone('geo!33.0,1.0', 'car', + ['1000', '2000']) + self.assertEqual(len(response), 1) + self.assertEqual(response[0]['range'], 1000) + self.assertEqual(response[0]['geom'], []) + + def test_non_listed_parameters_filter_works_properly(self, req_mock): + url = "{0}?start=geo%2133.0%2C1.0&mode=shortest%3Bcar".format( + self.isoline_url) + req_mock.register_uri('GET', url, text=self.GOOD_RESPONSE) + response = self.routing.calculate_isochrone('geo!33.0,1.0', 'car', + ['1000', '2000'], + ['singlecomponent=true', + 'resolution=3', + 'maxpoints=1000', + 'quality=2', + 'false_option=true']) + parsed_url = urlparse(req_mock.request_history[0].url) + url_params = parse_qs(parsed_url.query) + self.assertEqual(len(url_params), 8) + self.assertEqual('false_option' in url_params, False) + + def test_mode_parameters_works_properly(self, req_mock): + req_mock.register_uri('GET', requests_mock.ANY, + text=self.GOOD_RESPONSE) + response = self.routing.calculate_isochrone('geo!33.0,1.0', 'car', + ['1000', '2000'], + ['mode_type=fastest', + 'mode_feature=motorway', + 'mode_feature_weight=-1', + 'mode_traffic=false']) + parsed_url = urlparse(req_mock.request_history[0].url) + url_params = parse_qs(parsed_url.query) + self.assertEqual(url_params['mode'][0], + 'fastest;car;traffic:false;motorway:-1') + + def test_source_parameters_works_properly(self, req_mock): + req_mock.register_uri('GET', requests_mock.ANY, + text=self.GOOD_RESPONSE) + response = self.routing.calculate_isochrone('geo!33.0,1.0', 'car', + ['1000', '2000'], + ['is_destination=true']) + parsed_url = urlparse(req_mock.request_history[0].url) + url_params = parse_qs(parsed_url.query) + self.assertEqual(url_params['destination'][0], 'geo!33.0,1.0')