Compare commits

...

42 Commits

Author SHA1 Message Date
Mario de Frutos
c768f6c44c Merge pull request #81 from CartoDB/fix_imports_errors
Fixed some python import calls due the refactor made
2016-02-04 20:05:46 +01:00
Mario de Frutos
1665cf8069 Fixed some python import calls due the refactor made 2016-02-04 20:04:46 +01:00
Mario de Frutos
17ca4317f0 Merge pull request #80 from CartoDB/store_master_sql_file_0_2_0
Commit cdb_geocoder_server--0.2.0.sql into VC
2016-02-04 16:56:38 +01:00
Mario de Frutos
43b09da396 Commit cdb_geocoder_server--0.2.0.sql into VC 2016-02-04 16:30:59 +01:00
Mario de Frutos
db715d6d54 Added security definer permission to the sql part too 2016-02-04 15:42:22 +01:00
Mario de Frutos
6676b26250 Hotfix: Added to the downgrade file too 2016-02-04 11:48:24 +01:00
Mario de Frutos
7a0b927acd Hotfix: Other method that need security definer 2016-02-04 11:03:38 +01:00
Mario de Frutos
8176eee356 Hotfix: Permission problems without security definer 2016-02-04 10:57:54 +01:00
Mario de Frutos
d1f61dadd4 Merge pull request #70 from CartoDB/google_geocoder
Google geocoder
2016-02-04 10:39:07 +01:00
Mario de Frutos
1367c46957 Added automatic generated files warning header 2016-02-04 10:38:12 +01:00
Mario de Frutos
b1a703d696 Fix some integration tests problems to identify the real reason in the tests dataset import failure 2016-02-03 15:46:05 +01:00
Mario de Frutos
3960c13484 Refactor python library to unify and rename as cdb_services 2016-02-03 10:21:57 +01:00
Mario de Frutos
e7c58b9a51 Google geocoder working 2016-02-03 10:21:34 +01:00
Mario de Frutos
731eb8c74f Merge pull request #77 from CartoDB/store_master_sql_file
Commit cdb_geocoder_server--0.1.0.sql into VC
2016-02-03 10:19:56 +01:00
Mario de Frutos
a5d6650e2f Freeze the 0.1.0 complete version of server extension 2016-02-03 10:17:52 +01:00
Mario de Frutos
440dadb5e8 Commit cdb_geocoder_server--0.1.0.sql into VC
We need to store this files because they are used in the execution
of the make file and if one of this doesn't exists the make
execution is going to crash like this:

make: *** No rule to make target `cdb_geocoder_server--0.1.0.sql',
needed by `all'.  Stop.
2016-02-03 10:16:31 +01:00
Carlos Matallín
d34a6f2297 Merge pull request #76 from CartoDB/matallo-patch-1
Update reference.md
2016-02-02 13:06:57 +01:00
Carlos Matallín
75c19c1c1b Update reference.md 2016-02-02 12:45:21 +01:00
Carlos Matallín
69a0665dd0 Update reference.md 2016-02-02 12:43:57 +01:00
Rafa de la Torre
16bd866531 Merge pull request #74 from CartoDB/doc-small-fix
A small fix: let's have street adresses in the main page
2016-01-29 16:36:37 +01:00
Rafa de la Torre
95a68f68d6 A small fix: let's have street adresses in the main page 2016-01-29 15:34:20 +01:00
Mario de Frutos
626d2bb1f4 Merge pull request #72 from CartoDB/matallo-patch-1
Update reference.md
2016-01-29 14:32:59 +01:00
csobier
4f68d28ee0 Merge pull request #73 from CartoDB/bullet-formatting
fixed bullet formatting
2016-01-29 08:24:29 -05:00
csobier
3f0ccefa41 fixed bullet formatting 2016-01-29 08:23:08 -05:00
Carlos Matallín
8da211e77d Update reference.md 2016-01-29 12:49:48 +01:00
Rafa de la Torre
56c8d26c87 Merge pull request #71 from CartoDB/matallo-patch-1
Update reference.md
2016-01-29 11:47:25 +01:00
Carlos Matallín
5b729db139 Update reference.md
updated examples for consistency

CR @csobier @rafatower
2016-01-29 11:43:16 +01:00
Rafa de la Torre
8585d8ba51 Merge pull request #69 from CartoDB/geocode_street_doc
Add the geocode street function doc
2016-01-28 19:18:28 +01:00
csobier
23310dc024 Just applied some minor grammar edits 2016-01-28 12:33:17 -05:00
Rafa de la Torre
179c3fea35 Add street number to example queries 2016-01-28 18:25:38 +01:00
Rafa de la Torre
a147e76860 Small style fix (too much bold text) 2016-01-28 18:18:07 +01:00
Rafa de la Torre
a7e83dab49 Add some corrections to the documentation 2016-01-28 18:16:10 +01:00
Mario de Frutos
a1d6f79999 Add the geocode street function doc 2016-01-27 19:27:34 +01:00
Mario de Frutos
e73736cbec Restrict the python-dateutil library to avoid conflict with other projects that need that specific version 2016-01-26 19:57:18 +01:00
Mario de Frutos
a00c2eac32 Merge pull request #68 from CartoDB/change_street_signature
Change the signature name for the geocoder street point function
2016-01-26 16:50:52 +01:00
Mario de Frutos
bb629dfe4b Change the signature name for the geocoder street point function 2016-01-26 16:22:28 +01:00
Mario de Frutos
b417e8f1fc Merge pull request #67 from CartoDB/add_needed_security_changes_for_v2
Add security definer to connect redis function too
2016-01-26 13:16:37 +01:00
Mario de Frutos
343bcf539a Add security definer to connect redis function too 2016-01-26 13:15:15 +01:00
Mario de Frutos
b4b1cd9592 Merge pull request #65 from CartoDB/add_needed_security_changes_for_v2
Add needed security changes for v2
2016-01-26 13:06:04 +01:00
Mario de Frutos
2409d548b6 Grants to be applied again to give permissions to the new functions 2016-01-26 12:55:12 +01:00
Mario de Frutos
35b59c448f Add needed security definer to config function 2016-01-26 12:55:06 +01:00
Mario de Frutos
5309a6eae5 Fixed geocode street point integration tests 2016-01-25 18:48:27 +01:00
84 changed files with 3549 additions and 366 deletions

View File

@@ -31,7 +31,7 @@ BEGIN
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_street_point_v2 (searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_street_point (searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
@@ -46,15 +46,15 @@ BEGIN
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country) INTO ret;
SELECT cdb_geocoder_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret;
RETURN ret;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_street_point_v2 (username text, organization_name text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_street_point (username text, organization_name text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
RETURNS Geometry AS $$
CONNECT cdb_geocoder_client._server_conn_str();
SELECT cdb_geocoder_server.cdb_geocode_street_point_v2 (username, organization_name, searchtext, city, state_province, country);
SELECT cdb_geocoder_server.cdb_geocode_street_point (username, organization_name, searchtext, city, state_province, country);
$$ LANGUAGE plproxy;
GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_street_point_v2(searchtext text, city text, state_province text, country text) TO publicuser;
GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser;

View File

@@ -1,5 +1,8 @@
DROP FUNCTION IF EXISTS cdb_geocoder_client.cdb_geocode_street_point (text, text, text, text);
DROP FUNCTION IF EXISTS cdb_geocoder_client._cdb_geocode_street_point (text, text, text, text);
-- This functions could exists due a first deploy with this name
DROP FUNCTION IF EXISTS cdb_geocoder_client.cdb_geocode_street_point_v2 (text, text, text, text);
DROP FUNCTION IF EXISTS cdb_geocoder_client._cdb_geocode_street_point_v2 (text, text, text, text);
DROP FUNCTION IF EXISTS cdb_geocoder_client._cdb_geocode_street_point_v2 (text, text, text, text, text, text);
--
-- Get entity config function
@@ -32,4 +35,4 @@ BEGIN
result.organization_name = organization_name;
RETURN result;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;

View File

@@ -1,2 +1,3 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "CREATE EXTENSION cdb_geocoder_client" to load this file. \quit

View File

@@ -1,2 +1,3 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "CREATE EXTENSION cdb_geocoder_client" to load this file. \quit

View File

@@ -0,0 +1,101 @@
-- Add to the search path the schema
SET search_path TO public,cartodb,cdb_geocoder_client;
-- Mock the server functions
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
RETURNS Geometry AS $$
BEGIN
RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_geocoder_street_point invoked with params (%, %, %, %, %, %)', username, orgname, searchtext, city, state_province, country;
RETURN NULL;
END;
$$ LANGUAGE 'plpgsql';
-- Exercise the public and the proxied function
SELECT cdb_geocode_street_point('One street, 1');
NOTICE: cdb_geocoder_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point invoked with params (test_user, <NULL>, One street, 1, <NULL>, <NULL>, <NULL>)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country)"
PL/pgSQL function cdb_geocode_street_point(text,text,text,text) line 15 at SQL statement
cdb_geocode_street_point
--------------------------
(1 row)
SELECT cdb_geocode_street_point('One street', 'city');
NOTICE: cdb_geocoder_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point invoked with params (test_user, <NULL>, One street, city, <NULL>, <NULL>)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country)"
PL/pgSQL function cdb_geocode_street_point(text,text,text,text) line 15 at SQL statement
cdb_geocode_street_point
--------------------------
(1 row)
SELECT cdb_geocode_street_point('One street', 'city', 'state');
NOTICE: cdb_geocoder_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point invoked with params (test_user, <NULL>, One street, city, state, <NULL>)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country)"
PL/pgSQL function cdb_geocode_street_point(text,text,text,text) line 15 at SQL statement
cdb_geocode_street_point
--------------------------
(1 row)
SELECT cdb_geocode_street_point('One street', 'city', 'state', 'country');
NOTICE: cdb_geocoder_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point invoked with params (test_user, <NULL>, One street, city, state, country)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country)"
PL/pgSQL function cdb_geocode_street_point(text,text,text,text) line 15 at SQL statement
cdb_geocode_street_point
--------------------------
(1 row)
SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country');
NOTICE: cdb_geocoder_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point invoked with params (test_user, <NULL>, One street, city, <NULL>, country)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country)"
PL/pgSQL function cdb_geocode_street_point(text,text,text,text) line 15 at SQL statement
cdb_geocode_street_point
--------------------------
(1 row)
SELECT cdb_geocode_street_point('One street, 1');
NOTICE: cdb_geocoder_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point invoked with params (test_user, <NULL>, One street, 1, <NULL>, <NULL>, <NULL>)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country)"
PL/pgSQL function cdb_geocode_street_point(text,text,text,text) line 15 at SQL statement
cdb_geocode_street_point
--------------------------
(1 row)
SELECT cdb_geocode_street_point('One street', 'city');
NOTICE: cdb_geocoder_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point invoked with params (test_user, <NULL>, One street, city, <NULL>, <NULL>)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country)"
PL/pgSQL function cdb_geocode_street_point(text,text,text,text) line 15 at SQL statement
cdb_geocode_street_point
--------------------------
(1 row)
SELECT cdb_geocode_street_point('One street', 'city', 'state');
NOTICE: cdb_geocoder_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point invoked with params (test_user, <NULL>, One street, city, state, <NULL>)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country)"
PL/pgSQL function cdb_geocode_street_point(text,text,text,text) line 15 at SQL statement
cdb_geocode_street_point
--------------------------
(1 row)
SELECT cdb_geocode_street_point('One street', 'city', 'state', 'country');
NOTICE: cdb_geocoder_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point invoked with params (test_user, <NULL>, One street, city, state, country)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country)"
PL/pgSQL function cdb_geocode_street_point(text,text,text,text) line 15 at SQL statement
cdb_geocode_street_point
--------------------------
(1 row)
SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country');
NOTICE: cdb_geocoder_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point invoked with params (test_user, <NULL>, One street, city, <NULL>, country)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country)"
PL/pgSQL function cdb_geocode_street_point(text,text,text,text) line 15 at SQL statement
cdb_geocode_street_point
--------------------------
(1 row)

View File

@@ -1,101 +0,0 @@
-- Add to the search path the schema
SET search_path TO public,cartodb,cdb_geocoder_client;
-- Mock the server functions
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_street_point_v2 (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
RETURNS Geometry AS $$
BEGIN
RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (%, %, %, %, %, %)', username, orgname, searchtext, city, state_province, country;
RETURN NULL;
END;
$$ LANGUAGE 'plpgsql';
-- Exercise the public and the proxied function
SELECT cdb_geocode_street_point_v2('One street, 1');
NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, <NULL>, One street, 1, <NULL>, <NULL>, <NULL>)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)"
PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement
cdb_geocode_street_point_v2
-----------------------------
(1 row)
SELECT cdb_geocode_street_point_v2('One street', 'city');
NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, <NULL>, One street, city, <NULL>, <NULL>)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)"
PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement
cdb_geocode_street_point_v2
-----------------------------
(1 row)
SELECT cdb_geocode_street_point_v2('One street', 'city', 'state');
NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, <NULL>, One street, city, state, <NULL>)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)"
PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement
cdb_geocode_street_point_v2
-----------------------------
(1 row)
SELECT cdb_geocode_street_point_v2('One street', 'city', 'state', 'country');
NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, <NULL>, One street, city, state, country)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)"
PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement
cdb_geocode_street_point_v2
-----------------------------
(1 row)
SELECT cdb_geocode_street_point_v2('One street', 'city', NULL, 'country');
NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, <NULL>, One street, city, <NULL>, country)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)"
PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement
cdb_geocode_street_point_v2
-----------------------------
(1 row)
SELECT cdb_geocode_street_point_v2('One street, 1');
NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, <NULL>, One street, 1, <NULL>, <NULL>, <NULL>)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)"
PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement
cdb_geocode_street_point_v2
-----------------------------
(1 row)
SELECT cdb_geocode_street_point_v2('One street', 'city');
NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, <NULL>, One street, city, <NULL>, <NULL>)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)"
PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement
cdb_geocode_street_point_v2
-----------------------------
(1 row)
SELECT cdb_geocode_street_point_v2('One street', 'city', 'state');
NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, <NULL>, One street, city, state, <NULL>)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)"
PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement
cdb_geocode_street_point_v2
-----------------------------
(1 row)
SELECT cdb_geocode_street_point_v2('One street', 'city', 'state', 'country');
NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, <NULL>, One street, city, state, country)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)"
PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement
cdb_geocode_street_point_v2
-----------------------------
(1 row)
SELECT cdb_geocode_street_point_v2('One street', 'city', NULL, 'country');
NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, <NULL>, One street, city, <NULL>, country)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)"
PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement
cdb_geocode_street_point_v2
-----------------------------
(1 row)

View File

@@ -85,12 +85,12 @@ PL/pgSQL function cdb_geocode_ipaddress_point(text) line 15 at SQL statement
(1 row)
SELECT cdb_geocode_street_point_v2('one street, 1');
NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, <NULL>, one street, 1, <NULL>, <NULL>, <NULL>)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)"
PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement
cdb_geocode_street_point_v2
-----------------------------
SELECT cdb_geocode_street_point('one street, 1');
NOTICE: cdb_geocoder_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point invoked with params (test_user, <NULL>, one street, 1, <NULL>, <NULL>, <NULL>)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point(username, orgname, searchtext, city, state_province, country)"
PL/pgSQL function cdb_geocode_street_point(text,text,text,text) line 15 at SQL statement
cdb_geocode_street_point
--------------------------
(1 row)
@@ -113,5 +113,5 @@ SELECT _cdb_geocode_postalcode_point('evil_user', 'evil_orgname', '66666', 'Hell
ERROR: permission denied for function _cdb_geocode_postalcode_point
SELECT _cdb_geocode_ipaddress_point('evil_user', 'evil_orgname', '8.8.8.8');
ERROR: permission denied for function _cdb_geocode_ipaddress_point
SELECT _cdb_geocode_street_point_v2('evil_user', 'evil_orgname', 'one street, 1');
ERROR: permission denied for function _cdb_geocode_street_point_v2
SELECT _cdb_geocode_street_point('evil_user', 'evil_orgname', 'one street, 1');
ERROR: permission denied for function _cdb_geocode_street_point

View File

@@ -0,0 +1,24 @@
-- Add to the search path the schema
SET search_path TO public,cartodb,cdb_geocoder_client;
-- Mock the server functions
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
RETURNS Geometry AS $$
BEGIN
RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_geocoder_street_point invoked with params (%, %, %, %, %, %)', username, orgname, searchtext, city, state_province, country;
RETURN NULL;
END;
$$ LANGUAGE 'plpgsql';
-- Exercise the public and the proxied function
SELECT cdb_geocode_street_point('One street, 1');
SELECT cdb_geocode_street_point('One street', 'city');
SELECT cdb_geocode_street_point('One street', 'city', 'state');
SELECT cdb_geocode_street_point('One street', 'city', 'state', 'country');
SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country');
SELECT cdb_geocode_street_point('One street, 1');
SELECT cdb_geocode_street_point('One street', 'city');
SELECT cdb_geocode_street_point('One street', 'city', 'state');
SELECT cdb_geocode_street_point('One street', 'city', 'state', 'country');
SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country');

View File

@@ -1,24 +0,0 @@
-- Add to the search path the schema
SET search_path TO public,cartodb,cdb_geocoder_client;
-- Mock the server functions
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_street_point_v2 (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
RETURNS Geometry AS $$
BEGIN
RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (%, %, %, %, %, %)', username, orgname, searchtext, city, state_province, country;
RETURN NULL;
END;
$$ LANGUAGE 'plpgsql';
-- Exercise the public and the proxied function
SELECT cdb_geocode_street_point_v2('One street, 1');
SELECT cdb_geocode_street_point_v2('One street', 'city');
SELECT cdb_geocode_street_point_v2('One street', 'city', 'state');
SELECT cdb_geocode_street_point_v2('One street', 'city', 'state', 'country');
SELECT cdb_geocode_street_point_v2('One street', 'city', NULL, 'country');
SELECT cdb_geocode_street_point_v2('One street, 1');
SELECT cdb_geocode_street_point_v2('One street', 'city');
SELECT cdb_geocode_street_point_v2('One street', 'city', 'state');
SELECT cdb_geocode_street_point_v2('One street', 'city', 'state', 'country');
SELECT cdb_geocode_street_point_v2('One street', 'city', NULL, 'country');

View File

@@ -15,7 +15,7 @@ SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain');
SELECT cdb_geocode_postalcode_polygon('03204', 'Spain');
SELECT cdb_geocode_postalcode_point('03204', 'Spain');
SELECT cdb_geocode_ipaddress_point('8.8.8.8');
SELECT cdb_geocode_street_point_v2('one street, 1');
SELECT cdb_geocode_street_point('one street, 1');
-- Check the regular user has no permissions on private functions
SELECT _cdb_geocode_admin0_polygon('evil_user', 'evil_orgname', 'Hell');
@@ -27,4 +27,4 @@ SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol', 'Hell
SELECT _cdb_geocode_postalcode_polygon('evil_user', 'evil_orgname', '66666', 'Hell');
SELECT _cdb_geocode_postalcode_point('evil_user', 'evil_orgname', '66666', 'Hell');
SELECT _cdb_geocode_ipaddress_point('evil_user', 'evil_orgname', '8.8.8.8');
SELECT _cdb_geocode_street_point_v2('evil_user', 'evil_orgname', 'one street, 1');
SELECT _cdb_geocode_street_point('evil_user', 'evil_orgname', 'one street, 1');

View File

@@ -1,6 +1,6 @@
# Geocoder API
The CartoDB Geocoder API allows you to match your data with geometries on your map. This geocoding service can be used programatically to geocode datasets via the CartoDB SQL API. It is fed from _Open Data_ and it serves geometries for countries, provinces, states, cities, postal codes and IP addresses.
The CartoDB Geocoder API allows you to match your data with geometries on your map. This geocoding service can be used programatically to geocode datasets via the CartoDB SQL API. It is fed from _Open Data_ and it serves geometries for countries, provinces, states, cities, postal codes, IP addresses and street addresses.
## Documentation

View File

@@ -23,14 +23,14 @@ Geometry (polygon, EPSG 4326) or null
##### Select
```sql
```bash
SELECT cdb_geocode_admin0_polygon('France')
```
##### Update
```sql
UPDATE {tablename} SET {the_geom} = cdb_geocode_admin0_polygon({country_column})
```bash
UPDATE {tablename} SET the_geom = cdb_geocode_admin0_polygon({country_column})
```
@@ -54,14 +54,14 @@ Geometry (polygon, EPSG 4326) or null
##### Select
```sql
SELECT cdb_geocode_admin1_polygon('Alicante', 'Spain')
```bash
SELECT cdb_geocode_admin1_polygon('Alicante')
```
##### Update
```sql
UPDATE {tablename} SET the_geom = cdb_geocode_admin1_polygon({province_column}, {country_column})
```bash
UPDATE {tablename} SET the_geom = cdb_geocode_admin1_polygon({province_column})
```
### cdb_geocode_admin1_polygon(_admin1_name text, country_name text_)
@@ -81,13 +81,13 @@ Geometry (polygon, EPSG 4326) or null
##### Select
```sql
```bash
SELECT cdb_geocode_admin1_polygon('Alicante', 'Spain')
```
##### Update
```sql
```bash
UPDATE {tablename} SET the_geom = cdb_geocode_admin1_polygon({province_column}, {country_column})
```
@@ -112,13 +112,13 @@ Geometry (point, EPSG 4326) or null
##### Select
```sql
```bash
SELECT cdb_geocode_namedplace_point('Barcelona')
```
##### Update
```sql
```bash
UPDATE {tablename} SET the_geom = cdb_geocode_namedplace_point({city_column})
```
@@ -139,13 +139,13 @@ Geometry (point, EPSG 4326) or null
##### Select
```sql
```bash
SELECT cdb_geocode_namedplace_point('Barcelona', 'Spain')
```
##### Update
```sql
```bash
UPDATE {tablename} SET the_geom = cdb_geocode_namedplace_point({city_column}, 'Spain')
```
@@ -167,14 +167,14 @@ Geometry (point, EPSG 4326) or null
##### Select
```sql
```bash
SELECT cdb_geocode_namedplace_point('New York', 'New York', 'USA')
```
##### Update
```sql
UPDATE {tablename} SET the_geom = cdb_geocode_namedplace_point({city_column}, {province_column}, 'Spain')
```bash
UPDATE {tablename} SET the_geom = cdb_geocode_namedplace_point({city_column}, {province_column}, 'USA')
```
## Postal codes geocoder
@@ -198,14 +198,14 @@ Geometry (polygon, EPSG 4326) or null
##### Select
```sql
```bash
SELECT cdb_geocode_postalcode_polygon('11211', 'USA')
```
##### Update
```sql
UPDATE {tablename} SET the_geom = cdb_geocode_postalcode_polygon({postal_code_column}, 'Spain')
```bash
UPDATE {tablename} SET the_geom = cdb_geocode_postalcode_polygon({postal_code_column}, 'USA')
```
**Note:** For the USA, US Census ZCTAs are considered.
@@ -227,17 +227,17 @@ Geometry (point, EPSG 4326) or null
##### Select
```sql
```bash
SELECT cdb_geocode_postalcode_point('11211', 'USA')
```
##### Update
```sql
UPDATE {tablename} SET the_geom = cdb_geocode_postalcode_point({postal_code_column}, 'United States')
```bash
UPDATE {tablename} SET the_geom = cdb_geocode_postalcode_point({postal_code_column}, 'USA')
```
## IP addresses Geocoder
## IP addresses geocoder
This function provides an IP address geocoding service, for both IPv4 and IPv6 addresses.
@@ -258,12 +258,55 @@ Geometry (point, EPSG 4326) or null
##### Select
```sql
```bash
SELECT cdb_geocode_ipaddress_point('102.23.34.1')
```
##### Update
```sql
```bash
UPDATE {tablename} SET the_geom = cdb_geocode_ipaddress_point('102.23.34.1')
```
## Street-level geocoder
This function provides a street-level geocoding service. This service uses the street level geocoder defined for the user (currently, only the Here geocoder is available).
**This service is subject to quota limitations, and extra fees may apply.** Please view our [terms and conditions](https://cartodb.com/terms/)
Be mindful of the following when using this function:
- **One credit per function call will be consumed**, and the results are not cached. If the query applies to a N rows dataset, then N credits will be used.
- You are discouraged from using dynamic queries to the Geocoder API in your maps. This can result in credits consumption per map view. Note: **queries to the Geocoder API in your maps may be forbidden in the future**.
- You are advised to store results of Geocoder API queries into your datasets and refresh them as needed, so that you can have finer control on your credits' usage.
### cdb_geocode_street_point(_search_text text, [city text], [state text], [country text]_)
#### Arguments
Name | Type | Description
--- | --- | ---
`searchtext` | `text` | searchtext contains free-form text containing address elements. You can specify the searchtext parameter by itself, or you can specify it with other parameters to narrow your search. For example, you can specify the state or country parameters, along with a free-form address in the searchtext field.
`city` | `text` | (Optional) Name of the city
`state` | `text` | (Optional) Name of the state
`country` | `text` | (Optional) Name of the country
#### Returns
Geometry (point, EPSG 4326) or null
#### Example
##### Select
```bash
SELECT cdb_geocode_street_point('651 Lombard Street, San Francisco, California, United States')
SELECT cdb_geocode_street_point('651 Lombard Street', 'San Francisco')
SELECT cdb_geocode_street_point('651 Lombard Street', 'San Francisco', 'California')
SELECT cdb_geocode_street_point('651 Lombard Street', 'San Francisco', 'California', 'United States')
```
##### Update
```bash
UPDATE {tablename} SET the_geom = cdb_geocode_street_point({street_name_column})
```

View File

@@ -51,7 +51,7 @@
params:
- { name: ip_address, type: text}
- name: cdb_geocode_street_point_v2
- name: cdb_geocode_street_point
return_type: Geometry
params:
- { name: searchtext, type: text}

View File

@@ -2,4 +2,5 @@ results/
regression.diffs
regression.out
cdb_geocoder_server--0.0.1.sql
cdb_geocoder_server--0.1.0.sql
cdb_geocoder_server--0.1.0.sql
cdb_geocoder_server--0.2.0.sql

View File

@@ -12,8 +12,11 @@ NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql
# @see http://www.postgresql.org/docs/current/static/extend-pgxs.html
DATA = $(NEW_EXTENSION_ARTIFACT) \
cdb_geocoder_server--0.0.1.sql \
cdb_geocoder_server--0.1.0.sql \
cdb_geocoder_server--0.1.0--0.0.1.sql \
cdb_geocoder_server--0.0.1--0.1.0.sql
cdb_geocoder_server--0.0.1--0.1.0.sql \
cdb_geocoder_server--0.2.0--0.1.0.sql \
cdb_geocoder_server--0.1.0--0.2.0.sql
REGRESS = $(notdir $(basename $(wildcard test/$(EXTVERSION)/sql/*test.sql)))

View File

@@ -46,7 +46,8 @@ RETURNS boolean AS $$
'redis_metrics_connection': redis_metrics_connection,
}
return True
$$ LANGUAGE plpythonu;
$$ LANGUAGE plpythonu SECURITY DEFINER;
-- Get the Redis configuration from the _conf table --
CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_geocoder_config(username text, orgname text)
RETURNS boolean AS $$
@@ -71,9 +72,11 @@ RETURNS boolean AS $$
# --for this user session but...
GD[cache_key] = geocoder_config
return True
$$ LANGUAGE plpythonu;
$$ LANGUAGE plpythonu SECURITY DEFINER;
-- Geocodes a street address given a searchtext and a state and/or country
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_street_point_v2(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
DROP FUNCTION IF EXISTS cdb_geocoder_server.cdb_geocode_street_point(TEXT, TEXT, TEXT, TEXT);
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
@@ -130,3 +133,10 @@ RETURNS Geometry AS $$
plpy.error('Google geocoder is not available yet')
return None
$$ LANGUAGE plpythonu;
-- We apply again the grants to include the new functions
GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_geocoder_server TO geocoder_api;
GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO geocoder_api;
GRANT USAGE ON SCHEMA cdb_geocoder_server TO geocoder_api;
GRANT USAGE ON SCHEMA public TO geocoder_api;
GRANT SELECT ON ALL TABLES IN SCHEMA public TO geocoder_api;

View File

@@ -1,6 +1,29 @@
DROP FUNCTION IF EXISTS cdb_geocoder_server._get_redis_conf_v2(text);
DROP FUNCTION IF EXISTS cdb_geocoder_server._connect_to_redis(text);
DROP FUNCTION IF EXISTS cdb_geocoder_server._get_geocoder_config(text, text);
DROP FUNCTION IF EXISTS cdb_geocoder_server.cdb_geocode_street_point(TEXT, TEXT, TEXT, TEXT, TEXT, TEXT);
DROP FUNCTION IF EXISTS cdb_geocoder_server.cdb_geocode_street_point_v2(TEXT, TEXT, TEXT, TEXT, TEXT, TEXT);
DROP FUNCTION IF EXISTS cdb_geocoder_server._cdb_here_geocode_street_point(TEXT, TEXT, TEXT, TEXT, TEXT, TEXT);
DROP FUNCTION IF EXISTS cdb_geocoder_server._cdb_google_geocode_street_point(TEXT, TEXT, TEXT, TEXT, TEXT, TEXT);
DROP FUNCTION IF EXISTS cdb_geocoder_server._cdb_google_geocode_street_point(TEXT, TEXT, TEXT, TEXT, TEXT, TEXT);
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_street_point(searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry
AS $$
import json
from heremaps import heremapsgeocoder
heremaps_conf = json.loads(plpy.execute("SELECT cdb_geocoder_server._get_conf('heremaps')", 1)[0]['get_conf'])
app_id = heremaps_conf['geocoder']['app_id']
app_code = heremaps_conf['geocoder']['app_code']
geocoder = heremapsgeocoder.Geocoder(app_id, app_code)
results = geocoder.geocode_address(searchtext=searchtext, city=city, state=state_province, country=country)
coordinates = geocoder.extract_lng_lat_from_result(results[0])
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
return point['st_setsrid']
$$ LANGUAGE plpythonu;

View File

@@ -0,0 +1,122 @@
CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_geocoder_config(username text, orgname text)
RETURNS boolean AS $$
cache_key = "user_geocoder_config_{0}".format(username)
if cache_key in GD:
return False
else:
import json
from cartodb_services.metrics import GeocoderConfig
plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf']
if not heremaps_conf_json:
heremaps_app_id = None
heremaps_app_code = None
else:
heremaps_conf = json.loads(heremaps_conf_json)
heremaps_app_id = heremaps_conf['app_id']
heremaps_app_code = heremaps_conf['app_code']
geocoder_config = GeocoderConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code)
# --Think about the security concerns with this kind of global cache, it should be only available
# --for this user session but...
GD[cache_key] = geocoder_config
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;
-- Get the connection to redis from cache or create a new one
CREATE OR REPLACE FUNCTION cdb_geocoder_server._connect_to_redis(user_id text)
RETURNS boolean AS $$
cache_key = "redis_connection_{0}".format(user_id)
if cache_key in GD:
return False
else:
from cartodb_services.tools import RedisConnection
metadata_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port,
c.sentinel_master_id, c.timeout, c.redis_db
from cdb_geocoder_server._get_redis_conf_v2('redis_metadata_config') c;""")[0]
metrics_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port,
c.sentinel_master_id, c.timeout, c.redis_db
from cdb_geocoder_server._get_redis_conf_v2('redis_metrics_config') c;""")[0]
redis_metadata_connection = RedisConnection(metadata_config_params['sentinel_host'],
metadata_config_params['sentinel_port'],
metadata_config_params['sentinel_master_id'],
timeout=metadata_config_params['timeout'],
redis_db=metadata_config_params['redis_db']).redis_connection()
redis_metrics_connection = RedisConnection(metrics_config_params['sentinel_host'],
metrics_config_params['sentinel_port'],
metrics_config_params['sentinel_master_id'],
timeout=metrics_config_params['timeout'],
redis_db=metrics_config_params['redis_db']).redis_connection()
GD[cache_key] = {
'redis_metadata_connection': redis_metadata_connection,
'redis_metrics_connection': redis_metrics_connection,
}
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from cartodb_services.here import HereMapsGeocoder
from cartodb_services.metrics import QuotaService
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
# -- Check the quota
quota_service = QuotaService(user_geocoder_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reach the limit of your quota')
try:
geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code)
coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country)
if coordinates:
quota_service.increment_success_geocoder_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
return point['st_setsrid']
else:
quota_service.increment_empty_geocoder_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_geocoder_use()
error_msg = 'There was an error trying to geocode using here maps geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_geocoder_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from cartodb_services.google import GoogleMapsGeocoder
from cartodb_services.metrics import QuotaService
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
geocoder = GoogleMapsGeocoder(user_geocoder_config.google_client_id, user_geocoder_config.google_api_key)
coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country)
if coordinates:
quota_service.increment_success_geocoder_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
return point['st_setsrid']
else:
quota_service.increment_empty_geocoder_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_geocoder_use()
error_msg = 'There was an error trying to geocode using google maps geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_geocoder_use()
$$ LANGUAGE plpythonu;

View File

@@ -0,0 +1,615 @@
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "CREATE EXTENSION cdb_geocoder_server" to load this file. \quit
CREATE TYPE cdb_geocoder_server._redis_conf_params AS (
sentinel_host text,
sentinel_port int,
sentinel_master_id text,
redis_db text,
timeout float
);
-- Get the Redis configuration from the _conf table --
CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_redis_conf_v2(config_key text)
RETURNS cdb_geocoder_server._redis_conf_params AS $$
conf_query = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(config_key)
conf = plpy.execute(conf_query)[0]['conf']
if conf is None:
plpy.error("There is no redis configuration defined")
else:
import json
params = json.loads(conf)
return {
"sentinel_host": params['sentinel_host'],
"sentinel_port": params['sentinel_port'],
"sentinel_master_id": params['sentinel_master_id'],
"timeout": params['timeout'],
"redis_db": params['redis_db']
}
$$ LANGUAGE plpythonu;
-- Get the connection to redis from cache or create a new one
CREATE OR REPLACE FUNCTION cdb_geocoder_server._connect_to_redis(user_id text)
RETURNS boolean AS $$
cache_key = "redis_connection_{0}".format(user_id)
if cache_key in GD:
return False
else:
from cartodb_geocoder import redis_helper
metadata_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port,
c.sentinel_master_id, c.timeout, c.redis_db
from cdb_geocoder_server._get_redis_conf_v2('redis_metadata_config') c;""")[0]
metrics_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port,
c.sentinel_master_id, c.timeout, c.redis_db
from cdb_geocoder_server._get_redis_conf_v2('redis_metrics_config') c;""")[0]
redis_metadata_connection = redis_helper.RedisHelper(metadata_config_params['sentinel_host'],
metadata_config_params['sentinel_port'],
metadata_config_params['sentinel_master_id'],
timeout=metadata_config_params['timeout'],
redis_db=metadata_config_params['redis_db']).redis_connection()
redis_metrics_connection = redis_helper.RedisHelper(metrics_config_params['sentinel_host'],
metrics_config_params['sentinel_port'],
metrics_config_params['sentinel_master_id'],
timeout=metrics_config_params['timeout'],
redis_db=metrics_config_params['redis_db']).redis_connection()
GD[cache_key] = {
'redis_metadata_connection': redis_metadata_connection,
'redis_metrics_connection': redis_metrics_connection,
}
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;
-- Get the Redis configuration from the _conf table --
CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_geocoder_config(username text, orgname text)
RETURNS boolean AS $$
cache_key = "user_geocoder_config_{0}".format(username)
if cache_key in GD:
return False
else:
import json
from cartodb_geocoder import config_helper
plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf']
if not heremaps_conf_json:
heremaps_app_id = None
heremaps_app_code = None
else:
heremaps_conf = json.loads(heremaps_conf_json)
heremaps_app_id = heremaps_conf['app_id']
heremaps_app_code = heremaps_conf['app_code']
geocoder_config = config_helper.GeocoderConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code)
# --Think about the security concerns with this kind of global cache, it should be only available
# --for this user session but...
GD[cache_key] = geocoder_config
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;
-- Geocodes a street address given a searchtext and a state and/or country
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_geocoder_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
if user_geocoder_config.heremaps_geocoder:
here_plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
elif user_geocoder_config.google_geocoder:
google_plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
else:
plpy.error('Requested geocoder is not available')
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from heremaps import heremapsgeocoder
from cartodb_geocoder import quota_service
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
# -- Check the quota
quota_service = quota_service.QuotaService(user_geocoder_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reach the limit of your quota')
try:
geocoder = heremapsgeocoder.Geocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code)
results = geocoder.geocode_address(searchtext=searchtext, city=city, state=state_province, country=country)
coordinates = geocoder.extract_lng_lat_from_result(results[0])
quota_service.increment_success_geocoder_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
return point['st_setsrid']
except heremapsgeocoder.EmptyGeocoderResponse:
quota_service.increment_empty_geocoder_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_geocoder_use()
error_msg = 'There was an error trying to geocode using here maps geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
plpy.error('Google geocoder is not available yet')
return None
$$ LANGUAGE plpythonu;
-- Interface of the server extension
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text)
RETURNS Geometry AS $$
plpy.debug('Entering cdb_geocode_admin0_polygons')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_admin0_polygon($1) AS mypolygon", ["text"])
rv = plpy.execute(plan, [country_name], 1)
plpy.debug('Returning from Returning from cdb_geocode_admin0_polygons')
return rv[0]["mypolygon"]
$$ LANGUAGE plpythonu;
--------------------------------------------------------------------------------
-- Implementation of the server extension
-- Note: these functions depend on the cdb_geocoder extension
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_admin0_polygon(country_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT n.the_geom as geom INTO ret
FROM (SELECT q, lower(regexp_replace(q, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text x
FROM (SELECT country_name q) g) d
LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x
LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.the_geom, s.adm0_a3;
RETURN ret;
END
$$ LANGUAGE plpgsql;
-- Interfacess of the server extension
---- cdb_geocode_admin1_polygon(admin1_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text)
RETURNS Geometry AS $$
plpy.debug('Entering cdb_geocode_admin1_polygon(admin1_name text)')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_admin1_polygon($1) AS mypolygon", ["text"])
rv = plpy.execute(plan, [admin1_name], 1)
plpy.debug('Returning from Returning from cdb_geocode_admin1_polygons')
return rv[0]["mypolygon"]
$$ LANGUAGE plpythonu;
---- cdb_geocode_admin1_polygon(admin1_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text)
RETURNS Geometry AS $$
plpy.debug('Entering cdb_geocode_admin1_polygon(admin1_name text, country_name text)')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_admin1_polygon($1, $2) AS mypolygon", ["text", "text"])
rv = plpy.execute(plan, [admin1_name, country_name], 1)
plpy.debug('Returning from Returning from cdb_geocode_admin1_polygon(admin1_name text, country_name text)')
return rv[0]["mypolygon"]
$$ LANGUAGE plpythonu;
--------------------------------------------------------------------------------
-- Implementation of the server extension
-- Note: these functions depend on the cdb_geocoder extension
---- cdb_geocode_admin1_polygon(admin1_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_admin1_polygon(admin1_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
SELECT q, (
SELECT the_geom
FROM global_province_polygons
WHERE d.c = ANY (synonyms)
ORDER BY frequency DESC LIMIT 1
) geom
FROM (
SELECT
trim(replace(lower(admin1_name),'.',' ')) c, admin1_name q
) d
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
---- cdb_geocode_admin1_polygon(admin1_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_admin1_polygon(admin1_name text, country_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
WITH p AS (SELECT r.c, r.q, (SELECT iso3 FROM country_decoder WHERE lower(country_name) = ANY (synonyms)) i FROM (SELECT trim(replace(lower(admin1_name),'.',' ')) c, country_name q) r)
SELECT
geom INTO ret
FROM (
SELECT
q, (
SELECT the_geom
FROM global_province_polygons
WHERE p.c = ANY (synonyms)
AND iso3 = p.i
ORDER BY frequency DESC LIMIT 1
) geom
FROM p) n;
RETURN ret;
END
$$ LANGUAGE plpgsql;
-- Interfacess of the server extension
---- cdb_geocode_namedplace_point(city_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text)
RETURNS Geometry AS $$
plpy.debug('Entering cdb_geocode_namedplace_point(city_name text)')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_namedplace_point($1) AS mypoint", ["text"])
rv = plpy.execute(plan, [city_name], 1)
plpy.debug('Returning from Returning from geocode_namedplace')
return rv[0]["mypoint"]
$$ LANGUAGE plpythonu;
---- cdb_geocode_namedplace_point(city_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text)
RETURNS Geometry AS $$
plpy.debug('Entering cdb_geocode_namedplace_point(city_name text, country_name text)')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_namedplace_point($1, $2) AS mypoint", ["text", "text"])
rv = plpy.execute(plan, [city_name, country_name], 1)
plpy.debug('Returning from Returning from geocode_namedplace')
return rv[0]["mypoint"]
$$ LANGUAGE plpythonu;
---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text)
RETURNS Geometry AS $$
plpy.debug('Entering cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_namedplace_point($1, $2, $3) AS mypoint", ["text", "text", "text"])
rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1)
plpy.debug('Returning from Returning from geocode_namedplace')
return rv[0]["mypoint"]
$$ LANGUAGE plpythonu;
--------------------------------------------------------------------------------
-- Implementation of the server extension
-- Note: these functions depend on the cdb_geocoder extension
---- cdb_geocode_namedplace_point(city_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_namedplace_point(city_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
WITH best AS (SELECT s AS q, (SELECT the_geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT city_name as s) p),
next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT city_name as s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL))
SELECT q, geom, TRUE AS success FROM best WHERE geom IS NOT NULL
UNION ALL
SELECT q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
---- cdb_geocode_namedplace_point(city_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_namedplace_point(city_name text, country_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
WITH p AS (SELECT r.s, r.c, (SELECT iso2 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT city_name AS s, country_name::text AS c) r),
best AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p),
next AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE c = p.c AND geom IS NOT NULL))
SELECT geom FROM best WHERE geom IS NOT NULL
UNION ALL
SELECT geom FROM next
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
WITH inputcountry AS (
SELECT iso2 as isoTwo FROM country_decoder WHERE lower(country_name) = ANY (synonyms) LIMIT 1
),
p AS (
SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder, inputcountry WHERE lower(r.a1) = ANY (synonyms) AND admin1_decoder.iso2 = inputcountry.isoTwo LIMIT 1) i FROM (SELECT city_name AS s, admin1_name::text AS a1) r),
best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p),
next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL))
SELECT geom FROM best WHERE geom IS NOT NULL
UNION ALL
SELECT geom FROM next
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
-- Interface of the server extension
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_point(username text, orgname text, code text)
RETURNS Geometry AS $$
plpy.debug('Entering _cdb_geocode_postalcode_point')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_point($1) AS point", ["text"])
rv = plpy.execute(plan, [code], 1)
plpy.debug('Returning from _cdb_geocode_postalcode_point')
return rv[0]["point"]
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text)
RETURNS Geometry AS $$
plpy.debug('Entering _cdb_geocode_postalcode_point')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_point($1, $2) AS point", ["TEXT", "TEXT"])
rv = plpy.execute(plan, [code, country], 1)
plpy.debug('Returning from _cdb_geocode_postalcode_point')
return rv[0]["point"]
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text)
RETURNS Geometry AS $$
plpy.debug('Entering _cdb_geocode_postalcode_polygon')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_polygon($1) AS polygon", ["text"])
rv = plpy.execute(plan, [code], 1)
plpy.debug('Returning from _cdb_geocode_postalcode_polygon')
return rv[0]["polygon"]
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text)
RETURNS Geometry AS $$
plpy.debug('Entering _cdb_geocode_postalcode_point')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_polygon($1, $2) AS polygon", ["TEXT", "TEXT"])
rv = plpy.execute(plan, [code, country], 1)
plpy.debug('Returning from _cdb_geocode_postalcode_point')
return rv[0]["polygon"]
$$ LANGUAGE plpythonu;
--------------------------------------------------------------------------------
-- Implementation of the server extension
-- Note: these functions depend on the cdb_geocoder extension
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_point(code text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
SELECT
q, (
SELECT the_geom
FROM global_postal_code_points
WHERE postal_code = upper(d.q)
LIMIT 1
) geom
FROM (SELECT code q) d
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_point(code text, country text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
SELECT
q, (
SELECT the_geom
FROM global_postal_code_points
WHERE postal_code = upper(d.q)
AND iso3 = (
SELECT iso3 FROM country_decoder WHERE
lower(country) = ANY (synonyms) LIMIT 1
)
LIMIT 1
) geom
FROM (SELECT code q) d
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_polygon(code text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
SELECT
q, (
SELECT the_geom
FROM global_postal_code_polygons
WHERE postal_code = upper(d.q)
LIMIT 1
) geom
FROM (SELECT code q) d
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_polygon(code text, country text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
SELECT
q, (
SELECT the_geom
FROM global_postal_code_polygons
WHERE postal_code = upper(d.q)
AND iso3 = (
SELECT iso3 FROM country_decoder WHERE
lower(country) = ANY (synonyms) LIMIT 1
)
LIMIT 1
) geom
FROM (SELECT code q) d
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
-- Interface of the server extension
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text)
RETURNS Geometry AS $$
plpy.debug('Entering _cdb_geocode_ipaddress_point')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_ipaddress_point($1) AS point", ["TEXT"])
rv = plpy.execute(plan, [ip], 1)
plpy.debug('Returning from _cdb_geocode_ipaddress_point')
return rv[0]["point"]
$$ LANGUAGE plpythonu;
--------------------------------------------------------------------------------
-- Implementation of the server extension
-- Note: these functions depend on the cdb_geocoder extension
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_ipaddress_point(ip text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
new_ip INET;
BEGIN
BEGIN
IF family(ip::inet) = 6 THEN
new_ip := ip::inet;
ELSE
new_ip := ('::ffff:' || ip)::inet;
END IF;
EXCEPTION WHEN OTHERS THEN
SELECT NULL as geom INTO ret;
RETURN ret;
END;
WITH
ips AS (SELECT ip s, new_ip net),
matches AS (SELECT s, (SELECT the_geom FROM ip_address_locations WHERE network_start_ip <= ips.net ORDER BY network_start_ip DESC LIMIT 1) geom FROM ips)
SELECT geom INTO ret
FROM matches;
RETURN ret;
END
$$ LANGUAGE plpgsql;
DO $$
BEGIN
IF NOT EXISTS (
SELECT *
FROM pg_catalog.pg_user
WHERE usename = 'geocoder_api') THEN
CREATE USER geocoder_api;
END IF;
GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_geocoder_server TO geocoder_api;
GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO geocoder_api;
GRANT USAGE ON SCHEMA cdb_geocoder_server TO geocoder_api;
GRANT USAGE ON SCHEMA public TO geocoder_api;
GRANT SELECT ON ALL TABLES IN SCHEMA public TO geocoder_api;
END$$;

View File

@@ -0,0 +1,95 @@
-- Get the Redis configuration from the _conf table --
CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_geocoder_config(username text, orgname text)
RETURNS boolean AS $$
cache_key = "user_geocoder_config_{0}".format(username)
if cache_key in GD:
return False
else:
import json
from cartodb_geocoder import config_helper
plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf']
if not heremaps_conf_json:
heremaps_app_id = None
heremaps_app_code = None
else:
heremaps_conf = json.loads(heremaps_conf_json)
heremaps_app_id = heremaps_conf['app_id']
heremaps_app_code = heremaps_conf['app_code']
geocoder_config = config_helper.GeocoderConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code)
# --Think about the security concerns with this kind of global cache, it should be only available
# --for this user session but...
GD[cache_key] = geocoder_config
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;
-- Get the connection to redis from cache or create a new one
CREATE OR REPLACE FUNCTION cdb_geocoder_server._connect_to_redis(user_id text)
RETURNS boolean AS $$
cache_key = "redis_connection_{0}".format(user_id)
if cache_key in GD:
return False
else:
from cartodb_geocoder import redis_helper
metadata_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port,
c.sentinel_master_id, c.timeout, c.redis_db
from cdb_geocoder_server._get_redis_conf_v2('redis_metadata_config') c;""")[0]
metrics_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port,
c.sentinel_master_id, c.timeout, c.redis_db
from cdb_geocoder_server._get_redis_conf_v2('redis_metrics_config') c;""")[0]
redis_metadata_connection = redis_helper.RedisHelper(metadata_config_params['sentinel_host'],
metadata_config_params['sentinel_port'],
metadata_config_params['sentinel_master_id'],
timeout=metadata_config_params['timeout'],
redis_db=metadata_config_params['redis_db']).redis_connection()
redis_metrics_connection = redis_helper.RedisHelper(metrics_config_params['sentinel_host'],
metrics_config_params['sentinel_port'],
metrics_config_params['sentinel_master_id'],
timeout=metrics_config_params['timeout'],
redis_db=metrics_config_params['redis_db']).redis_connection()
GD[cache_key] = {
'redis_metadata_connection': redis_metadata_connection,
'redis_metrics_connection': redis_metrics_connection,
}
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from heremaps import heremapsgeocoder
from cartodb_geocoder import quota_service
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
# -- Check the quota
quota_service = quota_service.QuotaService(user_geocoder_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reach the limit of your quota')
try:
geocoder = heremapsgeocoder.Geocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code)
coordinates = geocoder.geocode_address(searchtext=searchtext, city=city, state=state_province, country=country)
if coordinates:
quota_service.increment_success_geocoder_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
return point['st_setsrid']
else:
quota_service.increment_empty_geocoder_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_geocoder_use()
error_msg = 'There was an error trying to geocode using here maps geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
plpy.error('Google geocoder is not available yet')
return None
$$ LANGUAGE plpythonu;

View File

@@ -0,0 +1,642 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "CREATE EXTENSION cdb_geocoder_server" to load this file. \quit
CREATE TYPE cdb_geocoder_server._redis_conf_params AS (
sentinel_host text,
sentinel_port int,
sentinel_master_id text,
redis_db text,
timeout float
);
-- Get the Redis configuration from the _conf table --
CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_redis_conf_v2(config_key text)
RETURNS cdb_geocoder_server._redis_conf_params AS $$
conf_query = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(config_key)
conf = plpy.execute(conf_query)[0]['conf']
if conf is None:
plpy.error("There is no redis configuration defined")
else:
import json
params = json.loads(conf)
return {
"sentinel_host": params['sentinel_host'],
"sentinel_port": params['sentinel_port'],
"sentinel_master_id": params['sentinel_master_id'],
"timeout": params['timeout'],
"redis_db": params['redis_db']
}
$$ LANGUAGE plpythonu;
-- Get the connection to redis from cache or create a new one
CREATE OR REPLACE FUNCTION cdb_geocoder_server._connect_to_redis(user_id text)
RETURNS boolean AS $$
cache_key = "redis_connection_{0}".format(user_id)
if cache_key in GD:
return False
else:
from cartodb_services.tools import RedisConnection
metadata_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port,
c.sentinel_master_id, c.timeout, c.redis_db
from cdb_geocoder_server._get_redis_conf_v2('redis_metadata_config') c;""")[0]
metrics_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port,
c.sentinel_master_id, c.timeout, c.redis_db
from cdb_geocoder_server._get_redis_conf_v2('redis_metrics_config') c;""")[0]
redis_metadata_connection = RedisConnection(metadata_config_params['sentinel_host'],
metadata_config_params['sentinel_port'],
metadata_config_params['sentinel_master_id'],
timeout=metadata_config_params['timeout'],
redis_db=metadata_config_params['redis_db']).redis_connection()
redis_metrics_connection = RedisConnection(metrics_config_params['sentinel_host'],
metrics_config_params['sentinel_port'],
metrics_config_params['sentinel_master_id'],
timeout=metrics_config_params['timeout'],
redis_db=metrics_config_params['redis_db']).redis_connection()
GD[cache_key] = {
'redis_metadata_connection': redis_metadata_connection,
'redis_metrics_connection': redis_metrics_connection,
}
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;
-- Get the Redis configuration from the _conf table --
CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_geocoder_config(username text, orgname text)
RETURNS boolean AS $$
cache_key = "user_geocoder_config_{0}".format(username)
if cache_key in GD:
return False
else:
import json
from cartodb_services.metrics import GeocoderConfig
plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf']
if not heremaps_conf_json:
heremaps_app_id = None
heremaps_app_code = None
else:
heremaps_conf = json.loads(heremaps_conf_json)
heremaps_app_id = heremaps_conf['app_id']
heremaps_app_code = heremaps_conf['app_code']
geocoder_config = GeocoderConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code)
# --Think about the security concerns with this kind of global cache, it should be only available
# --for this user session but...
GD[cache_key] = geocoder_config
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;
-- Geocodes a street address given a searchtext and a state and/or country
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_street_point_v2(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_geocoder_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
if user_geocoder_config.heremaps_geocoder:
here_plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
elif user_geocoder_config.google_geocoder:
google_plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
else:
plpy.error('Requested geocoder is not available')
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from cartodb_services.here import HereMapsGeocoder
from cartodb_services.metrics import QuotaService
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
# -- Check the quota
quota_service = QuotaService(user_geocoder_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reach the limit of your quota')
try:
geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code)
coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country)
if coordinates:
quota_service.increment_success_geocoder_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
return point['st_setsrid']
else:
quota_service.increment_empty_geocoder_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_geocoder_use()
error_msg = 'There was an error trying to geocode using here maps geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_geocoder_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from cartodb_services.google import GoogleMapsGeocoder
from cartodb_services.metrics import QuotaService
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
geocoder = GoogleMapsGeocoder(user_geocoder_config.google_client_id, user_geocoder_config.google_api_key)
coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country)
if coordinates:
quota_service.increment_success_geocoder_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
return point['st_setsrid']
else:
quota_service.increment_empty_geocoder_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_geocoder_use()
error_msg = 'There was an error trying to geocode using google maps geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_geocoder_use()
$$ LANGUAGE plpythonu;
-- Interface of the server extension
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text)
RETURNS Geometry AS $$
plpy.debug('Entering cdb_geocode_admin0_polygons')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_admin0_polygon($1) AS mypolygon", ["text"])
rv = plpy.execute(plan, [country_name], 1)
plpy.debug('Returning from Returning from cdb_geocode_admin0_polygons')
return rv[0]["mypolygon"]
$$ LANGUAGE plpythonu;
--------------------------------------------------------------------------------
-- Implementation of the server extension
-- Note: these functions depend on the cdb_geocoder extension
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_admin0_polygon(country_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT n.the_geom as geom INTO ret
FROM (SELECT q, lower(regexp_replace(q, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text x
FROM (SELECT country_name q) g) d
LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x
LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.the_geom, s.adm0_a3;
RETURN ret;
END
$$ LANGUAGE plpgsql;
-- Interfacess of the server extension
---- cdb_geocode_admin1_polygon(admin1_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text)
RETURNS Geometry AS $$
plpy.debug('Entering cdb_geocode_admin1_polygon(admin1_name text)')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_admin1_polygon($1) AS mypolygon", ["text"])
rv = plpy.execute(plan, [admin1_name], 1)
plpy.debug('Returning from Returning from cdb_geocode_admin1_polygons')
return rv[0]["mypolygon"]
$$ LANGUAGE plpythonu;
---- cdb_geocode_admin1_polygon(admin1_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text)
RETURNS Geometry AS $$
plpy.debug('Entering cdb_geocode_admin1_polygon(admin1_name text, country_name text)')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_admin1_polygon($1, $2) AS mypolygon", ["text", "text"])
rv = plpy.execute(plan, [admin1_name, country_name], 1)
plpy.debug('Returning from Returning from cdb_geocode_admin1_polygon(admin1_name text, country_name text)')
return rv[0]["mypolygon"]
$$ LANGUAGE plpythonu;
--------------------------------------------------------------------------------
-- Implementation of the server extension
-- Note: these functions depend on the cdb_geocoder extension
---- cdb_geocode_admin1_polygon(admin1_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_admin1_polygon(admin1_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
SELECT q, (
SELECT the_geom
FROM global_province_polygons
WHERE d.c = ANY (synonyms)
ORDER BY frequency DESC LIMIT 1
) geom
FROM (
SELECT
trim(replace(lower(admin1_name),'.',' ')) c, admin1_name q
) d
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
---- cdb_geocode_admin1_polygon(admin1_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_admin1_polygon(admin1_name text, country_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
WITH p AS (SELECT r.c, r.q, (SELECT iso3 FROM country_decoder WHERE lower(country_name) = ANY (synonyms)) i FROM (SELECT trim(replace(lower(admin1_name),'.',' ')) c, country_name q) r)
SELECT
geom INTO ret
FROM (
SELECT
q, (
SELECT the_geom
FROM global_province_polygons
WHERE p.c = ANY (synonyms)
AND iso3 = p.i
ORDER BY frequency DESC LIMIT 1
) geom
FROM p) n;
RETURN ret;
END
$$ LANGUAGE plpgsql;
-- Interfacess of the server extension
---- cdb_geocode_namedplace_point(city_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text)
RETURNS Geometry AS $$
plpy.debug('Entering cdb_geocode_namedplace_point(city_name text)')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_namedplace_point($1) AS mypoint", ["text"])
rv = plpy.execute(plan, [city_name], 1)
plpy.debug('Returning from Returning from geocode_namedplace')
return rv[0]["mypoint"]
$$ LANGUAGE plpythonu;
---- cdb_geocode_namedplace_point(city_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text)
RETURNS Geometry AS $$
plpy.debug('Entering cdb_geocode_namedplace_point(city_name text, country_name text)')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_namedplace_point($1, $2) AS mypoint", ["text", "text"])
rv = plpy.execute(plan, [city_name, country_name], 1)
plpy.debug('Returning from Returning from geocode_namedplace')
return rv[0]["mypoint"]
$$ LANGUAGE plpythonu;
---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text)
RETURNS Geometry AS $$
plpy.debug('Entering cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_namedplace_point($1, $2, $3) AS mypoint", ["text", "text", "text"])
rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1)
plpy.debug('Returning from Returning from geocode_namedplace')
return rv[0]["mypoint"]
$$ LANGUAGE plpythonu;
--------------------------------------------------------------------------------
-- Implementation of the server extension
-- Note: these functions depend on the cdb_geocoder extension
---- cdb_geocode_namedplace_point(city_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_namedplace_point(city_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
WITH best AS (SELECT s AS q, (SELECT the_geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT city_name as s) p),
next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT city_name as s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL))
SELECT q, geom, TRUE AS success FROM best WHERE geom IS NOT NULL
UNION ALL
SELECT q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
---- cdb_geocode_namedplace_point(city_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_namedplace_point(city_name text, country_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
WITH p AS (SELECT r.s, r.c, (SELECT iso2 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT city_name AS s, country_name::text AS c) r),
best AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p),
next AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE c = p.c AND geom IS NOT NULL))
SELECT geom FROM best WHERE geom IS NOT NULL
UNION ALL
SELECT geom FROM next
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
WITH inputcountry AS (
SELECT iso2 as isoTwo FROM country_decoder WHERE lower(country_name) = ANY (synonyms) LIMIT 1
),
p AS (
SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder, inputcountry WHERE lower(r.a1) = ANY (synonyms) AND admin1_decoder.iso2 = inputcountry.isoTwo LIMIT 1) i FROM (SELECT city_name AS s, admin1_name::text AS a1) r),
best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p),
next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL))
SELECT geom FROM best WHERE geom IS NOT NULL
UNION ALL
SELECT geom FROM next
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
-- Interface of the server extension
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_point(username text, orgname text, code text)
RETURNS Geometry AS $$
plpy.debug('Entering _cdb_geocode_postalcode_point')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_point($1) AS point", ["text"])
rv = plpy.execute(plan, [code], 1)
plpy.debug('Returning from _cdb_geocode_postalcode_point')
return rv[0]["point"]
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text)
RETURNS Geometry AS $$
plpy.debug('Entering _cdb_geocode_postalcode_point')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_point($1, $2) AS point", ["TEXT", "TEXT"])
rv = plpy.execute(plan, [code, country], 1)
plpy.debug('Returning from _cdb_geocode_postalcode_point')
return rv[0]["point"]
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text)
RETURNS Geometry AS $$
plpy.debug('Entering _cdb_geocode_postalcode_polygon')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_polygon($1) AS polygon", ["text"])
rv = plpy.execute(plan, [code], 1)
plpy.debug('Returning from _cdb_geocode_postalcode_polygon')
return rv[0]["polygon"]
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text)
RETURNS Geometry AS $$
plpy.debug('Entering _cdb_geocode_postalcode_point')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_polygon($1, $2) AS polygon", ["TEXT", "TEXT"])
rv = plpy.execute(plan, [code, country], 1)
plpy.debug('Returning from _cdb_geocode_postalcode_point')
return rv[0]["polygon"]
$$ LANGUAGE plpythonu;
--------------------------------------------------------------------------------
-- Implementation of the server extension
-- Note: these functions depend on the cdb_geocoder extension
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_point(code text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
SELECT
q, (
SELECT the_geom
FROM global_postal_code_points
WHERE postal_code = upper(d.q)
LIMIT 1
) geom
FROM (SELECT code q) d
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_point(code text, country text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
SELECT
q, (
SELECT the_geom
FROM global_postal_code_points
WHERE postal_code = upper(d.q)
AND iso3 = (
SELECT iso3 FROM country_decoder WHERE
lower(country) = ANY (synonyms) LIMIT 1
)
LIMIT 1
) geom
FROM (SELECT code q) d
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_polygon(code text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
SELECT
q, (
SELECT the_geom
FROM global_postal_code_polygons
WHERE postal_code = upper(d.q)
LIMIT 1
) geom
FROM (SELECT code q) d
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_polygon(code text, country text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
SELECT
q, (
SELECT the_geom
FROM global_postal_code_polygons
WHERE postal_code = upper(d.q)
AND iso3 = (
SELECT iso3 FROM country_decoder WHERE
lower(country) = ANY (synonyms) LIMIT 1
)
LIMIT 1
) geom
FROM (SELECT code q) d
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
-- Interface of the server extension
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text)
RETURNS Geometry AS $$
plpy.debug('Entering _cdb_geocode_ipaddress_point')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_ipaddress_point($1) AS point", ["TEXT"])
rv = plpy.execute(plan, [ip], 1)
plpy.debug('Returning from _cdb_geocode_ipaddress_point')
return rv[0]["point"]
$$ LANGUAGE plpythonu;
--------------------------------------------------------------------------------
-- Implementation of the server extension
-- Note: these functions depend on the cdb_geocoder extension
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_ipaddress_point(ip text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
new_ip INET;
BEGIN
BEGIN
IF family(ip::inet) = 6 THEN
new_ip := ip::inet;
ELSE
new_ip := ('::ffff:' || ip)::inet;
END IF;
EXCEPTION WHEN OTHERS THEN
SELECT NULL as geom INTO ret;
RETURN ret;
END;
WITH
ips AS (SELECT ip s, new_ip net),
matches AS (SELECT s, (SELECT the_geom FROM ip_address_locations WHERE network_start_ip <= ips.net ORDER BY network_start_ip DESC LIMIT 1) geom FROM ips)
SELECT geom INTO ret
FROM matches;
RETURN ret;
END
$$ LANGUAGE plpgsql;
DO $$
BEGIN
IF NOT EXISTS (
SELECT *
FROM pg_catalog.pg_user
WHERE usename = 'geocoder_api') THEN
CREATE USER geocoder_api;
END IF;
GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_geocoder_server TO geocoder_api;
GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO geocoder_api;
GRANT USAGE ON SCHEMA cdb_geocoder_server TO geocoder_api;
GRANT USAGE ON SCHEMA public TO geocoder_api;
GRANT SELECT ON ALL TABLES IN SCHEMA public TO geocoder_api;
END$$;

View File

@@ -1,6 +1,6 @@
# cdb geocoder server extension
comment = 'CartoDB server geocoder extension'
default_version = '0.1.0'
default_version = '0.2.0'
requires = 'plpythonu, postgis, cdb_geocoder'
superuser = true
schema = cdb_geocoder_server

View File

@@ -1,2 +1,3 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "CREATE EXTENSION cdb_geocoder_server" to load this file. \quit

View File

@@ -1,2 +1,3 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "CREATE EXTENSION cdb_geocoder_server" to load this file. \quit

View File

@@ -54,4 +54,4 @@ RETURNS boolean AS $$
'redis_metrics_connection': redis_metrics_connection,
}
return True
$$ LANGUAGE plpythonu;
$$ LANGUAGE plpythonu SECURITY DEFINER;

View File

@@ -22,4 +22,4 @@ RETURNS boolean AS $$
# --for this user session but...
GD[cache_key] = geocoder_config
return True
$$ LANGUAGE plpythonu;
$$ LANGUAGE plpythonu SECURITY DEFINER;

View File

@@ -1,5 +1,5 @@
-- Geocodes a street address given a searchtext and a state and/or country
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_street_point_v2(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']

View File

@@ -0,0 +1,3 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "CREATE EXTENSION cdb_geocoder_server" to load this file. \quit

View File

@@ -0,0 +1,57 @@
CREATE TYPE cdb_geocoder_server._redis_conf_params AS (
sentinel_host text,
sentinel_port int,
sentinel_master_id text,
redis_db text,
timeout float
);
-- Get the Redis configuration from the _conf table --
CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_redis_conf_v2(config_key text)
RETURNS cdb_geocoder_server._redis_conf_params AS $$
conf_query = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(config_key)
conf = plpy.execute(conf_query)[0]['conf']
if conf is None:
plpy.error("There is no redis configuration defined")
else:
import json
params = json.loads(conf)
return {
"sentinel_host": params['sentinel_host'],
"sentinel_port": params['sentinel_port'],
"sentinel_master_id": params['sentinel_master_id'],
"timeout": params['timeout'],
"redis_db": params['redis_db']
}
$$ LANGUAGE plpythonu;
-- Get the connection to redis from cache or create a new one
CREATE OR REPLACE FUNCTION cdb_geocoder_server._connect_to_redis(user_id text)
RETURNS boolean AS $$
cache_key = "redis_connection_{0}".format(user_id)
if cache_key in GD:
return False
else:
from cartodb_services.tools import RedisConnection
metadata_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port,
c.sentinel_master_id, c.timeout, c.redis_db
from cdb_geocoder_server._get_redis_conf_v2('redis_metadata_config') c;""")[0]
metrics_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port,
c.sentinel_master_id, c.timeout, c.redis_db
from cdb_geocoder_server._get_redis_conf_v2('redis_metrics_config') c;""")[0]
redis_metadata_connection = RedisConnection(metadata_config_params['sentinel_host'],
metadata_config_params['sentinel_port'],
metadata_config_params['sentinel_master_id'],
timeout=metadata_config_params['timeout'],
redis_db=metadata_config_params['redis_db']).redis_connection()
redis_metrics_connection = RedisConnection(metrics_config_params['sentinel_host'],
metrics_config_params['sentinel_port'],
metrics_config_params['sentinel_master_id'],
timeout=metrics_config_params['timeout'],
redis_db=metrics_config_params['redis_db']).redis_connection()
GD[cache_key] = {
'redis_metadata_connection': redis_metadata_connection,
'redis_metrics_connection': redis_metrics_connection,
}
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;

View File

@@ -0,0 +1,25 @@
-- Get the Redis configuration from the _conf table --
CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_geocoder_config(username text, orgname text)
RETURNS boolean AS $$
cache_key = "user_geocoder_config_{0}".format(username)
if cache_key in GD:
return False
else:
import json
from cartodb_services.metrics import GeocoderConfig
plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf']
if not heremaps_conf_json:
heremaps_app_id = None
heremaps_app_code = None
else:
heremaps_conf = json.loads(heremaps_conf_json)
heremaps_app_id = heremaps_conf['app_id']
heremaps_app_code = heremaps_conf['app_code']
geocoder_config = GeocoderConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code)
# --Think about the security concerns with this kind of global cache, it should be only available
# --for this user session but...
GD[cache_key] = geocoder_config
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;

View File

@@ -0,0 +1,84 @@
-- Geocodes a street address given a searchtext and a state and/or country
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_street_point_v2(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_geocoder_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
if user_geocoder_config.heremaps_geocoder:
here_plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
elif user_geocoder_config.google_geocoder:
google_plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
else:
plpy.error('Requested geocoder is not available')
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from cartodb_services.here import HereMapsGeocoder
from cartodb_services.metrics import QuotaService
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
# -- Check the quota
quota_service = QuotaService(user_geocoder_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reach the limit of your quota')
try:
geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code)
coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country)
if coordinates:
quota_service.increment_success_geocoder_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
return point['st_setsrid']
else:
quota_service.increment_empty_geocoder_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_geocoder_use()
error_msg = 'There was an error trying to geocode using here maps geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_geocoder_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from cartodb_services.google import GoogleMapsGeocoder
from cartodb_services.metrics import QuotaService
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
geocoder = GoogleMapsGeocoder(user_geocoder_config.google_client_id, user_geocoder_config.google_api_key)
coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country)
if coordinates:
quota_service.increment_success_geocoder_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
return point['st_setsrid']
else:
quota_service.increment_empty_geocoder_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_geocoder_use()
error_msg = 'There was an error trying to geocode using google maps geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_geocoder_use()
$$ LANGUAGE plpythonu;

View File

@@ -0,0 +1,37 @@
-- Interface of the server extension
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text)
RETURNS Geometry AS $$
plpy.debug('Entering cdb_geocode_admin0_polygons')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_admin0_polygon($1) AS mypolygon", ["text"])
rv = plpy.execute(plan, [country_name], 1)
plpy.debug('Returning from Returning from cdb_geocode_admin0_polygons')
return rv[0]["mypolygon"]
$$ LANGUAGE plpythonu;
--------------------------------------------------------------------------------
-- Implementation of the server extension
-- Note: these functions depend on the cdb_geocoder extension
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_admin0_polygon(country_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT n.the_geom as geom INTO ret
FROM (SELECT q, lower(regexp_replace(q, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text x
FROM (SELECT country_name q) g) d
LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x
LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.the_geom, s.adm0_a3;
RETURN ret;
END
$$ LANGUAGE plpgsql;

View File

@@ -0,0 +1,89 @@
-- Interfacess of the server extension
---- cdb_geocode_admin1_polygon(admin1_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text)
RETURNS Geometry AS $$
plpy.debug('Entering cdb_geocode_admin1_polygon(admin1_name text)')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_admin1_polygon($1) AS mypolygon", ["text"])
rv = plpy.execute(plan, [admin1_name], 1)
plpy.debug('Returning from Returning from cdb_geocode_admin1_polygons')
return rv[0]["mypolygon"]
$$ LANGUAGE plpythonu;
---- cdb_geocode_admin1_polygon(admin1_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text)
RETURNS Geometry AS $$
plpy.debug('Entering cdb_geocode_admin1_polygon(admin1_name text, country_name text)')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_admin1_polygon($1, $2) AS mypolygon", ["text", "text"])
rv = plpy.execute(plan, [admin1_name, country_name], 1)
plpy.debug('Returning from Returning from cdb_geocode_admin1_polygon(admin1_name text, country_name text)')
return rv[0]["mypolygon"]
$$ LANGUAGE plpythonu;
--------------------------------------------------------------------------------
-- Implementation of the server extension
-- Note: these functions depend on the cdb_geocoder extension
---- cdb_geocode_admin1_polygon(admin1_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_admin1_polygon(admin1_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
SELECT q, (
SELECT the_geom
FROM global_province_polygons
WHERE d.c = ANY (synonyms)
ORDER BY frequency DESC LIMIT 1
) geom
FROM (
SELECT
trim(replace(lower(admin1_name),'.',' ')) c, admin1_name q
) d
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
---- cdb_geocode_admin1_polygon(admin1_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_admin1_polygon(admin1_name text, country_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
WITH p AS (SELECT r.c, r.q, (SELECT iso3 FROM country_decoder WHERE lower(country_name) = ANY (synonyms)) i FROM (SELECT trim(replace(lower(admin1_name),'.',' ')) c, country_name q) r)
SELECT
geom INTO ret
FROM (
SELECT
q, (
SELECT the_geom
FROM global_province_polygons
WHERE p.c = ANY (synonyms)
AND iso3 = p.i
ORDER BY frequency DESC LIMIT 1
) geom
FROM p) n;
RETURN ret;
END
$$ LANGUAGE plpgsql;

View File

@@ -0,0 +1,121 @@
-- Interfacess of the server extension
---- cdb_geocode_namedplace_point(city_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text)
RETURNS Geometry AS $$
plpy.debug('Entering cdb_geocode_namedplace_point(city_name text)')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_namedplace_point($1) AS mypoint", ["text"])
rv = plpy.execute(plan, [city_name], 1)
plpy.debug('Returning from Returning from geocode_namedplace')
return rv[0]["mypoint"]
$$ LANGUAGE plpythonu;
---- cdb_geocode_namedplace_point(city_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text)
RETURNS Geometry AS $$
plpy.debug('Entering cdb_geocode_namedplace_point(city_name text, country_name text)')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_namedplace_point($1, $2) AS mypoint", ["text", "text"])
rv = plpy.execute(plan, [city_name, country_name], 1)
plpy.debug('Returning from Returning from geocode_namedplace')
return rv[0]["mypoint"]
$$ LANGUAGE plpythonu;
---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text)
RETURNS Geometry AS $$
plpy.debug('Entering cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_namedplace_point($1, $2, $3) AS mypoint", ["text", "text", "text"])
rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1)
plpy.debug('Returning from Returning from geocode_namedplace')
return rv[0]["mypoint"]
$$ LANGUAGE plpythonu;
--------------------------------------------------------------------------------
-- Implementation of the server extension
-- Note: these functions depend on the cdb_geocoder extension
---- cdb_geocode_namedplace_point(city_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_namedplace_point(city_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
WITH best AS (SELECT s AS q, (SELECT the_geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT city_name as s) p),
next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT city_name as s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL))
SELECT q, geom, TRUE AS success FROM best WHERE geom IS NOT NULL
UNION ALL
SELECT q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
---- cdb_geocode_namedplace_point(city_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_namedplace_point(city_name text, country_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
WITH p AS (SELECT r.s, r.c, (SELECT iso2 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT city_name AS s, country_name::text AS c) r),
best AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p),
next AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE c = p.c AND geom IS NOT NULL))
SELECT geom FROM best WHERE geom IS NOT NULL
UNION ALL
SELECT geom FROM next
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
WITH inputcountry AS (
SELECT iso2 as isoTwo FROM country_decoder WHERE lower(country_name) = ANY (synonyms) LIMIT 1
),
p AS (
SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder, inputcountry WHERE lower(r.a1) = ANY (synonyms) AND admin1_decoder.iso2 = inputcountry.isoTwo LIMIT 1) i FROM (SELECT city_name AS s, admin1_name::text AS a1) r),
best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p),
next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL))
SELECT geom FROM best WHERE geom IS NOT NULL
UNION ALL
SELECT geom FROM next
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;

View File

@@ -0,0 +1,162 @@
-- Interface of the server extension
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_point(username text, orgname text, code text)
RETURNS Geometry AS $$
plpy.debug('Entering _cdb_geocode_postalcode_point')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_point($1) AS point", ["text"])
rv = plpy.execute(plan, [code], 1)
plpy.debug('Returning from _cdb_geocode_postalcode_point')
return rv[0]["point"]
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text)
RETURNS Geometry AS $$
plpy.debug('Entering _cdb_geocode_postalcode_point')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_point($1, $2) AS point", ["TEXT", "TEXT"])
rv = plpy.execute(plan, [code, country], 1)
plpy.debug('Returning from _cdb_geocode_postalcode_point')
return rv[0]["point"]
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text)
RETURNS Geometry AS $$
plpy.debug('Entering _cdb_geocode_postalcode_polygon')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_polygon($1) AS polygon", ["text"])
rv = plpy.execute(plan, [code], 1)
plpy.debug('Returning from _cdb_geocode_postalcode_polygon')
return rv[0]["polygon"]
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text)
RETURNS Geometry AS $$
plpy.debug('Entering _cdb_geocode_postalcode_point')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_polygon($1, $2) AS polygon", ["TEXT", "TEXT"])
rv = plpy.execute(plan, [code, country], 1)
plpy.debug('Returning from _cdb_geocode_postalcode_point')
return rv[0]["polygon"]
$$ LANGUAGE plpythonu;
--------------------------------------------------------------------------------
-- Implementation of the server extension
-- Note: these functions depend on the cdb_geocoder extension
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_point(code text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
SELECT
q, (
SELECT the_geom
FROM global_postal_code_points
WHERE postal_code = upper(d.q)
LIMIT 1
) geom
FROM (SELECT code q) d
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_point(code text, country text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
SELECT
q, (
SELECT the_geom
FROM global_postal_code_points
WHERE postal_code = upper(d.q)
AND iso3 = (
SELECT iso3 FROM country_decoder WHERE
lower(country) = ANY (synonyms) LIMIT 1
)
LIMIT 1
) geom
FROM (SELECT code q) d
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_polygon(code text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
SELECT
q, (
SELECT the_geom
FROM global_postal_code_polygons
WHERE postal_code = upper(d.q)
LIMIT 1
) geom
FROM (SELECT code q) d
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_polygon(code text, country text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
SELECT
q, (
SELECT the_geom
FROM global_postal_code_polygons
WHERE postal_code = upper(d.q)
AND iso3 = (
SELECT iso3 FROM country_decoder WHERE
lower(country) = ANY (synonyms) LIMIT 1
)
LIMIT 1
) geom
FROM (SELECT code q) d
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;

View File

@@ -0,0 +1,49 @@
-- Interface of the server extension
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text)
RETURNS Geometry AS $$
plpy.debug('Entering _cdb_geocode_ipaddress_point')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_ipaddress_point($1) AS point", ["TEXT"])
rv = plpy.execute(plan, [ip], 1)
plpy.debug('Returning from _cdb_geocode_ipaddress_point')
return rv[0]["point"]
$$ LANGUAGE plpythonu;
--------------------------------------------------------------------------------
-- Implementation of the server extension
-- Note: these functions depend on the cdb_geocoder extension
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_ipaddress_point(ip text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
new_ip INET;
BEGIN
BEGIN
IF family(ip::inet) = 6 THEN
new_ip := ip::inet;
ELSE
new_ip := ('::ffff:' || ip)::inet;
END IF;
EXCEPTION WHEN OTHERS THEN
SELECT NULL as geom INTO ret;
RETURN ret;
END;
WITH
ips AS (SELECT ip s, new_ip net),
matches AS (SELECT s, (SELECT the_geom FROM ip_address_locations WHERE network_start_ip <= ips.net ORDER BY network_start_ip DESC LIMIT 1) geom FROM ips)
SELECT geom INTO ret
FROM matches;
RETURN ret;
END
$$ LANGUAGE plpgsql;

View File

@@ -0,0 +1,15 @@
DO $$
BEGIN
IF NOT EXISTS (
SELECT *
FROM pg_catalog.pg_user
WHERE usename = 'geocoder_api') THEN
CREATE USER geocoder_api;
END IF;
GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_geocoder_server TO geocoder_api;
GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO geocoder_api;
GRANT USAGE ON SCHEMA cdb_geocoder_server TO geocoder_api;
GRANT USAGE ON SCHEMA public TO geocoder_api;
GRANT SELECT ON ALL TABLES IN SCHEMA public TO geocoder_api;
END$$;

View File

@@ -3,7 +3,7 @@ SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_street_point_v2'
AND proname = 'cdb_geocode_street_point'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, text');
exists
--------

View File

@@ -3,5 +3,5 @@ SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_street_point_v2'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, text');
AND proname = 'cdb_geocode_street_point'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, text');

View File

@@ -0,0 +1,30 @@
-- Install dependencies
CREATE EXTENSION postgis;
CREATE EXTENSION schema_triggers;
CREATE EXTENSION plpythonu;
CREATE EXTENSION cartodb;
CREATE EXTENSION cdb_geocoder;
-- Install the extension
CREATE EXTENSION cdb_geocoder_server;
-- Mock the redis server connection to point to this very test db
SELECT cartodb.cdb_conf_setconf('redis_conf', '{"sentinel_host": "localhost", "sentinel_port": 26739, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}');
cdb_conf_setconf
------------------
(1 row)
-- Mock the varnish invalidation function
-- (used by cdb_geocoder tests)
CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$
BEGIN
RETURN;
END
$$
LANGUAGE plpgsql;
-- Set user quota
SELECT cartodb.CDB_SetUserQuotaInBytes(0);
cdb_setuserquotainbytes
-------------------------
0
(1 row)

View File

@@ -0,0 +1,12 @@
-- Check for namedplaces signatures
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_street_point_v2'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, text');
exists
--------
t
(1 row)

View File

@@ -0,0 +1,47 @@
-- Check that the public function is callable, even with no data
-- It should return NULL
SELECT cdb_geocoder_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'Spain');
cdb_geocode_admin0_polygon
----------------------------
(1 row)
-- Insert some dummy synonym
INSERT INTO admin0_synonyms (name, adm0_a3) VALUES ('Spain', 'ESP');
-- Insert some dummy geometry to return
INSERT INTO ne_admin0_v3 (adm0_a3, the_geom) VALUES('ESP', ST_GeomFromText(
'POLYGON((-71.1031880899493 42.3152774590236,
-71.1031627617667 42.3152960829043,
-71.102923838298 42.3149156848307,
-71.1031880899493 42.3152774590236))',4326)
);
-- This should return the polygon inserted above
SELECT cdb_geocoder_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'Spain');
cdb_geocode_admin0_polygon
--------------------------------------------------------------------------------------------------------------------------------------------------------------------
0103000020E61000000100000004000000D0EA37A29AC651C00FD603035B284540FEFCFB379AC651C0C0503E9F5B284540FFDDDD4D96C651C033AC3B284F284540D0EA37A29AC651C00FD603035B284540
(1 row)
-- Check for admin0 signatures
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_admin0_polygon'
AND oidvectortypes(p.proargtypes) = 'text, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_admin0_polygon'
AND oidvectortypes(p.proargtypes) = 'text');
exists
--------
t
(1 row)

View File

@@ -0,0 +1,81 @@
-- Check that the public function is callable, even with no data
-- It should return NULL
SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California');
cdb_geocode_admin1_polygon
----------------------------
(1 row)
SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California', 'United States');
cdb_geocode_admin1_polygon
----------------------------
(1 row)
-- Insert dummy data into country decoder table
INSERT INTO country_decoder (synonyms, iso3) VALUES (Array['united states'], 'USA');
-- Insert some dummy data and geometry to return
INSERT INTO global_province_polygons (synonyms, iso3, the_geom) VALUES (Array['california'], 'USA', ST_GeomFromText(
'POLYGON((-71.1031880899493 42.3152774590236,
-71.1031627617667 42.3152960829043,
-71.102923838298 42.3149156848307,
-71.1031880899493 42.3152774590236))',4326)
);
-- This should return the polygon inserted above
SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California');
cdb_geocode_admin1_polygon
--------------------------------------------------------------------------------------------------------------------------------------------------------------------
0103000020E61000000100000004000000D0EA37A29AC651C00FD603035B284540FEFCFB379AC651C0C0503E9F5B284540FFDDDD4D96C651C033AC3B284F284540D0EA37A29AC651C00FD603035B284540
(1 row)
SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California', 'United States');
cdb_geocode_admin1_polygon
--------------------------------------------------------------------------------------------------------------------------------------------------------------------
0103000020E61000000100000004000000D0EA37A29AC651C00FD603035B284540FEFCFB379AC651C0C0503E9F5B284540FFDDDD4D96C651C033AC3B284F284540D0EA37A29AC651C00FD603035B284540
(1 row)
-- Check for admin1 signatures
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_admin1_polygon'
AND oidvectortypes(p.proargtypes) = 'text, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_admin1_polygon'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_admin1_polygon'
AND oidvectortypes(p.proargtypes) = 'text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_admin1_polygon'
AND oidvectortypes(p.proargtypes) = 'text, text');
exists
--------
t
(1 row)

View File

@@ -0,0 +1,136 @@
-- Check that the public function is callable, even with no data
-- It should return NULL
SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx');
cdb_geocode_namedplace_point
------------------------------
(1 row)
SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain');
cdb_geocode_namedplace_point
------------------------------
(1 row)
SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain');
cdb_geocode_namedplace_point
------------------------------
(1 row)
-- Insert dummy data into points table
INSERT INTO global_cities_points_limited (geoname_id, name, iso2, admin1, admin2, population, lowername, the_geom) VALUES (3128760, 'Elche', 'ES', 'Valencia', 'AL', 34534, 'elche', ST_GeomFromText(
'POINT(0.6983 39.26787)',4326)
);
-- Insert dummy data into alternates table
INSERT INTO global_cities_alternates_limited (geoname_id, name, preferred, lowername, admin1_geonameid, iso2, admin1, the_geom) VALUES (3128760, 'Elx', true, 'elx', '000000', 'ES', 'Valencia', ST_GeomFromText(
'POINT(0.6983 39.26787)',4326)
);
-- Insert dummy data into country decoder table
INSERT INTO country_decoder (synonyms, iso2) VALUES (Array['spain'], 'ES');
-- Insert dummy data into admin1 decoder table
INSERT INTO admin1_decoder (admin1, synonyms, iso2) VALUES ('Valencia', Array['valencia', 'Valencia'], 'ES');
-- This should return the point inserted above
SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx');
cdb_geocode_namedplace_point
----------------------------------------------------
0101000020E6100000637FD93D7958E63F2ECA6C9049A24340
(1 row)
SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche');
cdb_geocode_namedplace_point
----------------------------------------------------
0101000020E6100000637FD93D7958E63F2ECA6C9049A24340
(1 row)
SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain');
cdb_geocode_namedplace_point
----------------------------------------------------
0101000020E6100000637FD93D7958E63F2ECA6C9049A24340
(1 row)
SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'Spain');
cdb_geocode_namedplace_point
----------------------------------------------------
0101000020E6100000637FD93D7958E63F2ECA6C9049A24340
(1 row)
SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain');
cdb_geocode_namedplace_point
----------------------------------------------------
0101000020E6100000637FD93D7958E63F2ECA6C9049A24340
(1 row)
SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'valencia', 'Spain');
cdb_geocode_namedplace_point
----------------------------------------------------
0101000020E6100000637FD93D7958E63F2ECA6C9049A24340
(1 row)
-- Check for namedplaces signatures
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_namedplace_point'
AND oidvectortypes(p.proargtypes) = 'text, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_namedplace_point'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_namedplace_point'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_namedplace_point'
AND oidvectortypes(p.proargtypes) = 'text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_namedplace_point'
AND oidvectortypes(p.proargtypes) = 'text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_namedplace_point'
AND oidvectortypes(p.proargtypes) = 'text, text, text');
exists
--------
t
(1 row)

View File

@@ -0,0 +1,163 @@
-- Make sure dbs are clean
DELETE FROM global_postal_code_points;
DELETE FROM global_postal_code_polygons;
DELETE FROM country_decoder;
DELETE FROM available_services;
DELETE FROM admin0_synonyms;
-- Check that the public function is callable, even with no data
-- It should return NULL
SELECT cdb_geocoder_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204');
cdb_geocode_postalcode_point
------------------------------
(1 row)
-- Insert dummy data into ip_address_locations
INSERT INTO global_postal_code_points (the_geom, iso3, postal_code, postal_code_num) VALUES (
'0101000020E61000000000000000E040408036B47414764840',
'ESP',
'03204',
3204
);
INSERT INTO global_postal_code_polygons (the_geom, iso3, postal_code, postal_code_num) VALUES (
'0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040',
'ESP',
'03204',
3204
);
INSERT INTO country_decoder (iso3, synonyms) VALUES (
'ESP',
Array['spain', 'Spain', 'ESP']
);
INSERT INTO available_services (adm0_a3, admin0, postal_code_points, postal_code_polygons) VALUES (
'ESP',
't',
't',
't'
);
INSERT INTO admin0_synonyms (adm0_a3, name, name_, rank) VALUES (
'ESP',
'Spain',
'spain',
3
);
-- This should return the polygon inserted above
SELECT cdb_geocoder_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204');
cdb_geocode_postalcode_point
----------------------------------------------------
0101000020E61000000000000000E040408036B47414764840
(1 row)
SELECT cdb_geocoder_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204', 'spain');
cdb_geocode_postalcode_point
----------------------------------------------------
0101000020E61000000000000000E040408036B47414764840
(1 row)
SELECT cdb_geocoder_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', '03204');
cdb_geocode_postalcode_polygon
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040
(1 row)
SELECT cdb_geocoder_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', '03204', 'spain');
cdb_geocode_postalcode_polygon
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040
(1 row)
-- Clean dbs
DELETE FROM global_postal_code_points;
DELETE FROM global_postal_code_polygons;
DELETE FROM country_decoder;
DELETE FROM available_services;
DELETE FROM admin0_synonyms;
-- Check for namedplaces signatures (point and polygon)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_postalcode_point'
AND oidvectortypes(p.proargtypes) = 'text, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_postalcode_point'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_postalcode_polygon'
AND oidvectortypes(p.proargtypes) = 'text, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_postalcode_polygon'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_postalcode_point'
AND oidvectortypes(p.proargtypes) = 'text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_postalcode_point'
AND oidvectortypes(p.proargtypes) = 'text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_postalcode_polygon'
AND oidvectortypes(p.proargtypes) = 'text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_postalcode_polygon'
AND oidvectortypes(p.proargtypes) = 'text, text');
exists
--------
t
(1 row)

View File

@@ -0,0 +1,40 @@
-- Check that the public function is callable, even with no data
-- It should return NULL
SELECT cdb_geocoder_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', '0.0.0.0');
cdb_geocode_ipaddress_point
-----------------------------
(1 row)
-- Insert dummy data into ip_address_locations
INSERT INTO ip_address_locations VALUES ('::ffff:0.0.0.0'::inet, (ST_SetSRID(ST_MakePoint('40.40', '3.71'), 4326)));
-- This should return the polygon inserted above
SELECT cdb_geocoder_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', '0.0.0.0');
cdb_geocode_ipaddress_point
----------------------------------------------------
0101000020E61000003333333333334440AE47E17A14AE0D40
(1 row)
-- Check for namedplaces signatures (point and polygon)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_ipaddress_point'
AND oidvectortypes(p.proargtypes) = 'text, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_ipaddress_point'
AND oidvectortypes(p.proargtypes) = 'text');
exists
--------
t
(1 row)

View File

@@ -0,0 +1,5 @@
REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_geocoder_server FROM geocoder_api;
REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA public FROM geocoder_api;
REVOKE USAGE ON SCHEMA cdb_geocoder_server FROM geocoder_api;
REVOKE USAGE ON SCHEMA public FROM geocoder_api;
REVOKE SELECT ON ALL TABLES IN SCHEMA public FROM geocoder_api;

View File

@@ -0,0 +1,24 @@
-- Install dependencies
CREATE EXTENSION postgis;
CREATE EXTENSION schema_triggers;
CREATE EXTENSION plpythonu;
CREATE EXTENSION cartodb;
CREATE EXTENSION cdb_geocoder;
-- Install the extension
CREATE EXTENSION cdb_geocoder_server;
-- Mock the redis server connection to point to this very test db
SELECT cartodb.cdb_conf_setconf('redis_conf', '{"sentinel_host": "localhost", "sentinel_port": 26739, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}');
-- Mock the varnish invalidation function
-- (used by cdb_geocoder tests)
CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$
BEGIN
RETURN;
END
$$
LANGUAGE plpgsql;
-- Set user quota
SELECT cartodb.CDB_SetUserQuotaInBytes(0);

View File

@@ -0,0 +1,7 @@
-- Check for namedplaces signatures
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_street_point_v2'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, text');

View File

@@ -0,0 +1,32 @@
-- Check that the public function is callable, even with no data
-- It should return NULL
SELECT cdb_geocoder_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'Spain');
-- Insert some dummy synonym
INSERT INTO admin0_synonyms (name, adm0_a3) VALUES ('Spain', 'ESP');
-- Insert some dummy geometry to return
INSERT INTO ne_admin0_v3 (adm0_a3, the_geom) VALUES('ESP', ST_GeomFromText(
'POLYGON((-71.1031880899493 42.3152774590236,
-71.1031627617667 42.3152960829043,
-71.102923838298 42.3149156848307,
-71.1031880899493 42.3152774590236))',4326)
);
-- This should return the polygon inserted above
SELECT cdb_geocoder_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'Spain');
-- Check for admin0 signatures
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_admin0_polygon'
AND oidvectortypes(p.proargtypes) = 'text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_admin0_polygon'
AND oidvectortypes(p.proargtypes) = 'text');

View File

@@ -0,0 +1,48 @@
-- Check that the public function is callable, even with no data
-- It should return NULL
SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California');
SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California', 'United States');
-- Insert dummy data into country decoder table
INSERT INTO country_decoder (synonyms, iso3) VALUES (Array['united states'], 'USA');
-- Insert some dummy data and geometry to return
INSERT INTO global_province_polygons (synonyms, iso3, the_geom) VALUES (Array['california'], 'USA', ST_GeomFromText(
'POLYGON((-71.1031880899493 42.3152774590236,
-71.1031627617667 42.3152960829043,
-71.102923838298 42.3149156848307,
-71.1031880899493 42.3152774590236))',4326)
);
-- This should return the polygon inserted above
SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California');
SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California', 'United States');
-- Check for admin1 signatures
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_admin1_polygon'
AND oidvectortypes(p.proargtypes) = 'text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_admin1_polygon'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_admin1_polygon'
AND oidvectortypes(p.proargtypes) = 'text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_admin1_polygon'
AND oidvectortypes(p.proargtypes) = 'text, text');

View File

@@ -0,0 +1,72 @@
-- Check that the public function is callable, even with no data
-- It should return NULL
SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx');
SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain');
SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain');
-- Insert dummy data into points table
INSERT INTO global_cities_points_limited (geoname_id, name, iso2, admin1, admin2, population, lowername, the_geom) VALUES (3128760, 'Elche', 'ES', 'Valencia', 'AL', 34534, 'elche', ST_GeomFromText(
'POINT(0.6983 39.26787)',4326)
);
-- Insert dummy data into alternates table
INSERT INTO global_cities_alternates_limited (geoname_id, name, preferred, lowername, admin1_geonameid, iso2, admin1, the_geom) VALUES (3128760, 'Elx', true, 'elx', '000000', 'ES', 'Valencia', ST_GeomFromText(
'POINT(0.6983 39.26787)',4326)
);
-- Insert dummy data into country decoder table
INSERT INTO country_decoder (synonyms, iso2) VALUES (Array['spain'], 'ES');
-- Insert dummy data into admin1 decoder table
INSERT INTO admin1_decoder (admin1, synonyms, iso2) VALUES ('Valencia', Array['valencia', 'Valencia'], 'ES');
-- This should return the point inserted above
SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx');
SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche');
SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain');
SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'Spain');
SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain');
SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'valencia', 'Spain');
-- Check for namedplaces signatures
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_namedplace_point'
AND oidvectortypes(p.proargtypes) = 'text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_namedplace_point'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_namedplace_point'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_namedplace_point'
AND oidvectortypes(p.proargtypes) = 'text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_namedplace_point'
AND oidvectortypes(p.proargtypes) = 'text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_namedplace_point'
AND oidvectortypes(p.proargtypes) = 'text, text, text');

View File

@@ -0,0 +1,117 @@
-- Make sure dbs are clean
DELETE FROM global_postal_code_points;
DELETE FROM global_postal_code_polygons;
DELETE FROM country_decoder;
DELETE FROM available_services;
DELETE FROM admin0_synonyms;
-- Check that the public function is callable, even with no data
-- It should return NULL
SELECT cdb_geocoder_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204');
-- Insert dummy data into ip_address_locations
INSERT INTO global_postal_code_points (the_geom, iso3, postal_code, postal_code_num) VALUES (
'0101000020E61000000000000000E040408036B47414764840',
'ESP',
'03204',
3204
);
INSERT INTO global_postal_code_polygons (the_geom, iso3, postal_code, postal_code_num) VALUES (
'0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040',
'ESP',
'03204',
3204
);
INSERT INTO country_decoder (iso3, synonyms) VALUES (
'ESP',
Array['spain', 'Spain', 'ESP']
);
INSERT INTO available_services (adm0_a3, admin0, postal_code_points, postal_code_polygons) VALUES (
'ESP',
't',
't',
't'
);
INSERT INTO admin0_synonyms (adm0_a3, name, name_, rank) VALUES (
'ESP',
'Spain',
'spain',
3
);
-- This should return the polygon inserted above
SELECT cdb_geocoder_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204');
SELECT cdb_geocoder_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204', 'spain');
SELECT cdb_geocoder_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', '03204');
SELECT cdb_geocoder_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', '03204', 'spain');
-- Clean dbs
DELETE FROM global_postal_code_points;
DELETE FROM global_postal_code_polygons;
DELETE FROM country_decoder;
DELETE FROM available_services;
DELETE FROM admin0_synonyms;
-- Check for namedplaces signatures (point and polygon)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_postalcode_point'
AND oidvectortypes(p.proargtypes) = 'text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_postalcode_point'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_postalcode_polygon'
AND oidvectortypes(p.proargtypes) = 'text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_postalcode_polygon'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_postalcode_point'
AND oidvectortypes(p.proargtypes) = 'text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_postalcode_point'
AND oidvectortypes(p.proargtypes) = 'text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_postalcode_polygon'
AND oidvectortypes(p.proargtypes) = 'text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_postalcode_polygon'
AND oidvectortypes(p.proargtypes) = 'text, text');

View File

@@ -0,0 +1,24 @@
-- Check that the public function is callable, even with no data
-- It should return NULL
SELECT cdb_geocoder_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', '0.0.0.0');
-- Insert dummy data into ip_address_locations
INSERT INTO ip_address_locations VALUES ('::ffff:0.0.0.0'::inet, (ST_SetSRID(ST_MakePoint('40.40', '3.71'), 4326)));
-- This should return the polygon inserted above
SELECT cdb_geocoder_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', '0.0.0.0');
-- Check for namedplaces signatures (point and polygon)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_ipaddress_point'
AND oidvectortypes(p.proargtypes) = 'text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_ipaddress_point'
AND oidvectortypes(p.proargtypes) = 'text');

View File

@@ -0,0 +1,5 @@
REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_geocoder_server FROM geocoder_api;
REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA public FROM geocoder_api;
REVOKE USAGE ON SCHEMA cdb_geocoder_server FROM geocoder_api;
REVOKE USAGE ON SCHEMA public FROM geocoder_api;
REVOKE SELECT ON ALL TABLES IN SCHEMA public FROM geocoder_api;

View File

@@ -1,7 +0,0 @@
redis==2.10.5
python-dateutil==2.4.2
# Test
mock==1.3.0
mockredispy==2.9.0.11
nose==1.3.7

View File

@@ -0,0 +1 @@
from geocoder import GoogleMapsGeocoder

View File

@@ -16,11 +16,6 @@ class NoGeocodingParams(Exception):
return repr('No params for geocoding specified')
class EmptyGeocoderResponse(Exception):
def __str__(self):
return repr('The request could not be geocoded')
class MalformedResult(Exception):
def __str__(self):
return repr('Result structure is malformed')

View File

@@ -0,0 +1,50 @@
#!/usr/local/bin/python
# -*- coding: utf-8 -*-
import googlemaps
from exceptions import MalformedResult
class GoogleMapsGeocoder:
"""A Google Maps Geocoder wrapper for python"""
def __init__(self, client_id, client_secret):
self.client_id = self._clean_client_id(client_id)
self.client_secret = client_secret
self.geocoder = googlemaps.Client(
client_id=self.client_id, client_secret=self.client_secret)
def geocode(self, searchtext, city=None, state=None,
country=None):
try:
opt_params = self._build_optional_parameters(city, state, country)
results = self.geocoder.geocode(address=searchtext,
components=opt_params)
if results:
return self._extract_lng_lat_from_result(results[0])
else:
return []
except KeyError:
raise MalformedResult()
def _extract_lng_lat_from_result(self, result):
location = result['geometry']['location']
longitude = location['lng']
latitude = location['lat']
return [longitude, latitude]
def _build_optional_parameters(self, city=None, state=None,
country=None):
optional_params = {}
if city:
optional_params['locality'] = city
if state:
optional_params['administrative_area'] = state
if country:
optional_params['country'] = country
return optional_params
def _clean_client_id(self, client_id):
# Consistency with how the client_id is saved in metadata
return client_id.replace('client=', '')

View File

@@ -0,0 +1 @@
from geocoder import HereMapsGeocoder

View File

@@ -0,0 +1,21 @@
#!/usr/local/bin/python
# -*- coding: utf-8 -*-
import json
class BadGeocodingParams(Exception):
def __init__(self, value):
self.value = value
def __str__(self):
return repr('Bad geocoding params: ' + json.dumps(self.value))
class NoGeocodingParams(Exception):
def __str__(self):
return repr('No params for geocoding specified')
class MalformedResult(Exception):
def __str__(self):
return repr('Result structure is malformed')

View File

@@ -4,13 +4,10 @@
import json
import requests
from heremaps.heremapsexceptions import BadGeocodingParams
from heremaps.heremapsexceptions import EmptyGeocoderResponse
from heremaps.heremapsexceptions import NoGeocodingParams
from heremaps.heremapsexceptions import MalformedResult
from exceptions import *
class Geocoder:
class HereMapsGeocoder:
'A Here Maps Geocoder wrapper for python'
PRODUCTION_GEOCODE_JSON_URL = 'https://geocoder.api.here.com/6.2/geocode.json'
@@ -50,10 +47,6 @@ class Geocoder:
'strictlanguagemode'
] + ADDRESS_PARAMS
app_id = ''
app_code = ''
maxresults = ''
def __init__(self, app_id, app_code, maxresults=DEFAULT_MAXRESULTS,
gen=DEFAULT_GEN, host=PRODUCTION_GEOCODE_JSON_URL):
self.app_id = app_id
@@ -62,18 +55,28 @@ class Geocoder:
self.gen = gen
self.host = host
def geocode(self, params):
def geocode(self, **kwargs):
params = {}
for key, value in kwargs.iteritems():
if value:
params[key] = value
if not params:
raise NoGeocodingParams()
return self._execute_geocode(params)
def _execute_geocode(self, params):
if not set(params.keys()).issubset(set(self.ADDRESS_PARAMS)):
raise BadGeocodingParams(params)
response = self.perform_request(params)
try:
results = response['Response']['View'][0]['Result']
response = self._perform_request(params)
results = response['Response']['View'][0]['Result'][0]
return self._extract_lng_lat_from_result(results)
except IndexError:
raise EmptyGeocoderResponse()
return []
except KeyError:
raise MalformedResult()
return results
def perform_request(self, params):
def _perform_request(self, params):
request_params = {
'app_id': self.app_id,
'app_code': self.app_code,
@@ -87,21 +90,8 @@ class Geocoder:
else:
response.raise_for_status()
def geocode_address(self, **kwargs):
params = {}
for key, value in kwargs.iteritems():
if value:
params[key] = value
if not params:
raise NoGeocodingParams()
return self.geocode(params)
def extract_lng_lat_from_result(self, result):
try:
location = result['Location']
except KeyError:
raise MalformedResult()
def _extract_lng_lat_from_result(self, result):
location = result['Location']
longitude = location['DisplayPosition']['Longitude']
latitude = location['DisplayPosition']['Latitude']

View File

@@ -0,0 +1,3 @@
from config import GeocoderConfig, ConfigException
from quota import QuotaService
from user import UserMetricsService

View File

@@ -1,4 +1,4 @@
import user_service
from user import UserMetricsService
from datetime import date
@@ -8,7 +8,7 @@ class QuotaService:
def __init__(self, user_geocoder_config, redis_connection):
self._user_geocoder_config = user_geocoder_config
self._user_service = user_service.UserService(
self._user_service = UserMetricsService(
self._user_geocoder_config, redis_connection)
def check_user_quota(self):
@@ -32,19 +32,16 @@ class QuotaService:
self._user_service.increment_service_use(
self._user_geocoder_config.service_type, "success_responses",
amount=amount)
self.increment_total_geocoder_use(amount)
def increment_empty_geocoder_use(self, amount=1):
self._user_service.increment_service_use(
self._user_geocoder_config.service_type, "empty_responses",
amount=amount)
self.increment_total_geocoder_use(amount)
def increment_failed_geocoder_use(self, amount=1):
self._user_service.increment_service_use(
self._user_geocoder_config.service_type, "fail_responses",
amount=amount)
self.increment_total_geocoder_use(amount)
def increment_total_geocoder_use(self, amount=1):
self._user_service.increment_service_use(

View File

@@ -2,7 +2,7 @@ from datetime import date, timedelta
from dateutil.relativedelta import relativedelta
class UserService:
class UserMetricsService:
""" Class to manage all the user info """
SERVICE_GEOCODER_NOKIA = 'geocoder_here'

View File

@@ -0,0 +1 @@
from redis_tools import RedisConnection

View File

@@ -1,7 +1,7 @@
from redis.sentinel import Sentinel
class RedisHelper:
class RedisConnection:
REDIS_DEFAULT_USER_DB = 5
REDIS_DEFAULT_TIMEOUT = 2 #seconds

View File

@@ -0,0 +1,13 @@
redis==2.10.5
hiredis==0.1.5
# Dependency with incsv in the import
python-dateutil==2.2
googlemaps==2.4.2
# Dependency for googlemaps package
requests<=2.9.1
# Test
mock==1.3.0
mockredispy==2.9.0.11
nose==1.3.7
requests-mock==0.7.0

View File

@@ -1,5 +1,5 @@
"""
CartoDB Geocoder Python Library
CartoDB Services Python Library
See:
https://github.com/CartoDB/geocoder-api
@@ -8,11 +8,11 @@ https://github.com/CartoDB/geocoder-api
from setuptools import setup, find_packages
setup(
name='cartodb_geocoder',
name='cartodb_services',
version='0.0.1',
version='0.1.0',
description='CartoDB Geocoder Python Library',
description='CartoDB Services API Python Library',
url='https://github.com/CartoDB/geocoder-api',
@@ -29,7 +29,7 @@ setup(
'Programming Language :: Python :: 2.7',
],
keywords='maps api mapping tools geocoder',
keywords='maps api mapping tools geocoder routing',
packages=find_packages(exclude=['contrib', 'docs', 'tests']),

View File

@@ -1,41 +1,41 @@
import test_helper
from cartodb_geocoder import config_helper
from cartodb_services.metrics import GeocoderConfig, ConfigException
from unittest import TestCase
from nose.tools import assert_raises
from mockredis import MockRedis
from datetime import datetime, timedelta
class TestConfigHelper(TestCase):
class TestConfig(TestCase):
def setUp(self):
self.redis_conn = MockRedis()
def test_should_return_list_of_nokia_geocoder_config_if_its_ok(self):
test_helper.build_redis_user_config(self.redis_conn, 'test_user')
geocoder_config = config_helper.GeocoderConfig(self.redis_conn,
geocoder_config = GeocoderConfig(self.redis_conn,
'test_user', None,
'nokia_id', 'nokia_cod')
assert geocoder_config.heremaps_geocoder == True
assert geocoder_config.heremaps_geocoder is True
assert geocoder_config.geocoding_quota == 100
assert geocoder_config.soft_geocoding_limit == False
assert geocoder_config.soft_geocoding_limit is False
def test_should_return_list_of_nokia_geocoder_config_ok_for_org(self):
yesterday = datetime.today() - timedelta(days=1)
test_helper.build_redis_user_config(self.redis_conn, 'test_user')
test_helper.build_redis_org_config(self.redis_conn, 'test_org',
quota=200, end_date=yesterday)
geocoder_config = config_helper.GeocoderConfig(self.redis_conn,
geocoder_config = GeocoderConfig(self.redis_conn,
'test_user', 'test_org',
'nokia_id', 'nokia_cod')
assert geocoder_config.heremaps_geocoder == True
assert geocoder_config.heremaps_geocoder is True
assert geocoder_config.geocoding_quota == 200
assert geocoder_config.soft_geocoding_limit == False
assert geocoder_config.soft_geocoding_limit is False
assert geocoder_config.period_end_date.date() == yesterday.date()
def test_should_raise_configuration_exception_when_missing_nokia_geocoder_parameters(self):
test_helper.build_redis_user_config(self.redis_conn, 'test_user')
assert_raises(config_helper.ConfigException,
config_helper.GeocoderConfig,
assert_raises(ConfigException,
GeocoderConfig,
self.redis_conn, 'test_user',
None, None, None)

View File

@@ -0,0 +1,119 @@
#!/usr/local/bin/python
# -*- coding: utf-8 -*-
import unittest
import requests_mock
from cartodb_services.google import GoogleMapsGeocoder
from cartodb_services.google.exceptions import BadGeocodingParams
from cartodb_services.google.exceptions import NoGeocodingParams
from cartodb_services.google.exceptions import MalformedResult
requests_mock.Mocker.TEST_PREFIX = 'test_'
@requests_mock.Mocker()
class GoogleGeocoderTestCase(unittest.TestCase):
GOOGLE_MAPS_GEOCODER_URL = 'https://maps.googleapis.com/maps/api/geocode/json'
EMPTY_RESPONSE = """{
"results" : [],
"status" : "ZERO_RESULTS"
}"""
GOOD_RESPONSE = """{
"results" : [
{
"address_components" : [
{
"long_name" : "1600",
"short_name" : "1600",
"types" : [ "street_number" ]
},
{
"long_name" : "Amphitheatre Pkwy",
"short_name" : "Amphitheatre Pkwy",
"types" : [ "route" ]
},
{
"long_name" : "Mountain View",
"short_name" : "Mountain View",
"types" : [ "locality", "political" ]
},
{
"long_name" : "Santa Clara County",
"short_name" : "Santa Clara County",
"types" : [ "administrative_area_level_2", "political" ]
},
{
"long_name" : "California",
"short_name" : "CA",
"types" : [ "administrative_area_level_1", "political" ]
},
{
"long_name" : "United States",
"short_name" : "US",
"types" : [ "country", "political" ]
},
{
"long_name" : "94043",
"short_name" : "94043",
"types" : [ "postal_code" ]
}
],
"formatted_address" : "1600 Amphitheatre Parkway, Mountain View, CA 94043, USA",
"geometry" : {
"location" : {
"lat" : 37.4224764,
"lng" : -122.0842499
},
"location_type" : "ROOFTOP",
"viewport" : {
"northeast" : {
"lat" : 37.4238253802915,
"lng" : -122.0829009197085
},
"southwest" : {
"lat" : 37.4211274197085,
"lng" : -122.0855988802915
}
}
},
"place_id" : "ChIJ2eUgeAK6j4ARbn5u_wAGqWA",
"types" : [ "street_address" ]
}
],
"status" : "OK"
}"""
MALFORMED_RESPONSE = """{"manolo": "escobar"}"""
def setUp(self):
self.geocoder = GoogleMapsGeocoder('dummy_client_id',
'MgxyOFxjZXIyOGO52jJlMzEzY1Oqy4hsO49E')
def test_geocode_address_with_valid_params(self, req_mock):
req_mock.register_uri('GET', self.GOOGLE_MAPS_GEOCODER_URL,
text=self.GOOD_RESPONSE)
response = self.geocoder.geocode(
searchtext='Calle Eloy Gonzalo 27',
city='Madrid',
country='España')
self.assertEqual(response[0], -122.0842499)
self.assertEqual(response[1], 37.4224764)
def test_geocode_address_empty_response(self, req_mock):
req_mock.register_uri('GET', self.GOOGLE_MAPS_GEOCODER_URL,
text=self.EMPTY_RESPONSE)
result = self.geocoder.geocode(searchtext='lkajfñlasjfñ')
self.assertEqual(result, [])
def test_geocode_with_malformed_result(self, req_mock):
req_mock.register_uri('GET', self.GOOGLE_MAPS_GEOCODER_URL,
text=self.MALFORMED_RESPONSE)
with self.assertRaises(MalformedResult):
self.geocoder.geocode(
searchtext='Calle Eloy Gonzalo 27',
city='Madrid',
country='España')

View File

@@ -2,25 +2,28 @@
# -*- coding: utf-8 -*-
import unittest
import requests_mock
from heremaps import heremapsgeocoder
from heremaps.heremapsexceptions import BadGeocodingParams
from heremaps.heremapsexceptions import EmptyGeocoderResponse
from heremaps.heremapsexceptions import NoGeocodingParams
from heremaps.heremapsexceptions import MalformedResult
from cartodb_services.here import HereMapsGeocoder
from cartodb_services.here.exceptions import BadGeocodingParams
from cartodb_services.here.exceptions import NoGeocodingParams
from cartodb_services.here.exceptions import MalformedResult
requests_mock.Mocker.TEST_PREFIX = 'test_'
class GeocoderTestCase(unittest.TestCase):
EMPTY_RESPONSE = {
@requests_mock.Mocker()
class HereMapsGeocoderTestCase(unittest.TestCase):
EMPTY_RESPONSE = """{
"Response": {
"MetaInfo": {
"Timestamp": "2015-11-04T16:31:57.273+0000"
},
"View": []
}
}
}"""
GOOD_RESPONSE = {
GOOD_RESPONSE = """{
"Response": {
"MetaInfo": {
"Timestamp": "2015-11-04T16:30:32.187+0000"
@@ -28,7 +31,7 @@ class GeocoderTestCase(unittest.TestCase):
"View": [{
"_type": "SearchResultsViewType",
"ViewId": 0,
"Result": [{
"Result": {
"Relevance": 0.89,
"MatchLevel": "street",
"MatchQuality": {
@@ -57,7 +60,7 @@ class GeocoderTestCase(unittest.TestCase):
}
},
"Address": {
"Label": "Calle de Eloy Gonzalo, Madrid, España",
"Label": "Calle de Eloy Gonzalo, Madrid, Espana",
"Country": "ESP",
"State": "Comunidad de Madrid",
"County": "Madrid",
@@ -65,7 +68,7 @@ class GeocoderTestCase(unittest.TestCase):
"District": "Trafalgar",
"Street": "Calle de Eloy Gonzalo",
"AdditionalData": [{
"value": "España",
"value": "Espana",
"key": "CountryName"
},
{
@@ -78,45 +81,52 @@ class GeocoderTestCase(unittest.TestCase):
}]
}
}
}]
}
}]
}
}
}"""
MALFORMED_RESPONSE = """{"manolo": "escobar"}"""
def setUp(self):
self.geocoder = heremapsgeocoder.Geocoder(None, None)
self.geocoder = HereMapsGeocoder(None, None)
def test_geocode_address_with_valid_params(self):
self.geocoder.perform_request = lambda x: self.GOOD_RESPONSE
response = self.geocoder.geocode_address(
def test_geocode_address_with_valid_params(self, req_mock):
req_mock.register_uri('GET', HereMapsGeocoder.PRODUCTION_GEOCODE_JSON_URL,
text=self.GOOD_RESPONSE)
response = self.geocoder.geocode(
searchtext='Calle Eloy Gonzalo 27',
city='Madrid',
country='España')
def test_geocode_address_with_invalid_params(self):
self.assertEqual(response[0], -3.70126)
self.assertEqual(response[1], 40.43433)
def test_geocode_address_with_invalid_params(self, req_mock):
req_mock.register_uri('GET', HereMapsGeocoder.PRODUCTION_GEOCODE_JSON_URL,
text=self.GOOD_RESPONSE)
with self.assertRaises(BadGeocodingParams):
self.geocoder.geocode_address(
self.geocoder.geocode(
searchtext='Calle Eloy Gonzalo 27',
manolo='escobar')
def test_geocode_address_with_no_params(self):
def test_geocode_address_with_no_params(self, req_mock):
req_mock.register_uri('GET', HereMapsGeocoder.PRODUCTION_GEOCODE_JSON_URL,
text=self.GOOD_RESPONSE)
with self.assertRaises(NoGeocodingParams):
self.geocoder.geocode_address()
self.geocoder.geocode()
def test_geocode_address_empty_response(self):
self.geocoder.perform_request = lambda x: self.EMPTY_RESPONSE
with self.assertRaises(EmptyGeocoderResponse):
self.geocoder.geocode_address(searchtext='lkajfñlasjfñ')
def test_extract_lng_lat_from_result(self):
result = self.GOOD_RESPONSE['Response']['View'][0]['Result'][0]
coordinates = self.geocoder.extract_lng_lat_from_result(result)
self.assertEqual(coordinates[0], -3.70126)
self.assertEqual(coordinates[1], 40.43433)
def test_extract_lng_lat_from_result_with_malformed_result(self):
result = {'manolo': 'escobar'}
def test_geocode_address_empty_response(self, req_mock):
req_mock.register_uri('GET', HereMapsGeocoder.PRODUCTION_GEOCODE_JSON_URL,
text=self.EMPTY_RESPONSE)
result = self.geocoder.geocode(searchtext='lkajfñlasjfñ')
self.assertEqual(result, [])
def test_geocode_with_malformed_result(self, req_mock):
req_mock.register_uri('GET', HereMapsGeocoder.PRODUCTION_GEOCODE_JSON_URL,
text=self.MALFORMED_RESPONSE)
with self.assertRaises(MalformedResult):
self.geocoder.extract_lng_lat_from_result(result)
self.geocoder.geocode(
searchtext='Calle Eloy Gonzalo 27',
city='Madrid',
country='España')

View File

@@ -1,7 +1,7 @@
import test_helper
from mockredis import MockRedis
from cartodb_geocoder import quota_service
from cartodb_geocoder import config_helper
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import GeocoderConfig
from unittest import TestCase
from nose.tools import assert_raises
from datetime import datetime, date
@@ -14,10 +14,6 @@ class TestQuotaService(TestCase):
# organization user
# org:<orgname>:<service>:<metric>:YYYYMM:DD
# def increment_geocoder_uses(self, username, orgname=None,
# date=date.today(), service='geocoder_here',
# metric='success_responses', amount=20):
def setUp(self):
self.redis_conn = MockRedis()
@@ -68,28 +64,19 @@ class TestQuotaService(TestCase):
def test_should_check_user_increment_and_quota_check_correctly(self):
qs = self.__build_quota_service('test_user', quota=2)
qs.increment_success_geocoder_use()
assert qs.check_user_quota() == True
assert qs.check_user_quota() is True
qs.increment_success_geocoder_use(amount=2)
assert qs.check_user_quota() == False
assert qs.check_user_quota() is False
month = date.today().strftime('%Y%m')
name = 'user:test_user:geocoder_here:total_requests:{0}'.format(month)
total_requests = self.redis_conn.zscore(name, date.today().day)
assert total_requests == 3
def test_should_check_org_increment_and_quota_check_correctly(self):
qs = self.__build_quota_service('test_user', orgname='test_org',
quota=2)
qs.increment_success_geocoder_use()
assert qs.check_user_quota() == True
assert qs.check_user_quota() is True
qs.increment_success_geocoder_use(amount=2)
assert qs.check_user_quota() == False
assert qs.check_user_quota() is False
month = date.today().strftime('%Y%m')
org_name = 'org:test_org:geocoder_here:total_requests:{0}'.format(month)
org_total_requests = self.redis_conn.zscore(org_name, date.today().day)
assert org_total_requests == 3
user_name = 'user:test_user:geocoder_here:total_requests:{0}'.format(month)
user_total_requests = self.redis_conn.zscore(user_name, date.today().day)
assert user_total_requests == 3
def __build_quota_service(self, username, quota=100, service='heremaps',
orgname=None, soft_limit=False,
@@ -101,9 +88,9 @@ class TestQuotaService(TestCase):
if orgname:
test_helper.build_redis_org_config(self.redis_conn, orgname,
quota=quota, end_date=end_date)
geocoder_config = config_helper.GeocoderConfig(self.redis_conn,
username, orgname,
'nokia_id', 'nokia_cod')
return quota_service.QuotaService(geocoder_config,
redis_connection = self.redis_conn)
geocoder_config = GeocoderConfig(self.redis_conn,
username, orgname,
'nokia_id', 'nokia_cod')
return QuotaService(geocoder_config,
redis_connection = self.redis_conn)

View File

@@ -1,7 +1,7 @@
import test_helper
from mockredis import MockRedis
from cartodb_geocoder import user_service
from cartodb_geocoder import config_helper
from cartodb_services.metrics import UserMetricsService
from cartodb_services.metrics import GeocoderConfig
from datetime import datetime, date
from unittest import TestCase
from nose.tools import assert_raises
@@ -84,7 +84,7 @@ class TestUserService(TestCase):
if orgname:
test_helper.build_redis_org_config(self.redis_conn, orgname,
quota=quota, end_date=end_date)
geocoder_config = config_helper.GeocoderConfig(self.redis_conn,
username, orgname,
'nokia_id', 'nokia_cod')
return user_service.UserService(geocoder_config, self.redis_conn)
geocoder_config = GeocoderConfig(self.redis_conn,
username, orgname,
'nokia_id', 'nokia_cod')
return UserMetricsService(geocoder_config, self.redis_conn)

View File

@@ -1,4 +0,0 @@
requests==2.9.1
# Test
nose==1.3.7

View File

@@ -1,41 +0,0 @@
"""
A Here Maps API Python wrapper
See:
https://developer.here.com
https://github.com/CartoDB/geocoder-api
"""
from setuptools import setup, find_packages
setup(
name='heremaps',
version='0.0.1',
description='A Here Maps API Python wrapper',
url='https://github.com/CartoDB/geocoder-api',
author='Data Services Team - CartoDB',
author_email='dataservices@cartodb.com',
license='MIT',
classifiers=[
'Development Status :: 5 - Production',
'Intended Audience :: Mapping comunity',
'Topic :: Maps :: Mapping Tools',
'License :: OSI Approved :: MIT License',
'Programming Language :: Python :: 2.7',
],
keywords='maps api mapping tools',
packages=find_packages(exclude=['contrib', 'docs', 'tests']),
extras_require={
'dev': ['unittest'],
'test': ['unittest'],
}
)

View File

@@ -1,3 +1,3 @@
id,country,province,city,postalcode,ip,street
1,Spain,Castilla y León,Valladolid,47010,8.8.8.8,Calle Amor de Dios
2,USA,New York,Manhattn,10001,8.8.8.8,NonExistentStreet
2,USA,New York,Manhattn,10001,8.8.8.8,135 W 30th St
1 id country province city postalcode ip street
2 1 Spain Castilla y León Valladolid 47010 8.8.8.8 Calle Amor de Dios
3 2 USA New York Manhattn 10001 8.8.8.8 NonExistentStreet 135 W 30th St

View File

@@ -1,6 +1,7 @@
import os
import requests
import json
import sys
import time
@@ -11,11 +12,12 @@ class ImportHelper:
def import_test_dataset(cls, username, api_key, host):
requests.packages.urllib3.disable_warnings()
url = "https://{0}.{1}/api/v1/imports/"\
"?type_guessing=false&privacy=public&api_key={2}".format(
username, host, api_key)
"?type_guessing=false&privacy=public&api_key={2}".format(
username, host, api_key)
dataset = {
'file': open('fixtures/geocoder_api_test_dataset.csv', 'rb')}
response = requests.post(url, files=dataset)
response.raise_for_status()
response_json = json.loads(response.text)
if not response_json['success']:
print "Error importing the test dataset: {0}".format(response.text)
@@ -40,8 +42,7 @@ class ImportHelper:
import_data_response = requests.get(import_url)
if import_data_response.status_code != 200:
print "Error getting the table name from " \
"the import data: {0}".format(
import_data_response.text)
"the import data: {0}".format(import_data_response.text)
sys.exit(1)
import_data_json = json.loads(import_data_response.text)

View File

@@ -15,7 +15,7 @@ class TestStreetFunctions(TestCase):
)
def test_if_select_with_street_point_is_ok(self):
query = "SELECT cdb_geocode_street_point_v2_(street) " \
query = "SELECT cdb_geocode_street_point(street) " \
"as geometry FROM {0} LIMIT 1&api_key={1}".format(
self.env_variables['table_name'],
self.env_variables['api_key'])
@@ -23,7 +23,7 @@ class TestStreetFunctions(TestCase):
assert_not_equal(geometry, None)
def test_if_select_with_street_without_api_key_raise_error(self):
query = "SELECT cdb_geocode_street_point_v2_(street) " \
query = "SELECT cdb_geocode_street_point(street) " \
"as geometry FROM {0} LIMIT 1".format(
self.env_variables['table_name'])
try: