From de0ced9434416a4b1ac9e734d1b0700a4cff5fe1 Mon Sep 17 00:00:00 2001 From: Carla Iriberri Date: Fri, 23 Oct 2015 18:44:26 +0200 Subject: [PATCH 01/32] First files --- geocoder/ip-addresses/extension/.gitignore | 3 + geocoder/ip-addresses/extension/Makefile | 8 ++ .../extension/cdb_geocoder_ipaddr--0.0.1.sql | 80 +++++++++++++++++++ .../extension/cdb_geocoder_ipaddr.control | 6 ++ 4 files changed, 97 insertions(+) create mode 100644 geocoder/ip-addresses/extension/.gitignore create mode 100644 geocoder/ip-addresses/extension/Makefile create mode 100644 geocoder/ip-addresses/extension/cdb_geocoder_ipaddr--0.0.1.sql create mode 100644 geocoder/ip-addresses/extension/cdb_geocoder_ipaddr.control diff --git a/geocoder/ip-addresses/extension/.gitignore b/geocoder/ip-addresses/extension/.gitignore new file mode 100644 index 0000000..e710f0e --- /dev/null +++ b/geocoder/ip-addresses/extension/.gitignore @@ -0,0 +1,3 @@ +results/ +regression.diffs +regression.out diff --git a/geocoder/ip-addresses/extension/Makefile b/geocoder/ip-addresses/extension/Makefile new file mode 100644 index 0000000..922a44b --- /dev/null +++ b/geocoder/ip-addresses/extension/Makefile @@ -0,0 +1,8 @@ +EXTENSION = cdb_geocoder_ipaddr +DATA = cdb_geocoder_ipaddr--0.0.1.sql +REGRESS = cdb_geocoder_ipaddr_test + +# postgres build stuff +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) diff --git a/geocoder/ip-addresses/extension/cdb_geocoder_ipaddr--0.0.1.sql b/geocoder/ip-addresses/extension/cdb_geocoder_ipaddr--0.0.1.sql new file mode 100644 index 0000000..7ce8c6b --- /dev/null +++ b/geocoder/ip-addresses/extension/cdb_geocoder_ipaddr--0.0.1.sql @@ -0,0 +1,80 @@ +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_geocoder_ipaddr" to load this file. \quit + +-- Response types for IP addresses geocoder +CREATE TYPE geocode_ip_v1 AS (q text, geom geometry, success boolean); + +-- Public API functions -- +--- Geocoding function --- +-- TODO: deal with permissions + +CREATE OR REPLACE FUNCTION geocode_ip(ip text[]) RETURNS SETOF geocode_ip_v1 + LANGUAGE plpgsql SECURITY DEFINER + AS $$ + DECLARE + ret geocode_ip_v1%rowtype; + n TEXT; + new_ips INET[]; + old_ips TEXT[]; + BEGIN + FOR n IN SELECT unnest(ip) LOOP + BEGIN + IF family(n::inet)=6 THEN + new_ips := array_append(new_ips, n::inet); + old_ips := array_append(old_ips, n); + ELSE + new_ips := array_append(new_ips, ('::ffff:'||n)::inet); + old_ips := array_append(old_ips, n); + END IF; + EXCEPTION WHEN OTHERS THEN + SELECT n AS q, NULL as geom, FALSE as success INTO ret; + RETURN NEXT ret; + END; + END LOOP; + FOR ret IN WITH ips AS (SELECT unnest(old_ips) s, unnest(new_ips) net), + matches AS (SELECT s, (SELECT the_geom FROM ip_address_locations WHERE network_start_ip <= ips.net ORDER BY network_start_ip DESC LIMIT 1) geom FROM ips) + SELECT s, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM matches + LOOP + RETURN NEXT ret; + END LOOP; + RETURN; +END +$$; + +-------------------------------------------------------------------------------- + +-- Support tables + +CREATE TABLE ip_address_locations ( + network_start_ip inet, + the_geom geometry(Geometry,4326), + cartodb_id integer NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL, + updated_at timestamp with time zone DEFAULT now() NOT NULL, + the_geom_webmercator geometry(Geometry,3857) +); + +CREATE SEQUENCE ip_address_locations_cartodb_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; +ALTER SEQUENCE ip_address_locations_cartodb_id_seq OWNED BY ip_address_locations.cartodb_id; +ALTER TABLE ONLY ip_address_locations ALTER COLUMN cartodb_id SET DEFAULT nextval('ip_address_locations_cartodb_id_seq'::regclass); + + +ALTER TABLE ONLY ip_address_locations + ADD CONSTRAINT ip_address_locations_cartodb_id_key UNIQUE (cartodb_id); +ALTER TABLE ONLY ip_address_locations + ADD CONSTRAINT ip_address_locations_pkey PRIMARY KEY (cartodb_id); + + +CREATE INDEX ip_address_locations_the_geom_idx ON ip_address_locations USING gist (the_geom); +CREATE INDEX ip_address_locations_the_geom_webmercator_idx ON ip_address_locations USING gist (the_geom_webmercator); +CREATE INDEX ip_address_locations_startip_idx ON ip_address_locations USING btree (network_start_ip); + +CREATE TRIGGER track_updates AFTER INSERT OR DELETE OR UPDATE OR TRUNCATE ON ip_address_locations FOR EACH STATEMENT EXECUTE PROCEDURE cartodb.cdb_tablemetadata_trigger(); +CREATE TRIGGER update_the_geom_webmercator_trigger BEFORE INSERT OR UPDATE OF the_geom ON ip_address_locations FOR EACH ROW EXECUTE PROCEDURE cartodb._cdb_update_the_geom_webmercator(); +CREATE TRIGGER update_updated_at_trigger BEFORE UPDATE ON ip_address_locations FOR EACH ROW EXECUTE PROCEDURE cartodb._cdb_update_updated_at(); + diff --git a/geocoder/ip-addresses/extension/cdb_geocoder_ipaddr.control b/geocoder/ip-addresses/extension/cdb_geocoder_ipaddr.control new file mode 100644 index 0000000..af9e981 --- /dev/null +++ b/geocoder/ip-addresses/extension/cdb_geocoder_ipaddr.control @@ -0,0 +1,6 @@ +# cdb geocoder ipaddr extension +comment = 'CartoDB admin0 internal geocoder' +default_version = '0.0.1' +relocatable = true +requires = cartodb +superuser = false From 21b5949d35d6f6d7dad62da3c1325b87c53921c8 Mon Sep 17 00:00:00 2001 From: Carla Iriberri Date: Mon, 26 Oct 2015 10:13:55 +0100 Subject: [PATCH 02/32] Adding more files --- geocoder/ip-addresses/extension/README.md | 36 +++++++++++++++++++ .../sql/cdb_geocoder_ipaddr_test.sql | 28 +++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 geocoder/ip-addresses/extension/README.md create mode 100644 geocoder/ip-addresses/extension/sql/cdb_geocoder_ipaddr_test.sql diff --git a/geocoder/ip-addresses/extension/README.md b/geocoder/ip-addresses/extension/README.md new file mode 100644 index 0000000..b814d22 --- /dev/null +++ b/geocoder/ip-addresses/extension/README.md @@ -0,0 +1,36 @@ +# CartoDB IP addresses geocoder extension +Postgres extension for the CartoDB admin0 geocoder. It is meant to contain the functions and related objects needed to geocode by IP addresses. It is not meant to contain the actual data used to geocode them. + +## Dependencies +This extension is thought to be used on top of CartoDB platform. Therefore a cartodb user is required to install the extension onto it. + +The following is a non-comprehensive list of dependencies: + +- Postgres 9.3+ +- Postgis extension +- Schema triggers extension +- CartoDB extension + +## Installation into the db cluster +This requires root privileges +``` +sudo make all install +``` + +## Execute tests +``` +PGUSER=postgres make installcheck +``` + +## Install onto a user's database +``` +psql -U development_cartodb_user_fe3b850a-01c0-48f9-8a26-a82f09e9b53f cartodb_dev_user_fe3b850a-01c0-48f9-8a26-a82f09e9b53f_db +``` + +and then: + +```sql +CREATE EXTENSION cdb_geocoder_ipaddr; +``` + +The extension creation in the user's db does not require special privileges. It can be even created from the sql api. diff --git a/geocoder/ip-addresses/extension/sql/cdb_geocoder_ipaddr_test.sql b/geocoder/ip-addresses/extension/sql/cdb_geocoder_ipaddr_test.sql new file mode 100644 index 0000000..6334ee4 --- /dev/null +++ b/geocoder/ip-addresses/extension/sql/cdb_geocoder_ipaddr_test.sql @@ -0,0 +1,28 @@ +CREATE EXTENSION postgis; +CREATE EXTENSION schema_triggers; +CREATE EXTENSION plpythonu; +CREATE EXTENSION cartodb; +CREATE EXTENSION cdb_geocoder_ipaddr; + +-- Check that the geocoding function is callable, should return success = false +SELECT (geocode_ip(Array['100.0.24.0'])).*; + +-- Mock the varnish invalidation function +CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$ +BEGIN + RETURN; +END +$$ +LANGUAGE plpgsql; + +-- Add a few IP sources +COPY ip_address_locations (network_start_ip, the_geom, cartodb_id, created_at, updated_at, the_geom_webmercator) FROM stdin; +::ffff:2.235.35.0 0101000020E610000072F90FE9B7CF22405DFE43FAEDC34640 2821226 2014-08-25 10:35:51.665546+00 2014-08-25 10:35:51.665546+00 0101000020110F000010801778FBF32F4109868FF8BCC35541 +::ffff:31.7.187.0 \N 2783250 2014-08-25 10:35:51.665546+00 2014-08-25 10:35:51.665546+00 \N +::ffff:64.110.146.0 \N 2783251 2014-08-25 10:35:51.665546+00 2014-08-25 10:35:51.665546+00 \N +::ffff:72.5.198.0 \N 2783252 2014-08-25 10:35:51.665546+00 2014-08-25 10:35:51.665546+00 \N +::ffff:77.73.184.0 \N 2783253 2014-08-25 10:35:51.665546+00 2014-08-25 10:35:51.665546+00 \N +\. + +-- Check that the geocoding function is callable, should return success = true +SELECT (geocode_ip(Array['2.235.35.0'])).*; From 1739000bdd2fb799306799c86a45923c5f561a93 Mon Sep 17 00:00:00 2001 From: Carla Iriberri Date: Mon, 26 Oct 2015 12:31:20 +0100 Subject: [PATCH 03/32] Fixes tests --- geocoder/ip-addresses/extension/README.md | 2 +- .../expected/cdb_geocoder_ipaddr_test.out | 27 +++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 geocoder/ip-addresses/extension/expected/cdb_geocoder_ipaddr_test.out diff --git a/geocoder/ip-addresses/extension/README.md b/geocoder/ip-addresses/extension/README.md index b814d22..908d8a3 100644 --- a/geocoder/ip-addresses/extension/README.md +++ b/geocoder/ip-addresses/extension/README.md @@ -24,7 +24,7 @@ PGUSER=postgres make installcheck ## Install onto a user's database ``` -psql -U development_cartodb_user_fe3b850a-01c0-48f9-8a26-a82f09e9b53f cartodb_dev_user_fe3b850a-01c0-48f9-8a26-a82f09e9b53f_db +psql -U cartodb_dev_user_367c0edc-b2ad-4bab-ad43-3d58a6179a93_db cartodb_dev_user_367c0edc-b2ad-4bab-ad43-3d58a6179a93_db ``` and then: diff --git a/geocoder/ip-addresses/extension/expected/cdb_geocoder_ipaddr_test.out b/geocoder/ip-addresses/extension/expected/cdb_geocoder_ipaddr_test.out new file mode 100644 index 0000000..1834323 --- /dev/null +++ b/geocoder/ip-addresses/extension/expected/cdb_geocoder_ipaddr_test.out @@ -0,0 +1,27 @@ +CREATE EXTENSION postgis; +CREATE EXTENSION schema_triggers; +CREATE EXTENSION plpythonu; +CREATE EXTENSION cartodb; +CREATE EXTENSION cdb_geocoder_ipaddr; +-- Check that the geocoding function is callable, should return success = false +SELECT (geocode_ip(Array['100.0.24.0'])).*; + q | geom | success +------------+------+--------- + 100.0.24.0 | | f +(1 row) + +-- Mock the varnish invalidation function +CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$ +BEGIN + RETURN; +END +$$ +LANGUAGE plpgsql; +-- Add a few IP sources +COPY ip_address_locations (network_start_ip, the_geom, cartodb_id, created_at, updated_at, the_geom_webmercator) FROM stdin; +-- Check that the geocoding function is callable, should return success = true +SELECT (geocode_ip(Array['2.235.35.0'])).*; + q | geom | success +------------+----------------------------------------------------+--------- + 2.235.35.0 | 0101000020E610000072F90FE9B7CF22405DFE43FAEDC34640 | t +(1 row) From 3734ab3dbaf3fbea91fd7f6ad30a6ecebae2e679 Mon Sep 17 00:00:00 2001 From: Carla Date: Mon, 26 Oct 2015 12:49:36 +0100 Subject: [PATCH 04/32] Update README.md --- geocoder/ip-addresses/extension/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geocoder/ip-addresses/extension/README.md b/geocoder/ip-addresses/extension/README.md index 908d8a3..9944371 100644 --- a/geocoder/ip-addresses/extension/README.md +++ b/geocoder/ip-addresses/extension/README.md @@ -1,5 +1,5 @@ # CartoDB IP addresses geocoder extension -Postgres extension for the CartoDB admin0 geocoder. It is meant to contain the functions and related objects needed to geocode by IP addresses. It is not meant to contain the actual data used to geocode them. +Postgres extension for the CartoDB IP addresses geocoder. It is meant to contain the functions and related objects needed to geocode by IP addresses. It is not meant to contain the actual data used to geocode them. ## Dependencies This extension is thought to be used on top of CartoDB platform. Therefore a cartodb user is required to install the extension onto it. From 585cee81d2d7ed1fa67052a19e40124d7949baa0 Mon Sep 17 00:00:00 2001 From: Carla Iriberri Date: Mon, 26 Oct 2015 16:01:49 +0100 Subject: [PATCH 05/32] Adding file for namedplaces extension --- geocoder/namedplace/extension/.gitignore | 3 + geocoder/namedplace/extension/Makefile | 8 + geocoder/namedplace/extension/README.md | 36 ++ .../cdb_geocoder_namedplaces--0.0.1.sql | 497 ++++++++++++++++++ .../cdb_geocoder_namedplaces.control | 6 + .../cdb_geocoder_namedplaces_test.out | 64 +++ .../sql/cdb_geocoder_namedplaces_test.sql | 35 ++ 7 files changed, 649 insertions(+) create mode 100644 geocoder/namedplace/extension/.gitignore create mode 100644 geocoder/namedplace/extension/Makefile create mode 100644 geocoder/namedplace/extension/README.md create mode 100644 geocoder/namedplace/extension/cdb_geocoder_namedplaces--0.0.1.sql create mode 100644 geocoder/namedplace/extension/cdb_geocoder_namedplaces.control create mode 100644 geocoder/namedplace/extension/expected/cdb_geocoder_namedplaces_test.out create mode 100644 geocoder/namedplace/extension/sql/cdb_geocoder_namedplaces_test.sql diff --git a/geocoder/namedplace/extension/.gitignore b/geocoder/namedplace/extension/.gitignore new file mode 100644 index 0000000..e710f0e --- /dev/null +++ b/geocoder/namedplace/extension/.gitignore @@ -0,0 +1,3 @@ +results/ +regression.diffs +regression.out diff --git a/geocoder/namedplace/extension/Makefile b/geocoder/namedplace/extension/Makefile new file mode 100644 index 0000000..ab1f0dc --- /dev/null +++ b/geocoder/namedplace/extension/Makefile @@ -0,0 +1,8 @@ +EXTENSION = cdb_geocoder_namedplaces +DATA = cdb_geocoder_namedplaces--0.0.1.sql +REGRESS = cdb_geocoder_namedplaces_test + +# postgres build stuff +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) diff --git a/geocoder/namedplace/extension/README.md b/geocoder/namedplace/extension/README.md new file mode 100644 index 0000000..478c892 --- /dev/null +++ b/geocoder/namedplace/extension/README.md @@ -0,0 +1,36 @@ +# CartoDB named places geocoder extension +Postgres extension for the CartoDB named places geocoder. It is meant to contain the functions and related objects needed to geocode by city names. It is not meant to contain the actual data used to geocode them. + +## Dependencies +This extension is thought to be used on top of CartoDB platform. Therefore a cartodb user is required to install the extension onto it. + +The following is a non-comprehensive list of dependencies: + +- Postgres 9.3+ +- Postgis extension +- Schema triggers extension +- CartoDB extension + +## Installation into the db cluster +This requires root privileges +``` +sudo make all install +``` + +## Execute tests +``` +PGUSER=postgres make installcheck +``` + +## Install onto a user's database +``` +psql -U cartodb_dev_user_367c0edc-b2ad-4bab-ad43-3d58a6179a93_db cartodb_dev_user_367c0edc-b2ad-4bab-ad43-3d58a6179a93_db +``` + +and then: + +```sql +CREATE EXTENSION cdb_geocoder_namedplaces; +``` + +The extension creation in the user's db does not require special privileges. It can be even created from the sql api. diff --git a/geocoder/namedplace/extension/cdb_geocoder_namedplaces--0.0.1.sql b/geocoder/namedplace/extension/cdb_geocoder_namedplaces--0.0.1.sql new file mode 100644 index 0000000..6dc537e --- /dev/null +++ b/geocoder/namedplace/extension/cdb_geocoder_namedplaces--0.0.1.sql @@ -0,0 +1,497 @@ +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_geocoder_namedplaces" to load this file. \quit + +-- Response types for namedplaces geocoder +CREATE TYPE geocode_namedplace_country_v1 AS (q text, c text, geom geometry, success boolean); +CREATE TYPE geocode_namedplace_v1 AS (q text, geom geometry, success boolean); +CREATE TYPE geocode_admin1_country_v1 AS (q text, a1 text, c text, geom geometry, success boolean); +CREATE TYPE geocode_admin_country_v1 AS (q text, c text, geom geometry, success boolean); + +-- Public API functions -- +--- Geocoding function --- +-- TODO: deal with permissions + +CREATE OR REPLACE FUNCTION geocode_namedplace(places text[], country text[]) RETURNS SETOF geocode_namedplace_country_v1 + LANGUAGE plpgsql IMMUTABLE SECURITY DEFINER + AS $$ + DECLARE + ret geocode_namedplace_country_v1%rowtype; + nans TEXT[]; + BEGIN + + SELECT array_agg(p) INTO nans FROM (SELECT unnest(places) p, unnest(country) c) g WHERE c IS NULL; + + IF 0 < array_length(nans, 1) THEN + SELECT array_agg(p), array_agg(c) INTO places, country FROM (SELECT unnest(places) p, unnest(country) c) g WHERE c IS NOT NULL; + FOR ret IN SELECT g.q, NULL as c, g.geom, g.success FROM (SELECT (geocode_namedplace(nans)).*) g LOOP + RETURN NEXT ret; + END LOOP; + END IF; + + SELECT array_agg(p) INTO nans FROM (SELECT unnest(places) p, unnest(country) c) g WHERE c=''; + IF 0 < array_length(nans, 1) THEN + SELECT array_agg(p), array_agg(c) INTO places, country FROM (SELECT unnest(places) p, unnest(country) c) g WHERE c!=''; + FOR ret IN SELECT g.q, '' as c, g.geom, g.success FROM (SELECT (geocode_namedplace(nans)).*) g LOOP + RETURN NEXT ret; + END LOOP; + END IF; + + FOR ret IN WITH + p AS (SELECT r.s, r.c, (SELECT iso2 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT unnest(places) AS s, unnest(country)::text AS c) r), + best AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE c = p.c AND geom IS NOT NULL)) + SELECT q, c, geom, TRUE AS success FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT q, c, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next + LOOP + RETURN NEXT ret; + END LOOP; + RETURN; +END +$$; + + +CREATE OR REPLACE FUNCTION geocode_namedplace(places text[], inputcountry text) RETURNS SETOF geocode_admin_country_v1 + LANGUAGE plpgsql IMMUTABLE SECURITY DEFINER + AS $$ + DECLARE + ret geocode_admin_country_v1%rowtype; + isoTwo TEXT := NULL; + has_country BOOLEAN; + BEGIN + has_country := TRUE; + -- find the iso2 code for the input country string, else NULL + IF inputcountry IS NULL THEN + has_country := FALSE; + ELSIF inputcountry = '' THEN + has_country := FALSE; + END IF; + + IF has_country THEN + SELECT iso2 INTO isoTwo FROM country_decoder WHERE lower(inputcountry) = ANY (synonyms) LIMIT 1; + FOR ret IN WITH + best AS (SELECT p.s AS q, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = isoTwo ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT unnest(places) AS s) p), + next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = isoTwo AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT unnest(places) AS s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT q, inputcountry c, geom, TRUE AS success FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT q, inputcountry c, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next + LOOP + RETURN NEXT ret; + END LOOP; + -- no country included, or iso interpretation found + ELSE + FOR ret IN + SELECT g.q as q, inputcountry as c, g.geom as geom, g.success as success FROM (SELECT (geocode_namedplace(places)).*) g + LOOP + RETURN NEXT ret; + END LOOP; + END IF; + RETURN; +END +$$; + + +CREATE OR REPLACE FUNCTION geocode_namedplace(places text[], admin1s text, inputcountry text) RETURNS SETOF geocode_admin1_country_v1 + LANGUAGE plpgsql IMMUTABLE SECURITY DEFINER + AS $$ + DECLARE + ret geocode_admin1_country_v1%rowtype; + has_country BOOLEAN; + has_admin1s BOOLEAN; + admin1s_a TEXT[]; + BEGIN + + has_country := TRUE; + has_admin1s := TRUE; + + IF inputcountry IS NULL THEN + has_country := FALSE; + ELSIF inputcountry = '' THEN + has_country := FALSE; + END IF; + + IF admin1s IS NULL THEN + has_admin1s := FALSE; + ELSIF admin1s = '' THEN + has_admin1s := FALSE; + END IF; + + -- no country value + IF has_country IS FALSE THEN + -- no country no admin1 value + IF has_admin1s IS FALSE THEN + FOR ret IN SELECT g.q, admin1s AS a1, inputcountry as c, g.geom, g.success FROM (SELECT (geocode_namedplace(places)).*) g LOOP + RETURN NEXT ret; + END LOOP; + -- no country, has admin1 value + ELSE + FOR ret IN + SELECT g.q, admin1s AS a1, inputcountry as c, g.geom, g.success FROM ( + SELECT ( + geocode_namedplace( + places, + (SELECT array_agg(a) FROM (SELECT admin1s a FROM GENERATE_SERIES(1, Array_Length(places, 1)) s) r), + NULL + ) + ).*) g LOOP + RETURN NEXT ret; + END LOOP; + END IF; + -- has country value + ELSE + -- has country, no admin1 value + IF has_admin1s IS FALSE THEN + FOR ret IN SELECT g.q, admin1s AS a1, inputcountry as c, g.geom, g.success FROM (SELECT (geocode_namedplace(places, inputcountry)).*) g LOOP + RETURN NEXT ret; + END LOOP; + -- has country, has admin1 value + ELSE + FOR ret IN + SELECT g.q, admin1s AS a1, inputcountry as c, g.geom, g.success FROM ( + SELECT ( + geocode_namedplace( + places, + (SELECT array_agg(a) FROM (SELECT admin1s a FROM GENERATE_SERIES(1, Array_Length(places, 1)) s) r), + inputcountry + ) + ).*) g LOOP + RETURN NEXT ret; + END LOOP; + END IF; + END IF; + RETURN; +END +$$; + + +CREATE OR REPLACE FUNCTION geocode_namedplace(places text[], admin1s text[], inputcountry text) RETURNS SETOF geocode_admin1_country_v1 + LANGUAGE plpgsql IMMUTABLE SECURITY DEFINER + AS $$ + DECLARE + ret geocode_admin1_country_v1%rowtype; + nans TEXT[]; + isoTwo TEXT := NULL; + has_country BOOLEAN; + BEGIN + has_country := TRUE; + -- find the iso2 code for the input country string, else NULL + IF inputcountry IS NULL THEN + has_country := FALSE; + ELSIF inputcountry = '' THEN + has_country := FALSE; + END IF; + IF has_country THEN + SELECT iso2 INTO isoTwo FROM country_decoder WHERE lower(inputcountry) = ANY (synonyms) LIMIT 1; + END IF; + + -- find all cases where admin1 is NULL + SELECT array_agg(p) INTO nans FROM (SELECT unnest(places) p, unnest(admin1s) c) g WHERE c IS NULL; + + IF 0 < array_length(nans, 1) THEN + SELECT array_agg(p), array_agg(c) INTO places, admin1s FROM (SELECT unnest(places) p, unnest(admin1s) c) g WHERE c IS NOT NULL; + IF has_country THEN + -- geocode our named place without admin1 but with our iso2 + FOR ret IN SELECT g.q, null AS a1, inputcountry as c, g.geom, g.success FROM (SELECT (geocode_namedplace(nans, inputcountry)).*) g LOOP + RETURN NEXT ret; + END LOOP; + ELSE + -- geocode our named place without admin1 and without iso2 + FOR ret IN SELECT g.q, NULL as a1, inputcountry as c, g.geom, g.success FROM (SELECT (geocode_namedplace(nans)).*) g LOOP + RETURN NEXT ret; + END LOOP; + END IF; + END IF; + + -- find all cases where admin1 is and empty string + SELECT array_agg(p) INTO nans FROM (SELECT unnest(places) p, unnest(admin1s) c) g WHERE c=''; + + IF 0 < array_length(nans, 1) THEN + SELECT array_agg(p), array_agg(c) INTO places, admin1s FROM (SELECT unnest(places) p, unnest(admin1s) c) g WHERE c!=''; + IF has_country THEN + -- geocode our named place without admin1 but with our iso2 + FOR ret IN + SELECT g.q, '' AS a1, inputcountry as c, g.geom, g.success FROM (SELECT (geocode_namedplace(nans, inputcountry)).*) g + LOOP + RETURN NEXT ret; + END LOOP; + ELSE + -- geocode our named place without admin1 and without iso2 + FOR ret IN + SELECT g.q, '' AS a1, inputcountry as c, g.geom, g.success FROM (SELECT (geocode_namedplace(nans)).*) g + LOOP + RETURN NEXT ret; + END LOOP; + END IF; + END IF; + + -- geocode all the cases where admin1 is available + IF has_country THEN + FOR ret IN WITH + -- return c=iso2 and search without country + p AS ( + SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder WHERE lower(r.a1) = ANY (synonyms) AND admin1_decoder.iso2 = isoTwo LIMIT 1) i FROM (SELECT unnest(places) AS s, unnest(admin1s)::text AS a1) r), + best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT q, a1, inputcountry as c, geom, TRUE AS success FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT q, a1, inputcountry as c, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next + LOOP + RETURN NEXT ret; + END LOOP; + ELSE + -- return c=NULL and search without country + FOR ret IN WITH + p AS ( + SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder WHERE lower(r.a1) = ANY (synonyms) LIMIT 1) i FROM (SELECT unnest(places) AS s, unnest(admin1s)::text AS a1) r WHERE a1 IS NOT NULL and a1 != ''), + best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT q, a1, inputcountry as c, geom, TRUE AS success FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT q, a1, inputcountry as c, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next + LOOP + RETURN NEXT ret; + END LOOP; + END IF; + RETURN; +END +$$; + + + +CREATE OR REPLACE FUNCTION geocode_namedplace(places text[], admin1s text[], inputcountry text[]) RETURNS SETOF geocode_admin1_country_v1 + LANGUAGE plpgsql IMMUTABLE SECURITY DEFINER + AS $$ + DECLARE + ret geocode_admin1_country_v1%rowtype; + BEGIN + IF admin1s IS NULL THEN + FOR ret IN SELECT g.q as q, NULL as a1, g.c as c, g.geom as geom, g.success as success FROM (SELECT (geocode_namedplace(places, inputcountry)).*) g LOOP + RETURN NEXT ret; + END LOOP; + ELSE + FOR ret IN WITH clean AS (SELECT array_agg(p) p, array_agg(a) a, c FROM (SELECT p, a, c FROM (SELECT unnest(places) p, unnest(admin1s) a, unnest(inputcountry) c) z GROUP BY p, a, c) y GROUP BY c) + SELECT (geocode_namedplace(p, a, c)).* FROM clean LOOP + RETURN NEXT ret; + END LOOP; + END IF; + RETURN; +END +$$; + + +CREATE OR REPLACE FUNCTION geocode_namedplace(places text[]) RETURNS SETOF geocode_namedplace_v1 + LANGUAGE plpgsql IMMUTABLE SECURITY DEFINER + AS $$ + DECLARE + ret geocode_namedplace_v1%rowtype; + BEGIN + FOR ret IN WITH best AS (SELECT s AS q, (SELECT the_geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT unnest(places) as s) p), + next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT unnest(places) as s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT q, geom, TRUE AS success FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next + LOOP + RETURN NEXT ret; + END LOOP; + RETURN; +END +$$; + +-------------------------------------------------------------------------------- + +-- Support tables + +CREATE TABLE admin1_decoder ( + name text, + admin1 text, + iso2 text, + geoname_id integer, + cartodb_id integer NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL, + updated_at timestamp with time zone DEFAULT now() NOT NULL, + the_geom geometry(Geometry,4326), + the_geom_webmercator geometry(Geometry,3857), + synonyms text[], + iso3 text, + users double precision +); + + +CREATE SEQUENCE admin1_decoder_cartodb_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; +ALTER SEQUENCE admin1_decoder_cartodb_id_seq OWNED BY admin1_decoder.cartodb_id; +ALTER TABLE ONLY admin1_decoder ALTER COLUMN cartodb_id SET DEFAULT nextval('admin1_decoder_cartodb_id_seq'::regclass); + + +ALTER TABLE ONLY admin1_decoder + ADD CONSTRAINT admin1_decoder_cartodb_id_key UNIQUE (cartodb_id); +ALTER TABLE ONLY admin1_decoder + ADD CONSTRAINT admin1_decoder_pkey PRIMARY KEY (cartodb_id); + + +CREATE INDEX admin1_decoder_the_geom_idx ON admin1_decoder USING gist (the_geom); +CREATE INDEX admin1_decoder_admin1_idx ON admin1_decoder USING btree (admin1); +CREATE INDEX admin1_decoder_geoname_id_idx ON admin1_decoder USING btree (geoname_id); +CREATE INDEX admin1_decoder_iso2_idx ON admin1_decoder USING btree (iso2); +CREATE INDEX admin1_decoder_iso3_idx ON admin1_decoder USING btree (iso3); +CREATE INDEX admin1_decoder_name_idx ON admin1_decoder USING btree (name); + + +CREATE TRIGGER track_updates AFTER INSERT OR DELETE OR UPDATE OR TRUNCATE ON admin1_decoder FOR EACH STATEMENT EXECUTE PROCEDURE cartodb.cdb_tablemetadata_trigger(); +CREATE TRIGGER update_the_geom_webmercator_trigger BEFORE INSERT OR UPDATE OF the_geom ON admin1_decoder FOR EACH ROW EXECUTE PROCEDURE cartodb._cdb_update_the_geom_webmercator(); +CREATE TRIGGER update_updated_at_trigger BEFORE UPDATE ON admin1_decoder FOR EACH ROW EXECUTE PROCEDURE cartodb._cdb_update_updated_at(); + + +CREATE TABLE country_decoder ( + name text, + nativename text, + tld text, + iso2 text, + ccn3 text, + iso3 text, + currency text, + callingcode text, + capital text, + altspellings text, + relevance text, + region text, + subregion text, + language text, + languagescodes text, + translations text, + population text, + latlng text, + demonym text, + borders text, + the_geom geometry(Geometry,4326), + cartodb_id integer NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL, + updated_at timestamp with time zone DEFAULT now() NOT NULL, + the_geom_webmercator geometry(Geometry,3857), + synbu text[], + synonyms text[], + users double precision +); + + +CREATE SEQUENCE country_decoder_cartodb_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; +ALTER SEQUENCE country_decoder_cartodb_id_seq OWNED BY country_decoder.cartodb_id; +ALTER TABLE ONLY country_decoder ALTER COLUMN cartodb_id SET DEFAULT nextval('country_decoder_cartodb_id_seq'::regclass); + +ALTER TABLE ONLY country_decoder + ADD CONSTRAINT country_decoder_cartodb_id_key UNIQUE (cartodb_id); +ALTER TABLE ONLY country_decoder + ADD CONSTRAINT country_decoder_pkey PRIMARY KEY (cartodb_id); +ALTER TABLE country_decoder CLUSTER ON country_decoder_pkey; + + +CREATE INDEX country_decoder_the_geom_idx ON country_decoder USING gist (the_geom); +CREATE INDEX country_decoder_the_geom_webmercator_idx ON country_decoder USING gist (the_geom_webmercator); + +CREATE TRIGGER track_updates AFTER INSERT OR DELETE OR UPDATE OR TRUNCATE ON country_decoder FOR EACH STATEMENT EXECUTE PROCEDURE cartodb.cdb_tablemetadata_trigger(); +CREATE TRIGGER update_the_geom_webmercator_trigger BEFORE INSERT OR UPDATE OF the_geom ON country_decoder FOR EACH ROW EXECUTE PROCEDURE cartodb._cdb_update_the_geom_webmercator(); +CREATE TRIGGER update_updated_at_trigger BEFORE UPDATE ON country_decoder FOR EACH ROW EXECUTE PROCEDURE cartodb._cdb_update_updated_at(); + + +CREATE TABLE global_cities_points_limited ( + geoname_id integer, + name text, + asciiname text, + altnames text, + featclass text, + featcode text, + iso2 text, + cc2 text, + admin1 text, + admin2 text, + admin3 text, + admin4 text, + population double precision, + gtopo30 integer, + the_geom geometry(Point,4326), + created_at timestamp with time zone DEFAULT now() NOT NULL, + updated_at timestamp with time zone DEFAULT now() NOT NULL, + the_geom_webmercator geometry(Geometry,3857), + cartodb_id integer NOT NULL, + lowername text +); + + +CREATE SEQUENCE global_cities_points_limited_cartodb_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; +ALTER SEQUENCE global_cities_points_limited_cartodb_id_seq OWNED BY global_cities_points_limited.cartodb_id; +ALTER TABLE ONLY global_cities_points_limited ALTER COLUMN cartodb_id SET DEFAULT nextval('global_cities_points_limited_cartodb_id_seq'::regclass); + +ALTER TABLE ONLY global_cities_points_limited + ADD CONSTRAINT global_cities_points_limited_cartodb_id_key UNIQUE (cartodb_id); +ALTER TABLE ONLY global_cities_points_limited + ADD CONSTRAINT global_cities_points_limited_pkey PRIMARY KEY (cartodb_id); + + +CREATE INDEX global_cities_points_limited_the_geom_idx ON global_cities_points_limited USING gist (the_geom); +CREATE INDEX global_cities_points_limited_the_geom_webmercator_idx ON global_cities_points_limited USING gist (the_geom_webmercator); +CREATE INDEX global_cities_points_limited_lower_iso2_idx ON global_cities_points_limited USING btree (lowername, iso2); +CREATE INDEX global_cities_points_limited_admin1_idx ON global_cities_points_limited USING btree (admin1); +CREATE INDEX global_cities_points_limited_geoname_id_idx ON global_cities_points_limited USING btree (geoname_id); + + +CREATE TRIGGER track_updates AFTER INSERT OR DELETE OR UPDATE OR TRUNCATE ON global_cities_points_limited FOR EACH STATEMENT EXECUTE PROCEDURE cartodb.cdb_tablemetadata_trigger(); +CREATE TRIGGER update_the_geom_webmercator_trigger BEFORE INSERT OR UPDATE OF the_geom ON global_cities_points_limited FOR EACH ROW EXECUTE PROCEDURE cartodb._cdb_update_the_geom_webmercator(); +CREATE TRIGGER update_updated_at_trigger BEFORE UPDATE ON global_cities_points_limited FOR EACH ROW EXECUTE PROCEDURE cartodb._cdb_update_updated_at(); + + +CREATE TABLE global_cities_alternates_limited ( + geoname_id integer, + name text, + the_geom geometry(Geometry,4326), + created_at timestamp with time zone DEFAULT now() NOT NULL, + updated_at timestamp with time zone DEFAULT now() NOT NULL, + the_geom_webmercator geometry(Geometry,3857), + preferred boolean, + lowername text, + cartodb_id integer NOT NULL, + admin1_geonameid integer, + iso2 text, + admin1 text +); + + +CREATE SEQUENCE global_cities_alternates_limited_cartodb_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; +ALTER SEQUENCE global_cities_alternates_limited_cartodb_id_seq OWNED BY global_cities_alternates_limited.cartodb_id; +ALTER TABLE ONLY global_cities_alternates_limited ALTER COLUMN cartodb_id SET DEFAULT nextval('global_cities_alternates_limited_cartodb_id_seq'::regclass); + +ALTER TABLE ONLY global_cities_alternates_limited + ADD CONSTRAINT points_cities_alternates_limited_cartodb_id_key UNIQUE (cartodb_id); +ALTER TABLE ONLY global_cities_alternates_limited + ADD CONSTRAINT global_cities_alternates_limited_pkey PRIMARY KEY (cartodb_id); + + +CREATE INDEX global_cities_alternates_limited_the_geom_idx ON global_cities_alternates_limited USING gist (the_geom); +CREATE INDEX global_cities_alternates_limited_the_geom_webmercator_idx ON global_cities_alternates_limited USING gist (the_geom_webmercator); +CREATE INDEX global_cities_alternates_limited_admin1_idx ON global_cities_alternates_limited USING btree (admin1); +CREATE INDEX global_cities_alternates_limited_admin1_geonameid_idx ON global_cities_alternates_limited USING btree (admin1_geonameid); +CREATE INDEX global_cities_alternates_limited_lowername_idx ON global_cities_alternates_limited USING btree (lowername); + + +CREATE TRIGGER track_updates AFTER INSERT OR DELETE OR UPDATE OR TRUNCATE ON global_cities_alternates_limited FOR EACH STATEMENT EXECUTE PROCEDURE cartodb.cdb_tablemetadata_trigger(); +CREATE TRIGGER update_the_geom_webmercator_trigger BEFORE INSERT OR UPDATE OF the_geom ON global_cities_alternates_limited FOR EACH ROW EXECUTE PROCEDURE cartodb._cdb_update_the_geom_webmercator(); +CREATE TRIGGER update_updated_at_trigger BEFORE UPDATE ON global_cities_alternates_limited FOR EACH ROW EXECUTE PROCEDURE cartodb._cdb_update_updated_at(); + diff --git a/geocoder/namedplace/extension/cdb_geocoder_namedplaces.control b/geocoder/namedplace/extension/cdb_geocoder_namedplaces.control new file mode 100644 index 0000000..7da3164 --- /dev/null +++ b/geocoder/namedplace/extension/cdb_geocoder_namedplaces.control @@ -0,0 +1,6 @@ +# cdb geocoder namedplaces extension +comment = 'CartoDB named places internal geocoder' +default_version = '0.0.1' +relocatable = true +requires = cartodb +superuser = false diff --git a/geocoder/namedplace/extension/expected/cdb_geocoder_namedplaces_test.out b/geocoder/namedplace/extension/expected/cdb_geocoder_namedplaces_test.out new file mode 100644 index 0000000..d09e4ce --- /dev/null +++ b/geocoder/namedplace/extension/expected/cdb_geocoder_namedplaces_test.out @@ -0,0 +1,64 @@ +CREATE EXTENSION postgis; +CREATE EXTENSION schema_triggers; +CREATE EXTENSION plpythonu; +CREATE EXTENSION cartodb; +CREATE EXTENSION cdb_geocoder_namedplaces; +-- Check that the different geocoding functions are callable, should return success = false +SELECT (geocode_namedplace(Array['Madrid', 'New York City', 'sunapee'])).*; + q | geom | success +---------------+------+--------- + Madrid | | f + New York City | | f + sunapee | | f +(3 rows) +SELECT (geocode_namedplace(Array['Elche', 'Granada', 'Madrid'], 'Spain')).*; + q | c | geom | success +---------+-------+------+--------- + Elche | Spain | | f + Granada | Spain | | f + Madrid | Spain | | f + (3 rows) +SELECT (geocode_namedplace(Array['sunapee', 'sunapeeee', 'New York City', 'Madrid'], Array['', 'US', 'United States', NULL])).*; + q | c | geom | success +---------------+---------------+------+--------- + Madrid | | | f + sunapee | | | f + sunapeeee | US | | f + New York City | United States | | f +(4 rows) +SELECT (geocode_namedplace(Array['Portland', 'Portland', 'New York City'], Array['Maine', 'Oregon', NULL], 'USA')).*; + q | a1 | c | geom | success +---------------+--------+-----+------+--------- + New York City | | USA | | f + Portland | Maine | USA | | f + Portland | Oregon | USA | | f +(3 rows) +SELECT (geocode_namedplace(Array['Portland'], 'Oregon', 'USA')).*; + q | a1 | c | geom | success +----------+--------+-----+------+--------- + Portland | Oregon | USA | | f +(1 row) +SELECT (geocode_namedplace(Array['Portland', 'Portland', 'New York City'], Array['Maine', 'Oregon', NULL], Array['USA'])).*; + q | a1 | c | geom | success +---------------+--------+-----+------+--------- + New York City | | USA | | f + Portland | Oregon | USA | | f + Portland | Maine | USA | | f +(3 rows) + +-- Mock the varnish invalidation function +CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$ +BEGIN + RETURN; +END +$$ +LANGUAGE plpgsql; +-- Add a named place source +COPY global_cities_alternates_limited (geoname_id, name, the_geom, created_at, updated_at, the_geom_webmercator, preferred, lowername, cartodb_id, admin1_geonameid, iso2, admin1) FROM stdin; +COPY global_cities_points_limited (geoname_id, name, asciiname, altnames, featclass, featcode, iso2, admin1, admin2, population, the_geom, created_at, updated_at, the_geom_webmercator, cartodb_id, lowername) FROM stdin; +-- Check that the geocoding function is callable, should return success = true +SELECT (geocode_namedplace(Array['Barcelona'])).* + q | geom | success +-----------+----------------------------------------------------+--------- + Barcelona | 0101000020E6100000CA15DEE522E653C0A4C2D842902B4540 | t +(1 row) \ No newline at end of file diff --git a/geocoder/namedplace/extension/sql/cdb_geocoder_namedplaces_test.sql b/geocoder/namedplace/extension/sql/cdb_geocoder_namedplaces_test.sql new file mode 100644 index 0000000..6f5c422 --- /dev/null +++ b/geocoder/namedplace/extension/sql/cdb_geocoder_namedplaces_test.sql @@ -0,0 +1,35 @@ +CREATE EXTENSION postgis; +CREATE EXTENSION schema_triggers; +CREATE EXTENSION plpythonu; +CREATE EXTENSION cartodb; +CREATE EXTENSION cdb_geocoder_namedplaces; + +-- Check that the different geocoding functions are callable, should return success = false +SELECT (geocode_namedplace(Array['Madrid', 'New York City', 'sunapee'])).*; +SELECT (geocode_namedplace(Array['Elche', 'Granada', 'Madrid'], 'Spain')).*; +SELECT (geocode_namedplace(Array['sunapee', 'sunapeeee', 'New York City', 'Madrid'], Array['', 'US', 'United States', NULL])).*; +SELECT (geocode_namedplace(Array['Portland', 'Portland', 'New York City'], Array['Maine', 'Oregon', NULL], 'USA')).*; +SELECT (geocode_namedplace(Array['Portland'], 'Oregon', 'USA')).*; +SELECT (geocode_namedplace(Array['Portland', 'Portland', 'New York City'], Array['Maine', 'Oregon', NULL], Array['USA'])).* + + +-- Mock the varnish invalidation function +CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$ +BEGIN + RETURN; +END +$$ +LANGUAGE plpgsql; + +-- Add a named place source +COPY global_cities_alternates_limited (geoname_id, name, the_geom, created_at, updated_at, the_geom_webmercator, preferred, lowername, cartodb_id, admin1_geonameid, iso2, admin1) FROM stdin; +3128760 barcelona \N 2014-02-11 18:23:18.115612+00 2014-02-25 16:41:15.278786+00 \N t barcelona 7530944 409419 \N 56 +\. + +COPY global_cities_points_limited (geoname_id, name, asciiname, altnames, featclass, featcode, iso2, admin1, admin2, population, the_geom, created_at, updated_at, the_geom_webmercator, cartodb_id, lowername) FROM stdin; +2421056 Barcelona Barcelona P PPLA ES B 185 0101000020E6100000CA15DEE522E653C0A4C2D842902B4540 2015-06-13 14:48:34.341372+00 2015-06-15 16:53:41.067784+00 0101000020110F00000643969A73E660C10FF27276F0E15341 8653176 barcelona +\. + +-- Check that the geocoding function is callable, should return success = true +SELECT (geocode_namedplace(Array['Barcelona'])).* + From bfd2ed3f495a1dca7f9b254d211e9995db482538 Mon Sep 17 00:00:00 2001 From: Carla Iriberri Date: Mon, 26 Oct 2015 18:38:06 +0100 Subject: [PATCH 06/32] Adds admin1 fils --- geocoder/admin1/extension/.gitignore | 3 + geocoder/admin1/extension/Makefile | 8 + geocoder/admin1/extension/README.md | 36 +++ .../extension/cdb_geocoder_admin1--0.0.1.sql | 265 ++++++++++++++++++ .../extension/cdb_geocoder_admin1.control | 6 + .../expected/cdb_geocoder_admin1_test.out | 45 +++ .../sql/cdb_geocoder_admin1_test.sql | 29 ++ 7 files changed, 392 insertions(+) create mode 100644 geocoder/admin1/extension/.gitignore create mode 100644 geocoder/admin1/extension/Makefile create mode 100644 geocoder/admin1/extension/README.md create mode 100644 geocoder/admin1/extension/cdb_geocoder_admin1--0.0.1.sql create mode 100644 geocoder/admin1/extension/cdb_geocoder_admin1.control create mode 100644 geocoder/admin1/extension/expected/cdb_geocoder_admin1_test.out create mode 100644 geocoder/admin1/extension/sql/cdb_geocoder_admin1_test.sql diff --git a/geocoder/admin1/extension/.gitignore b/geocoder/admin1/extension/.gitignore new file mode 100644 index 0000000..e710f0e --- /dev/null +++ b/geocoder/admin1/extension/.gitignore @@ -0,0 +1,3 @@ +results/ +regression.diffs +regression.out diff --git a/geocoder/admin1/extension/Makefile b/geocoder/admin1/extension/Makefile new file mode 100644 index 0000000..b472643 --- /dev/null +++ b/geocoder/admin1/extension/Makefile @@ -0,0 +1,8 @@ +EXTENSION = cdb_geocoder_admin1 +DATA = cdb_geocoder_admin1--0.0.1.sql +REGRESS = cdb_geocoder_admin1_test + +# postgres build stuff +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) diff --git a/geocoder/admin1/extension/README.md b/geocoder/admin1/extension/README.md new file mode 100644 index 0000000..ac99e36 --- /dev/null +++ b/geocoder/admin1/extension/README.md @@ -0,0 +1,36 @@ +# CartoDB admin1 geocoder extension +Postgres extension for the CartoDB admin1 geocoder. It is meant to contain the functions and related objects needed to geocode by admin1 regions. It is not meant to contain the actual data used to geocode them. + +## Dependencies +This extension is thought to be used on top of CartoDB platform. Therefore a cartodb user is required to install the extension onto it. + +The following is a non-comprehensive list of dependencies: + +- Postgres 9.3+ +- Postgis extension +- Schema triggers extension +- CartoDB extension + +## Installation into the db cluster +This requires root privileges +``` +sudo make all install +``` + +## Execute tests +``` +PGUSER=postgres make installcheck +``` + +## Install onto a user's database +``` +psql -U development_cartodb_user_fe3b850a-01c0-48f9-8a26-a82f09e9b53f cartodb_dev_user_fe3b850a-01c0-48f9-8a26-a82f09e9b53f_db +``` + +and then: + +```sql +CREATE EXTENSION cdb_geocoder_admin1; +``` + +The extension creation in the user's db does not require special privileges. It can be even created from the sql api. diff --git a/geocoder/admin1/extension/cdb_geocoder_admin1--0.0.1.sql b/geocoder/admin1/extension/cdb_geocoder_admin1--0.0.1.sql new file mode 100644 index 0000000..617beee --- /dev/null +++ b/geocoder/admin1/extension/cdb_geocoder_admin1--0.0.1.sql @@ -0,0 +1,265 @@ +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_geocoder_admin1" to load this file. \quit + +-- Response types for admin1 geocoder +-- TODO: check if the types exist already in the db + +CREATE TYPE geocode_admin_v1 AS (q TEXT, geom GEOMETRY, success BOOLEAN); +CREATE TYPE geocode_admin_country_v1 AS (q TEXT, c TEXT, geom GEOMETRY, success BOOLEAN); + + +-- Public API functions -- +--- Geocoding function --- +-- TODO: deal with permissions +CREATE OR REPLACE FUNCTION geocode_admin1_polygons(name text[]) RETURNS SETOF geocode_admin_v1 + LANGUAGE plpgsql SECURITY DEFINER + AS $$ + DECLARE + ret geocode_admin_v1%rowtype; + BEGIN + FOR ret IN + SELECT + q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_province_polygons + WHERE d.c = ANY (synonyms) + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM (SELECT trim(replace(lower(unnest(name)),'.',' ')) c, unnest(name) q) d + ) v + LOOP + RETURN NEXT ret; + END LOOP; + RETURN; +END +$$; + + +CREATE OR REPLACE FUNCTION geocode_admin1_polygons(name text[], inputcountry text) RETURNS SETOF geocode_admin_v1 + LANGUAGE plpgsql SECURITY DEFINER + AS $$ + DECLARE + ret geocode_admin_v1%rowtype; + BEGIN + + FOR ret IN WITH + p AS (SELECT r.c, r.q, (SELECT iso3 FROM country_decoder WHERE lower(inputcountry) = ANY (synonyms)) i FROM (SELECT trim(replace(lower(unnest(name)),'.',' ')) c, unnest(name) q) r) + SELECT + q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_province_polygons + WHERE p.c = ANY (synonyms) + AND iso3 = p.i + -- To calculate frequency, I simply counted the number of users + -- we had signed up in each country. Countries with more users, + -- we favor higher in the geocoder :) + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM p) n + LOOP + RETURN NEXT ret; + END LOOP; + RETURN; +END +$$; + + +CREATE OR REPLACE FUNCTION geocode_admin1_polygons(names text[], country text[]) RETURNS SETOF geocode_admin_country_v1 + LANGUAGE plpgsql SECURITY DEFINER + AS $$ + DECLARE + ret geocode_admin_country_v1%rowtype; + nans TEXT[]; + BEGIN + + + SELECT array_agg(p) INTO nans FROM (SELECT unnest(names) p, unnest(country) c) g WHERE c IS NULL; + + IF 0 < array_length(nans, 1) THEN + SELECT array_agg(p), array_agg(c) INTO names, country FROM (SELECT unnest(names) p, unnest(country) c) g WHERE c IS NOT NULL; + FOR ret IN SELECT g.q, NULL as c, g.geom, g.success FROM (SELECT (geocode_admin1_polygons(nans)).*) g LOOP + RETURN NEXT ret; + END LOOP; + END IF; + + + FOR ret IN WITH + p AS (SELECT r.p, r.q, c, (SELECT iso3 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT trim(replace(lower(unnest(names)),'.',' ')) p, unnest(names) q, unnest(country) c) r) + SELECT + q, c, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success + FROM ( + SELECT + q, c, ( + SELECT the_geom + FROM global_province_polygons + WHERE p.p = ANY (synonyms) + AND iso3 = p.i + -- To calculate frequency, I simply counted the number of users + -- we had signed up in each country. Countries with more users, + -- we favor higher in the geocoder :) + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM p) n + LOOP + RETURN NEXT ret; + END LOOP; + RETURN; +END +$$; + +-------------------------------------------------------------------------------- + +-- Support tables + +CREATE TABLE country_decoder ( + name text, + nativename text, + tld text, + iso2 text, + ccn3 text, + iso3 text, + currency text, + callingcode text, + capital text, + altspellings text, + relevance text, + region text, + subregion text, + language text, + languagescodes text, + translations text, + population text, + latlng text, + demonym text, + borders text, + the_geom geometry(Geometry,4326), + cartodb_id integer NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL, + updated_at timestamp with time zone DEFAULT now() NOT NULL, + the_geom_webmercator geometry(Geometry,3857), + synbu text[], + synonyms text[], + users double precision +); + +CREATE SEQUENCE country_decoder_cartodb_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; +ALTER SEQUENCE country_decoder_cartodb_id_seq OWNED BY country_decoder.cartodb_id; +ALTER TABLE ONLY country_decoder ALTER COLUMN cartodb_id SET DEFAULT nextval('country_decoder_cartodb_id_seq'::regclass); + + +ALTER TABLE ONLY country_decoder + ADD CONSTRAINT country_decoder_cartodb_id_key UNIQUE (cartodb_id); +ALTER TABLE ONLY country_decoder + ADD CONSTRAINT country_decoder_pkey PRIMARY KEY (cartodb_id); + +ALTER TABLE country_decoder CLUSTER ON country_decoder_pkey; + + +CREATE INDEX country_decoder_the_geom_idx ON country_decoder USING gist (the_geom); +CREATE INDEX country_decoder_the_geom_webmercator_idx ON country_decoder USING gist (the_geom_webmercator); + + +CREATE TRIGGER track_updates AFTER INSERT OR DELETE OR UPDATE OR TRUNCATE ON country_decoder FOR EACH STATEMENT EXECUTE PROCEDURE cartodb.cdb_tablemetadata_trigger(); +CREATE TRIGGER update_the_geom_webmercator_trigger BEFORE INSERT OR UPDATE OF the_geom ON country_decoder FOR EACH ROW EXECUTE PROCEDURE cartodb._cdb_update_the_geom_webmercator(); +CREATE TRIGGER update_updated_at_trigger BEFORE UPDATE ON country_decoder FOR EACH ROW EXECUTE PROCEDURE cartodb._cdb_update_updated_at(); + + +CREATE TABLE global_province_polygons ( + the_geom geometry(Geometry,4326), + adm1_code text, + objectid_1 integer, + diss_me integer, + adm1_cod_1 text, + iso_3166_2 text, + wikipedia text, + iso_a2 text, + adm0_sr integer, + name text, + name_alt text, + name_local text, + type text, + type_en text, + code_local text, + code_hasc text, + note text, + hasc_maybe text, + region text, + region_cod text, + provnum_ne integer, + gadm_level integer, + check_me integer, + scalerank integer, + datarank integer, + abbrev text, + postal text, + area_sqkm double precision, + sameascity integer, + labelrank integer, + featurecla text, + name_len integer, + mapcolor9 integer, + mapcolor13 integer, + fips text, + fips_alt text, + woe_id integer, + woe_label text, + woe_name text, + latitude double precision, + longitude double precision, + sov_a3 text, + iso3 text, + adm0_label integer, + admin text, + geonunit text, + gu_a3 text, + gn_id integer, + gn_name text, + gns_id integer, + gns_name text, + gn_level integer, + gn_region text, + gn_a1_code text, + region_sub text, + sub_code text, + gns_level integer, + gns_lang text, + gns_adm1 text, + gns_region text, + cartodb_id integer NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL, + updated_at timestamp with time zone DEFAULT now() NOT NULL, + the_geom_webmercator geometry(Geometry,3857), + synonyms text[], + frequency double precision +); + +CREATE SEQUENCE global_province_polygons_cartodb_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; +ALTER SEQUENCE global_province_polygons_cartodb_id_seq OWNED BY global_province_polygons.cartodb_id; +ALTER TABLE ONLY global_province_polygons ALTER COLUMN cartodb_id SET DEFAULT nextval('global_province_polygons_cartodb_id_seq'::regclass); +ALTER TABLE ONLY global_province_polygons + ADD CONSTRAINT global_province_polygons_cartodb_id_key UNIQUE (cartodb_id); +ALTER TABLE ONLY global_province_polygons + ADD CONSTRAINT global_province_polygons_pkey PRIMARY KEY (cartodb_id); + +CREATE INDEX global_province_polygons_the_geom_idx ON global_province_polygons USING gist (the_geom); +CREATE INDEX global_province_polygons_the_geom_webmercator_idx ON global_province_polygons USING gist (the_geom_webmercator); + +CREATE TRIGGER track_updates AFTER INSERT OR DELETE OR UPDATE OR TRUNCATE ON global_province_polygons FOR EACH STATEMENT EXECUTE PROCEDURE cartodb.cdb_tablemetadata_trigger(); +CREATE TRIGGER update_the_geom_webmercator_trigger BEFORE INSERT OR UPDATE OF the_geom ON global_province_polygons FOR EACH ROW EXECUTE PROCEDURE cartodb._cdb_update_the_geom_webmercator(); +CREATE TRIGGER update_updated_at_trigger BEFORE UPDATE ON global_province_polygons FOR EACH ROW EXECUTE PROCEDURE cartodb._cdb_update_updated_at(); diff --git a/geocoder/admin1/extension/cdb_geocoder_admin1.control b/geocoder/admin1/extension/cdb_geocoder_admin1.control new file mode 100644 index 0000000..7621dcb --- /dev/null +++ b/geocoder/admin1/extension/cdb_geocoder_admin1.control @@ -0,0 +1,6 @@ +# cdb geocoder admin1 extension +comment = 'CartoDB admin1 internal geocoder' +default_version = '0.0.1' +relocatable = true +requires = cartodb +superuser = false diff --git a/geocoder/admin1/extension/expected/cdb_geocoder_admin1_test.out b/geocoder/admin1/extension/expected/cdb_geocoder_admin1_test.out new file mode 100644 index 0000000..963689f --- /dev/null +++ b/geocoder/admin1/extension/expected/cdb_geocoder_admin1_test.out @@ -0,0 +1,45 @@ +CREATE EXTENSION postgis; +CREATE EXTENSION schema_triggers; +CREATE EXTENSION plpythonu; +CREATE EXTENSION cartodb; +CREATE EXTENSION cdb_geocoder_admin1; +-- Check that the geocoding functions are callable, should return NULL +SELECT (geocode_admin1_polygons(Array['TX','Cuidad Real', 'sevilla'])).*; + q | geom | success +-------------+------+--------- + TX | | f + Cuidad Real | | f + sevilla | | f +(3 rows) +SELECT (geocode_admin1_polygons(Array['NH', 'Vermont'], 'United States')).*; + q | geom | success +---------+------+--------- + NH | | f + Vermont | | f +(2 rows) +SELECT (geocode_admin1_polygons(Array['az', 'az'], Array['Ecuador', 'USA'])).*; + q | c | geom | success +----+---------+------+--------- + az | Ecuador | | f + az | USA | | f +(2 rows) + +-- Mock the varnish invalidation function +CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$ +BEGIN + RETURN; +END +$$ +LANGUAGE plpgsql; + +-- Add a few data to the sources +COPY global_province_polygons (the_geom,adm1_code, objectid_1, diss_me, adm1_cod_1, iso_3166_2, wikipedia, iso_a2, adm0_sr, name, name_alt, name_local, type, type_en, code_local, code_hasc, note, hasc_maybe, region, region_cod, provnum_ne, gadm_level, check_me, scalerank, datarank, abbrev, postal, area_sqkm, sameascity, labelrank, featurecla, name_len, mapcolor9, mapcolor13, fips, fips_alt, woe_id, woe_label, woe_name, latitude, longitude, sov_a3, iso3, adm0_label, admin, geonunit, gu_a3, gn_id, gn_name, gns_id, gns_name, gn_level, gn_region, gn_a1_code, region_sub, sub_code, gns_level, gns_lang, gns_adm1, gns_region, cartodb_id, created_at, updated_at, synonyms, frequency) FROM stdin; +0106000020E6100000010000000103000000010000000400000000000000005009C07FB86AC523AF4340FFFFFFFFFFEF0EC0C3CF8DC4FADB42400000000000E0F0BFC3CF8DC4FADB424000000000005009C07FB86AC523AF4340 SVN-1035 1473 1035 SVN-1035 SI- \N SI 1 Vipava \N \N Opcine Commune|Municipality \N SI.SP.VI \N \N Goriška \N 162 2 0 10 8 \N VI 0 -99 10 Admin-1 scale rank 6 2 12 SIE1 \N -55848385 \N \N 45.8271000000000015 13.9723000000000006 SVN SVN 2 Slovenia Slovenia SVN 3239075 Obcina Vipava 243467 Vipava, Obcina 1 \N SI.E1 \N \N 1 div SI05 \N 3700 2014-02-18 19:53:50.080158+00 2014-04-01 15:18:54.094644+00 {vipava,"obcina vipava","vipava,obcina",si.e1,si.sp.vi,svn-1035,svn-1035} 5 +\. + +-- Check that the synonym function is callable, should return true +SELECT (geocode_admin1_polygons(Array['obcina vipava'])).success; + success +--------- + t +(1 row) \ No newline at end of file diff --git a/geocoder/admin1/extension/sql/cdb_geocoder_admin1_test.sql b/geocoder/admin1/extension/sql/cdb_geocoder_admin1_test.sql new file mode 100644 index 0000000..276317d --- /dev/null +++ b/geocoder/admin1/extension/sql/cdb_geocoder_admin1_test.sql @@ -0,0 +1,29 @@ +CREATE EXTENSION postgis; +CREATE EXTENSION schema_triggers; +CREATE EXTENSION plpythonu; +CREATE EXTENSION cartodb; +CREATE EXTENSION cdb_geocoder_admin1; + +-- Check that the geocoding functions are callable, should return NULL +SELECT (geocode_admin1_polygons(Array['TX','Cuidad Real', 'sevilla'])).*; +SELECT (geocode_admin1_polygons(Array['NH', 'Vermont'], 'United States')).*; +SELECT (geocode_admin1_polygons(Array['az', 'az'], Array['Ecuador', 'USA'])).*; + +-- Mock the varnish invalidation function +CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$ +BEGIN + RETURN; +END +$$ +LANGUAGE plpgsql; + +-- Add a few data to the sources +COPY global_province_polygons (the_geom, adm1_code, objectid_1, diss_me, adm1_cod_1, iso_3166_2, wikipedia, iso_a2, adm0_sr, name, name_alt, name_local, type, type_en, code_local, code_hasc, note, hasc_maybe, region, region_cod, provnum_ne, gadm_level, check_me, scalerank, datarank, abbrev, postal, area_sqkm, sameascity, labelrank, featurecla, name_len, mapcolor9, mapcolor13, fips, fips_alt, woe_id, woe_label, woe_name, latitude, longitude, sov_a3, iso3, adm0_label, admin, geonunit, gu_a3, gn_id, gn_name, gns_id, gns_name, gn_level, gn_region, gn_a1_code, region_sub, sub_code, gns_level, gns_lang, gns_adm1, gns_region, cartodb_id, created_at, updated_at, the_geom_webmercator, synonyms, frequency) FROM stdin; +0106000020E6100000010000000103000000010000001C000000701C468693172C405BCFC9FB20EB464010E0C037081E2C40F7FF710981E74640606B2301AC1A2C40B407F55702E64640200BD34801192C40A44EE24698E34640503B8E674E162C4084868DF1EEE246400025D4480D132C40B43156A168E14640701EE41F19082C400CFADB0CA1E24640E0F6B6C345062C401815CAFBB1E24640A00B4B5DB3022C40EF46C124BCE2464010E49A3E6E012C40CB84EFBA8CE24640105943E9D0FA2B40ECD210DDEBE14640008981CA8FF72B40071D4DCAEFE24640B05E6F7FDDDC2B40F3246E6C88E5464060FB4FFA19CE2B40A46CBD87B5E746404058ECF694C52B408CF11C3411EA4640FFF7D7E59FC82B40BCC34EE79AEB46400077C737A4D02B403C3FEF3AB8EC4640E04B53FA05D82B4090EF60F885ED464090EBDCBC6BDD2B40E048DFA985EE464000A5771CC0E92B40BC5B8EF1D4EF4640005BBBFD66F22B40208A181748F04640805CE1823AF92B402CA5060659F0464040FC6A45A0FE2B404B4FC4C11DF046408061640B9F022C4060B922EEBEEF4640FFD76EE28A072C405421278BBDEE4640E09BADEC49092C40FB7EEC1D7AED4640AF09D3481D0B2C40D86DDB0C8DEC4640701C468693172C405BCFC9FB20EB4640 SVN-1035 1473 1035 SVN-1035 SI- \N SI 1 Vipava \N \N Opcine Commune|Municipality \N SI.SP.VI \N + \N Goriška \N 162 2 0 10 8 \N VI 0 -99 10 Admin-1 scale rank 6 2 12 SIE1 \N -55848385 + \N \N 45.8271000000000015 13.9723000000000006 SVN SVN 2 Slovenia Slovenia SVN 3239075 Obcina Vipava 243467 Vipava, Obcina 1 + \N SI.E1 \N \N 1 div SI05 \N 3700 2014-02-18 19:53:50.080158+00 2014-04-01 15:18:54.094644+00 0106000020110F0000010000000103000000010000001C00000023340ED8CEDB3741C0404B7264F35541C85B157A4AE13741B323798AF9EE5541CEDCBDE56FDE37414004235027ED5541489EE87B05DD3741D970658A36EA55418D272EB8BADA374116C6965868E95541A8917D26F7D73741F9BBEB238DE75541A6C99293A9CE37417F31C28009E95541DA9DBCA51CCD3741A5FAE31E1EE9554121BD100C14CA37414E69CD7D2AE95541124037ECFFC8374162390EC3F0E85541332B2FC661C337417AB6C1E72CE855418A0578349EC037417308826069E95541BCFC69D6F1A93741B086FDCC92EC55419F7FD8E3679D3741462E0A8039EF554160E7068E2B963741A150E52919F25541861B631AC198374148B84A13F9F3554123289F23909F3741CCF276F254F55541F49EE41CD5A53741895CFAD14FF655415E8C59A46AAA3741EE95F4A487F755414B69D859E3B43741E59BB99720F95541A6ECFD723CBC3741762F910EADF9554144635A9E08C23741DD3BD2B6C1F95541C650CF259EC637415377246A79F9554105D616CA02CA374149377DBD05F9554170AA9CC530CE3741A52C14CCCBF755419A5F7271ACCF37414FD3A85D41F65541A2FB415F39D13741C55D614C20F5554123340ED8CEDB3741C0404B7264F35541 {vipava,"obcina vipava","vipava, obcina",si.e1,si.sp.vi,svn-1035,svn-1035} 5 +\. + +-- Check that the synonym function is callable, should return true +SELECT (geocode_admin1_polygons(Array['obcina vipava'])).success; From 052d22d80a49fbf45b640891f20a9f6f7805968a Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 3 Nov 2015 11:29:48 +0100 Subject: [PATCH 07/32] Moving admin0 to common extension dir (WIP) --- geocoder/{admin0 => }/extension/.gitignore | 0 geocoder/{admin0 => }/extension/Makefile | 0 geocoder/{admin0 => }/extension/README.md | 0 .../cdb_geocoder.control} | 0 .../{admin0 => }/extension/expected/cdb_geocoder_admin0_test.out | 0 .../sql/0.0.1/cdb_geocoder_admin0.sql} | 0 geocoder/{admin0 => }/extension/sql/cdb_geocoder_admin0_test.sql | 0 7 files changed, 0 insertions(+), 0 deletions(-) rename geocoder/{admin0 => }/extension/.gitignore (100%) rename geocoder/{admin0 => }/extension/Makefile (100%) rename geocoder/{admin0 => }/extension/README.md (100%) rename geocoder/{admin0/extension/cdb_geocoder_admin0.control => extension/cdb_geocoder.control} (100%) rename geocoder/{admin0 => }/extension/expected/cdb_geocoder_admin0_test.out (100%) rename geocoder/{admin0/extension/cdb_geocoder_admin0--0.0.1.sql => extension/sql/0.0.1/cdb_geocoder_admin0.sql} (100%) rename geocoder/{admin0 => }/extension/sql/cdb_geocoder_admin0_test.sql (100%) diff --git a/geocoder/admin0/extension/.gitignore b/geocoder/extension/.gitignore similarity index 100% rename from geocoder/admin0/extension/.gitignore rename to geocoder/extension/.gitignore diff --git a/geocoder/admin0/extension/Makefile b/geocoder/extension/Makefile similarity index 100% rename from geocoder/admin0/extension/Makefile rename to geocoder/extension/Makefile diff --git a/geocoder/admin0/extension/README.md b/geocoder/extension/README.md similarity index 100% rename from geocoder/admin0/extension/README.md rename to geocoder/extension/README.md diff --git a/geocoder/admin0/extension/cdb_geocoder_admin0.control b/geocoder/extension/cdb_geocoder.control similarity index 100% rename from geocoder/admin0/extension/cdb_geocoder_admin0.control rename to geocoder/extension/cdb_geocoder.control diff --git a/geocoder/admin0/extension/expected/cdb_geocoder_admin0_test.out b/geocoder/extension/expected/cdb_geocoder_admin0_test.out similarity index 100% rename from geocoder/admin0/extension/expected/cdb_geocoder_admin0_test.out rename to geocoder/extension/expected/cdb_geocoder_admin0_test.out diff --git a/geocoder/admin0/extension/cdb_geocoder_admin0--0.0.1.sql b/geocoder/extension/sql/0.0.1/cdb_geocoder_admin0.sql similarity index 100% rename from geocoder/admin0/extension/cdb_geocoder_admin0--0.0.1.sql rename to geocoder/extension/sql/0.0.1/cdb_geocoder_admin0.sql diff --git a/geocoder/admin0/extension/sql/cdb_geocoder_admin0_test.sql b/geocoder/extension/sql/cdb_geocoder_admin0_test.sql similarity index 100% rename from geocoder/admin0/extension/sql/cdb_geocoder_admin0_test.sql rename to geocoder/extension/sql/cdb_geocoder_admin0_test.sql From 5257b944b75c90232ad8dc2eadc2a69ed9a68083 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 3 Nov 2015 11:15:19 +0000 Subject: [PATCH 08/32] First version of viable Makefile --- geocoder/extension/.gitignore | 1 + geocoder/extension/Makefile | 15 +++++++++++++-- geocoder/extension/cdb_geocoder.control | 4 ++-- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/geocoder/extension/.gitignore b/geocoder/extension/.gitignore index e710f0e..9819e6c 100644 --- a/geocoder/extension/.gitignore +++ b/geocoder/extension/.gitignore @@ -1,3 +1,4 @@ results/ regression.diffs regression.out +cdb_geocoder--0.0.1.sql diff --git a/geocoder/extension/Makefile b/geocoder/extension/Makefile index cf92715..d1bef4b 100644 --- a/geocoder/extension/Makefile +++ b/geocoder/extension/Makefile @@ -1,8 +1,19 @@ -EXTENSION = cdb_geocoder_admin0 -DATA = cdb_geocoder_admin0--0.0.1.sql +EXTENSION = cdb_geocoder +EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/") + +DATA = cdb_geocoder--0.0.1.sql REGRESS = cdb_geocoder_admin0_test # postgres build stuff PG_CONFIG = pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) + + +SOURCES_0_0_1 = $(wildcard sql/0.0.1/*.sql) + +cdb_geocoder--0.0.1.sql: $(SOURCES_0_0_1) + rm -f $@ + cat $(SOURCES_0_0_1) >> $@ + +all: $(DATA) diff --git a/geocoder/extension/cdb_geocoder.control b/geocoder/extension/cdb_geocoder.control index eedb9b1..2532160 100644 --- a/geocoder/extension/cdb_geocoder.control +++ b/geocoder/extension/cdb_geocoder.control @@ -1,5 +1,5 @@ -# cdb geocoder admin0 extension -comment = 'CartoDB admin0 internal geocoder' +# cdb geocoder extension +comment = 'CartoDB internal geocoder' default_version = '0.0.1' relocatable = true requires = cartodb From f1cccc243196fd13a1aea257e22f870fac8807bf Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 3 Nov 2015 11:19:58 +0000 Subject: [PATCH 09/32] Improvements on Makefile --- geocoder/extension/Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/geocoder/extension/Makefile b/geocoder/extension/Makefile index d1bef4b..a0ce595 100644 --- a/geocoder/extension/Makefile +++ b/geocoder/extension/Makefile @@ -1,7 +1,7 @@ EXTENSION = cdb_geocoder EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/") -DATA = cdb_geocoder--0.0.1.sql +DATA = $(EXTENSION)--$(EXTVERSION).sql REGRESS = cdb_geocoder_admin0_test # postgres build stuff @@ -10,10 +10,10 @@ PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) -SOURCES_0_0_1 = $(wildcard sql/0.0.1/*.sql) +SOURCES_DATA = $(wildcard sql/$(EXTVERSION)/*.sql) -cdb_geocoder--0.0.1.sql: $(SOURCES_0_0_1) +$(DATA): $(SOURCES_DATA) rm -f $@ - cat $(SOURCES_0_0_1) >> $@ + cat $(SOURCES_DATA) >> $@ all: $(DATA) From 699b5af31b0972c75a74b0188fa46c5ed8095c63 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 3 Nov 2015 11:31:47 +0000 Subject: [PATCH 10/32] Split into header and admin0 --- geocoder/extension/Makefile | 6 ++++++ geocoder/extension/sql/0.0.1/00_header.sql | 2 ++ .../sql/0.0.1/{cdb_geocoder_admin0.sql => 10_admin0.sql} | 3 --- 3 files changed, 8 insertions(+), 3 deletions(-) create mode 100644 geocoder/extension/sql/0.0.1/00_header.sql rename geocoder/extension/sql/0.0.1/{cdb_geocoder_admin0.sql => 10_admin0.sql} (98%) diff --git a/geocoder/extension/Makefile b/geocoder/extension/Makefile index a0ce595..d79a16c 100644 --- a/geocoder/extension/Makefile +++ b/geocoder/extension/Makefile @@ -1,3 +1,5 @@ +# Makefile to generate the extension out of separate sql source files. +# Once a version is released, it is not meant to be changed. E.g: once version 0.0.1 is out, it SHALL NOT be changed. EXTENSION = cdb_geocoder EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/") @@ -17,3 +19,7 @@ $(DATA): $(SOURCES_DATA) cat $(SOURCES_DATA) >> $@ all: $(DATA) + +# Only meant for development time, do not use once a version is released +devclean: + rm -f $(DATA) diff --git a/geocoder/extension/sql/0.0.1/00_header.sql b/geocoder/extension/sql/0.0.1/00_header.sql new file mode 100644 index 0000000..94ac67c --- /dev/null +++ b/geocoder/extension/sql/0.0.1/00_header.sql @@ -0,0 +1,2 @@ +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_geocoder" to load this file. \quit diff --git a/geocoder/extension/sql/0.0.1/cdb_geocoder_admin0.sql b/geocoder/extension/sql/0.0.1/10_admin0.sql similarity index 98% rename from geocoder/extension/sql/0.0.1/cdb_geocoder_admin0.sql rename to geocoder/extension/sql/0.0.1/10_admin0.sql index ba53e19..100ff49 100644 --- a/geocoder/extension/sql/0.0.1/cdb_geocoder_admin0.sql +++ b/geocoder/extension/sql/0.0.1/10_admin0.sql @@ -1,6 +1,3 @@ --- Complain if script is sourced in psql, rather than via CREATE EXTENSION -\echo Use "CREATE EXTENSION cdb_geocoder_admin0" to load this file. \quit - -- Response types for admin0 geocoder CREATE TYPE geocode_admin_v1 AS (q TEXT, geom GEOMETRY, success BOOLEAN); CREATE TYPE synonym_lookup_v1 AS (q TEXT, adm0_a3 TEXT); From 6e5836a1231abcad5dc2dac3addfad3385b3a3c2 Mon Sep 17 00:00:00 2001 From: Carla Date: Tue, 3 Nov 2015 12:35:16 +0100 Subject: [PATCH 11/32] Removing admin0 mentions in README --- geocoder/extension/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/geocoder/extension/README.md b/geocoder/extension/README.md index 96dd39a..dfa5430 100644 --- a/geocoder/extension/README.md +++ b/geocoder/extension/README.md @@ -1,5 +1,5 @@ -# CartoDB admin0 geocoder extension -Postgres extension for the CartoDB admin0 geocoder. It is meant to contain the functions and related objects needed to geocode by admin0 regions. It is not meant to contain the actual data used to geocode them. +# CartoDB geocoder extension +Postgres extension for the CartoDB geocoder. It is meant to contain the functions and related objects needed to provide a geocoding service for administrative areas of level 0, administrative areas of level 1, postal codes, IP addresses and city names. It is not meant to contain the actual data used to geocode them. ## Dependencies This extension is thought to be used on top of CartoDB platform. Therefore a cartodb user is required to install the extension onto it. @@ -30,7 +30,7 @@ psql -U development_cartodb_user_fe3b850a-01c0-48f9-8a26-a82f09e9b53f cartodb_de and then: ```sql -CREATE EXTENSION cdb_geocoder_admin0; +CREATE EXTENSION cdb_geocoder; ``` The extension creation in the user's db does not require special privileges. It can be even created from the sql api. From 28a08b71a994d18c162b81e145ed996638e5cf00 Mon Sep 17 00:00:00 2001 From: Carla Iriberri Date: Tue, 3 Nov 2015 12:59:47 +0100 Subject: [PATCH 12/32] Fixes tests --- geocoder/admin1/extension/expected/cdb_geocoder_admin1_test.out | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geocoder/admin1/extension/expected/cdb_geocoder_admin1_test.out b/geocoder/admin1/extension/expected/cdb_geocoder_admin1_test.out index 963689f..0c9f15c 100644 --- a/geocoder/admin1/extension/expected/cdb_geocoder_admin1_test.out +++ b/geocoder/admin1/extension/expected/cdb_geocoder_admin1_test.out @@ -34,7 +34,7 @@ LANGUAGE plpgsql; -- Add a few data to the sources COPY global_province_polygons (the_geom,adm1_code, objectid_1, diss_me, adm1_cod_1, iso_3166_2, wikipedia, iso_a2, adm0_sr, name, name_alt, name_local, type, type_en, code_local, code_hasc, note, hasc_maybe, region, region_cod, provnum_ne, gadm_level, check_me, scalerank, datarank, abbrev, postal, area_sqkm, sameascity, labelrank, featurecla, name_len, mapcolor9, mapcolor13, fips, fips_alt, woe_id, woe_label, woe_name, latitude, longitude, sov_a3, iso3, adm0_label, admin, geonunit, gu_a3, gn_id, gn_name, gns_id, gns_name, gn_level, gn_region, gn_a1_code, region_sub, sub_code, gns_level, gns_lang, gns_adm1, gns_region, cartodb_id, created_at, updated_at, synonyms, frequency) FROM stdin; -0106000020E6100000010000000103000000010000000400000000000000005009C07FB86AC523AF4340FFFFFFFFFFEF0EC0C3CF8DC4FADB42400000000000E0F0BFC3CF8DC4FADB424000000000005009C07FB86AC523AF4340 SVN-1035 1473 1035 SVN-1035 SI- \N SI 1 Vipava \N \N Opcine Commune|Municipality \N SI.SP.VI \N \N Goriška \N 162 2 0 10 8 \N VI 0 -99 10 Admin-1 scale rank 6 2 12 SIE1 \N -55848385 \N \N 45.8271000000000015 13.9723000000000006 SVN SVN 2 Slovenia Slovenia SVN 3239075 Obcina Vipava 243467 Vipava, Obcina 1 \N SI.E1 \N \N 1 div SI05 \N 3700 2014-02-18 19:53:50.080158+00 2014-04-01 15:18:54.094644+00 {vipava,"obcina vipava","vipava,obcina",si.e1,si.sp.vi,svn-1035,svn-1035} 5 +0106000020E6100000010000000103000000010000000400000000000000005009C07FB86AC523AF4340FFFFFFFFFFEF0EC0C3CF8DC4FADB42400000000000E0F0BFC3CF8DC4FADB424000000000005009C07FB86AC523AF4340 SVN-1035 1473 1035 SVN-1035 SI- \N SI 1 Vipava \N \N Opcine Commune|Municipality \N SI.SP.VI \N \N Goriška \N 162 2 0 10 8 \N VI 0 -99 10 Admin-1 scale rank 6 2 12 SIE1 \N -55848385 \N \N 45.8271000000000015 13.9723000000000006 SVN SVN 2 Slovenia Slovenia SVN 323075 Obcina Vipava 243467 Vipava, Obcina 1 \N SI.E1 \N \N 1 div SI05 \N 3700 2014-02-18 19:53:50.080158+00 2014-04-01 15:18:54.094644+00 {vipava,"obcina vipava","vipava,obcina",si.e1,si.sp.vi,svn-1035,svn-1035} 5 \. -- Check that the synonym function is callable, should return true From a9292f6842e56abdbce4432fae46e3f456672e17 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 3 Nov 2015 14:27:57 +0000 Subject: [PATCH 13/32] Fix test for admin1 --- .../expected/cdb_geocoder_admin1_test.out | 22 ++++++++++++++----- .../sql/cdb_geocoder_admin1_test.sql | 15 ++++++++----- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/geocoder/admin1/extension/expected/cdb_geocoder_admin1_test.out b/geocoder/admin1/extension/expected/cdb_geocoder_admin1_test.out index 0c9f15c..88fa96f 100644 --- a/geocoder/admin1/extension/expected/cdb_geocoder_admin1_test.out +++ b/geocoder/admin1/extension/expected/cdb_geocoder_admin1_test.out @@ -11,12 +11,14 @@ SELECT (geocode_admin1_polygons(Array['TX','Cuidad Real', 'sevilla'])).*; Cuidad Real | | f sevilla | | f (3 rows) + SELECT (geocode_admin1_polygons(Array['NH', 'Vermont'], 'United States')).*; q | geom | success ---------+------+--------- NH | | f Vermont | | f (2 rows) + SELECT (geocode_admin1_polygons(Array['az', 'az'], Array['Ecuador', 'USA'])).*; q | c | geom | success ----+---------+------+--------- @@ -31,15 +33,23 @@ BEGIN END $$ LANGUAGE plpgsql; - -- Add a few data to the sources -COPY global_province_polygons (the_geom,adm1_code, objectid_1, diss_me, adm1_cod_1, iso_3166_2, wikipedia, iso_a2, adm0_sr, name, name_alt, name_local, type, type_en, code_local, code_hasc, note, hasc_maybe, region, region_cod, provnum_ne, gadm_level, check_me, scalerank, datarank, abbrev, postal, area_sqkm, sameascity, labelrank, featurecla, name_len, mapcolor9, mapcolor13, fips, fips_alt, woe_id, woe_label, woe_name, latitude, longitude, sov_a3, iso3, adm0_label, admin, geonunit, gu_a3, gn_id, gn_name, gns_id, gns_name, gn_level, gn_region, gn_a1_code, region_sub, sub_code, gns_level, gns_lang, gns_adm1, gns_region, cartodb_id, created_at, updated_at, synonyms, frequency) FROM stdin; -0106000020E6100000010000000103000000010000000400000000000000005009C07FB86AC523AF4340FFFFFFFFFFEF0EC0C3CF8DC4FADB42400000000000E0F0BFC3CF8DC4FADB424000000000005009C07FB86AC523AF4340 SVN-1035 1473 1035 SVN-1035 SI- \N SI 1 Vipava \N \N Opcine Commune|Municipality \N SI.SP.VI \N \N Goriška \N 162 2 0 10 8 \N VI 0 -99 10 Admin-1 scale rank 6 2 12 SIE1 \N -55848385 \N \N 45.8271000000000015 13.9723000000000006 SVN SVN 2 Slovenia Slovenia SVN 323075 Obcina Vipava 243467 Vipava, Obcina 1 \N SI.E1 \N \N 1 div SI05 \N 3700 2014-02-18 19:53:50.080158+00 2014-04-01 15:18:54.094644+00 {vipava,"obcina vipava","vipava,obcina",si.e1,si.sp.vi,svn-1035,svn-1035} 5 -\. - +INSERT INTO global_province_polygons (the_geom, synonyms, iso3) VALUES ( + '0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040', + Array['vipava','obcina vipava','vipava, obcina'], + 'SVN' +); -- Check that the synonym function is callable, should return true SELECT (geocode_admin1_polygons(Array['obcina vipava'])).success; success --------- t -(1 row) \ No newline at end of file +(1 row) + +-- Check that it returns the mocked geometry above +SELECT (geocode_admin1_polygons(Array['obcina vipava'])).geom; + geom +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + 0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040 +(1 row) + diff --git a/geocoder/admin1/extension/sql/cdb_geocoder_admin1_test.sql b/geocoder/admin1/extension/sql/cdb_geocoder_admin1_test.sql index 276317d..8be88fc 100644 --- a/geocoder/admin1/extension/sql/cdb_geocoder_admin1_test.sql +++ b/geocoder/admin1/extension/sql/cdb_geocoder_admin1_test.sql @@ -18,12 +18,15 @@ $$ LANGUAGE plpgsql; -- Add a few data to the sources -COPY global_province_polygons (the_geom, adm1_code, objectid_1, diss_me, adm1_cod_1, iso_3166_2, wikipedia, iso_a2, adm0_sr, name, name_alt, name_local, type, type_en, code_local, code_hasc, note, hasc_maybe, region, region_cod, provnum_ne, gadm_level, check_me, scalerank, datarank, abbrev, postal, area_sqkm, sameascity, labelrank, featurecla, name_len, mapcolor9, mapcolor13, fips, fips_alt, woe_id, woe_label, woe_name, latitude, longitude, sov_a3, iso3, adm0_label, admin, geonunit, gu_a3, gn_id, gn_name, gns_id, gns_name, gn_level, gn_region, gn_a1_code, region_sub, sub_code, gns_level, gns_lang, gns_adm1, gns_region, cartodb_id, created_at, updated_at, the_geom_webmercator, synonyms, frequency) FROM stdin; -0106000020E6100000010000000103000000010000001C000000701C468693172C405BCFC9FB20EB464010E0C037081E2C40F7FF710981E74640606B2301AC1A2C40B407F55702E64640200BD34801192C40A44EE24698E34640503B8E674E162C4084868DF1EEE246400025D4480D132C40B43156A168E14640701EE41F19082C400CFADB0CA1E24640E0F6B6C345062C401815CAFBB1E24640A00B4B5DB3022C40EF46C124BCE2464010E49A3E6E012C40CB84EFBA8CE24640105943E9D0FA2B40ECD210DDEBE14640008981CA8FF72B40071D4DCAEFE24640B05E6F7FDDDC2B40F3246E6C88E5464060FB4FFA19CE2B40A46CBD87B5E746404058ECF694C52B408CF11C3411EA4640FFF7D7E59FC82B40BCC34EE79AEB46400077C737A4D02B403C3FEF3AB8EC4640E04B53FA05D82B4090EF60F885ED464090EBDCBC6BDD2B40E048DFA985EE464000A5771CC0E92B40BC5B8EF1D4EF4640005BBBFD66F22B40208A181748F04640805CE1823AF92B402CA5060659F0464040FC6A45A0FE2B404B4FC4C11DF046408061640B9F022C4060B922EEBEEF4640FFD76EE28A072C405421278BBDEE4640E09BADEC49092C40FB7EEC1D7AED4640AF09D3481D0B2C40D86DDB0C8DEC4640701C468693172C405BCFC9FB20EB4640 SVN-1035 1473 1035 SVN-1035 SI- \N SI 1 Vipava \N \N Opcine Commune|Municipality \N SI.SP.VI \N - \N Goriška \N 162 2 0 10 8 \N VI 0 -99 10 Admin-1 scale rank 6 2 12 SIE1 \N -55848385 - \N \N 45.8271000000000015 13.9723000000000006 SVN SVN 2 Slovenia Slovenia SVN 3239075 Obcina Vipava 243467 Vipava, Obcina 1 - \N SI.E1 \N \N 1 div SI05 \N 3700 2014-02-18 19:53:50.080158+00 2014-04-01 15:18:54.094644+00 0106000020110F0000010000000103000000010000001C00000023340ED8CEDB3741C0404B7264F35541C85B157A4AE13741B323798AF9EE5541CEDCBDE56FDE37414004235027ED5541489EE87B05DD3741D970658A36EA55418D272EB8BADA374116C6965868E95541A8917D26F7D73741F9BBEB238DE75541A6C99293A9CE37417F31C28009E95541DA9DBCA51CCD3741A5FAE31E1EE9554121BD100C14CA37414E69CD7D2AE95541124037ECFFC8374162390EC3F0E85541332B2FC661C337417AB6C1E72CE855418A0578349EC037417308826069E95541BCFC69D6F1A93741B086FDCC92EC55419F7FD8E3679D3741462E0A8039EF554160E7068E2B963741A150E52919F25541861B631AC198374148B84A13F9F3554123289F23909F3741CCF276F254F55541F49EE41CD5A53741895CFAD14FF655415E8C59A46AAA3741EE95F4A487F755414B69D859E3B43741E59BB99720F95541A6ECFD723CBC3741762F910EADF9554144635A9E08C23741DD3BD2B6C1F95541C650CF259EC637415377246A79F9554105D616CA02CA374149377DBD05F9554170AA9CC530CE3741A52C14CCCBF755419A5F7271ACCF37414FD3A85D41F65541A2FB415F39D13741C55D614C20F5554123340ED8CEDB3741C0404B7264F35541 {vipava,"obcina vipava","vipava, obcina",si.e1,si.sp.vi,svn-1035,svn-1035} 5 -\. +INSERT INTO global_province_polygons (the_geom, synonyms, iso3) VALUES ( + '0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040', + Array['vipava','obcina vipava','vipava, obcina'], + 'SVN' +); -- Check that the synonym function is callable, should return true SELECT (geocode_admin1_polygons(Array['obcina vipava'])).success; + +-- Check that it returns the mocked geometry above +SELECT (geocode_admin1_polygons(Array['obcina vipava'])).geom; + From ddd66aff08d344ce8253640ab9c06b281944931a Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 3 Nov 2015 16:03:44 +0100 Subject: [PATCH 14/32] First try at moving admin1 to merged extension --- geocoder/admin1/extension/.gitignore | 3 -- geocoder/admin1/extension/Makefile | 8 ----- geocoder/admin1/extension/README.md | 36 ------------------- .../extension/cdb_geocoder_admin1.control | 6 ---- geocoder/extension/Makefile | 4 ++- geocoder/extension/README.md | 6 ++++ .../expected/cdb_geocoder_admin1_test.out | 0 .../sql/0.0.1/20_admin1.sql} | 3 -- .../sql/cdb_geocoder_admin1_test.sql | 0 9 files changed, 9 insertions(+), 57 deletions(-) delete mode 100644 geocoder/admin1/extension/.gitignore delete mode 100644 geocoder/admin1/extension/Makefile delete mode 100644 geocoder/admin1/extension/README.md delete mode 100644 geocoder/admin1/extension/cdb_geocoder_admin1.control rename geocoder/{admin1 => }/extension/expected/cdb_geocoder_admin1_test.out (100%) rename geocoder/{admin1/extension/cdb_geocoder_admin1--0.0.1.sql => extension/sql/0.0.1/20_admin1.sql} (98%) rename geocoder/{admin1 => }/extension/sql/cdb_geocoder_admin1_test.sql (100%) diff --git a/geocoder/admin1/extension/.gitignore b/geocoder/admin1/extension/.gitignore deleted file mode 100644 index e710f0e..0000000 --- a/geocoder/admin1/extension/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -results/ -regression.diffs -regression.out diff --git a/geocoder/admin1/extension/Makefile b/geocoder/admin1/extension/Makefile deleted file mode 100644 index b472643..0000000 --- a/geocoder/admin1/extension/Makefile +++ /dev/null @@ -1,8 +0,0 @@ -EXTENSION = cdb_geocoder_admin1 -DATA = cdb_geocoder_admin1--0.0.1.sql -REGRESS = cdb_geocoder_admin1_test - -# postgres build stuff -PG_CONFIG = pg_config -PGXS := $(shell $(PG_CONFIG) --pgxs) -include $(PGXS) diff --git a/geocoder/admin1/extension/README.md b/geocoder/admin1/extension/README.md deleted file mode 100644 index ac99e36..0000000 --- a/geocoder/admin1/extension/README.md +++ /dev/null @@ -1,36 +0,0 @@ -# CartoDB admin1 geocoder extension -Postgres extension for the CartoDB admin1 geocoder. It is meant to contain the functions and related objects needed to geocode by admin1 regions. It is not meant to contain the actual data used to geocode them. - -## Dependencies -This extension is thought to be used on top of CartoDB platform. Therefore a cartodb user is required to install the extension onto it. - -The following is a non-comprehensive list of dependencies: - -- Postgres 9.3+ -- Postgis extension -- Schema triggers extension -- CartoDB extension - -## Installation into the db cluster -This requires root privileges -``` -sudo make all install -``` - -## Execute tests -``` -PGUSER=postgres make installcheck -``` - -## Install onto a user's database -``` -psql -U development_cartodb_user_fe3b850a-01c0-48f9-8a26-a82f09e9b53f cartodb_dev_user_fe3b850a-01c0-48f9-8a26-a82f09e9b53f_db -``` - -and then: - -```sql -CREATE EXTENSION cdb_geocoder_admin1; -``` - -The extension creation in the user's db does not require special privileges. It can be even created from the sql api. diff --git a/geocoder/admin1/extension/cdb_geocoder_admin1.control b/geocoder/admin1/extension/cdb_geocoder_admin1.control deleted file mode 100644 index 7621dcb..0000000 --- a/geocoder/admin1/extension/cdb_geocoder_admin1.control +++ /dev/null @@ -1,6 +0,0 @@ -# cdb geocoder admin1 extension -comment = 'CartoDB admin1 internal geocoder' -default_version = '0.0.1' -relocatable = true -requires = cartodb -superuser = false diff --git a/geocoder/extension/Makefile b/geocoder/extension/Makefile index d79a16c..288e327 100644 --- a/geocoder/extension/Makefile +++ b/geocoder/extension/Makefile @@ -4,7 +4,9 @@ EXTENSION = cdb_geocoder EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/") DATA = $(EXTENSION)--$(EXTVERSION).sql -REGRESS = cdb_geocoder_admin0_test +REGRESS = \ + cdb_geocoder_admin0_test \ + cdb_geocoder_admin1_test # postgres build stuff PG_CONFIG = pg_config diff --git a/geocoder/extension/README.md b/geocoder/extension/README.md index dfa5430..bee0de1 100644 --- a/geocoder/extension/README.md +++ b/geocoder/extension/README.md @@ -22,6 +22,12 @@ sudo make all install PGUSER=postgres make installcheck ``` +## Build, install & test +One-liner: +``` +sudo PGUSER=postgres make all install installcheck +``` + ## Install onto a user's database ``` psql -U development_cartodb_user_fe3b850a-01c0-48f9-8a26-a82f09e9b53f cartodb_dev_user_fe3b850a-01c0-48f9-8a26-a82f09e9b53f_db diff --git a/geocoder/admin1/extension/expected/cdb_geocoder_admin1_test.out b/geocoder/extension/expected/cdb_geocoder_admin1_test.out similarity index 100% rename from geocoder/admin1/extension/expected/cdb_geocoder_admin1_test.out rename to geocoder/extension/expected/cdb_geocoder_admin1_test.out diff --git a/geocoder/admin1/extension/cdb_geocoder_admin1--0.0.1.sql b/geocoder/extension/sql/0.0.1/20_admin1.sql similarity index 98% rename from geocoder/admin1/extension/cdb_geocoder_admin1--0.0.1.sql rename to geocoder/extension/sql/0.0.1/20_admin1.sql index 617beee..3d9060d 100644 --- a/geocoder/admin1/extension/cdb_geocoder_admin1--0.0.1.sql +++ b/geocoder/extension/sql/0.0.1/20_admin1.sql @@ -1,6 +1,3 @@ --- Complain if script is sourced in psql, rather than via CREATE EXTENSION -\echo Use "CREATE EXTENSION cdb_geocoder_admin1" to load this file. \quit - -- Response types for admin1 geocoder -- TODO: check if the types exist already in the db diff --git a/geocoder/admin1/extension/sql/cdb_geocoder_admin1_test.sql b/geocoder/extension/sql/cdb_geocoder_admin1_test.sql similarity index 100% rename from geocoder/admin1/extension/sql/cdb_geocoder_admin1_test.sql rename to geocoder/extension/sql/cdb_geocoder_admin1_test.sql From 5a7e1ca26207943a0ef92894c4c76e60f9419160 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 3 Nov 2015 16:11:17 +0100 Subject: [PATCH 15/32] Add an install test --- geocoder/extension/Makefile | 1 + geocoder/extension/expected/00_install_test.out | 14 ++++++++++++++ .../expected/cdb_geocoder_admin0_test.out | 12 ------------ .../expected/cdb_geocoder_admin1_test.out | 12 ------------ geocoder/extension/sql/0.0.1/20_admin1.sql | 1 - geocoder/extension/sql/00_install_test.sql | 16 ++++++++++++++++ .../extension/sql/cdb_geocoder_admin0_test.sql | 14 -------------- .../extension/sql/cdb_geocoder_admin1_test.sql | 14 -------------- 8 files changed, 31 insertions(+), 53 deletions(-) create mode 100644 geocoder/extension/expected/00_install_test.out create mode 100644 geocoder/extension/sql/00_install_test.sql diff --git a/geocoder/extension/Makefile b/geocoder/extension/Makefile index 288e327..2985b9b 100644 --- a/geocoder/extension/Makefile +++ b/geocoder/extension/Makefile @@ -5,6 +5,7 @@ EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/defau DATA = $(EXTENSION)--$(EXTVERSION).sql REGRESS = \ + 00_install_test \ cdb_geocoder_admin0_test \ cdb_geocoder_admin1_test diff --git a/geocoder/extension/expected/00_install_test.out b/geocoder/extension/expected/00_install_test.out new file mode 100644 index 0000000..1bc65bc --- /dev/null +++ b/geocoder/extension/expected/00_install_test.out @@ -0,0 +1,14 @@ +-- Install dependencies +CREATE EXTENSION postgis; +CREATE EXTENSION schema_triggers; +CREATE EXTENSION plpythonu; +CREATE EXTENSION cartodb; +-- Install the extension +CREATE EXTENSION cdb_geocoder; +-- Mock the varnish invalidation function +CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$ +BEGIN + RETURN; +END +$$ +LANGUAGE plpgsql; diff --git a/geocoder/extension/expected/cdb_geocoder_admin0_test.out b/geocoder/extension/expected/cdb_geocoder_admin0_test.out index c6e846c..f4ae329 100644 --- a/geocoder/extension/expected/cdb_geocoder_admin0_test.out +++ b/geocoder/extension/expected/cdb_geocoder_admin0_test.out @@ -1,8 +1,3 @@ -CREATE EXTENSION postgis; -CREATE EXTENSION schema_triggers; -CREATE EXTENSION plpythonu; -CREATE EXTENSION cartodb; -CREATE EXTENSION cdb_geocoder_admin0; -- Check that the synonym function is callable, should return NULL SELECT (admin0_synonym_lookup(Array['United States', 'ESP'])).*; q | adm0_a3 @@ -20,13 +15,6 @@ SELECT (geocode_admin0_polygons(Array['Spain', 'USA', ''])).*; USA | | f (3 rows) --- Mock the varnish invalidation function -CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$ -BEGIN - RETURN; -END -$$ -LANGUAGE plpgsql; -- Add a few synonyms COPY admin0_synonyms (name, rank, created_at, updated_at, the_geom, the_geom_webmercator, cartodb_id, adm0_a3, name_) FROM stdin; -- Check that the synonym function is callable, should return their iso codes diff --git a/geocoder/extension/expected/cdb_geocoder_admin1_test.out b/geocoder/extension/expected/cdb_geocoder_admin1_test.out index 88fa96f..fee381e 100644 --- a/geocoder/extension/expected/cdb_geocoder_admin1_test.out +++ b/geocoder/extension/expected/cdb_geocoder_admin1_test.out @@ -1,8 +1,3 @@ -CREATE EXTENSION postgis; -CREATE EXTENSION schema_triggers; -CREATE EXTENSION plpythonu; -CREATE EXTENSION cartodb; -CREATE EXTENSION cdb_geocoder_admin1; -- Check that the geocoding functions are callable, should return NULL SELECT (geocode_admin1_polygons(Array['TX','Cuidad Real', 'sevilla'])).*; q | geom | success @@ -26,13 +21,6 @@ SELECT (geocode_admin1_polygons(Array['az', 'az'], Array['Ecuador', 'USA'])).*; az | USA | | f (2 rows) --- Mock the varnish invalidation function -CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$ -BEGIN - RETURN; -END -$$ -LANGUAGE plpgsql; -- Add a few data to the sources INSERT INTO global_province_polygons (the_geom, synonyms, iso3) VALUES ( '0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040', diff --git a/geocoder/extension/sql/0.0.1/20_admin1.sql b/geocoder/extension/sql/0.0.1/20_admin1.sql index 3d9060d..f2a85f7 100644 --- a/geocoder/extension/sql/0.0.1/20_admin1.sql +++ b/geocoder/extension/sql/0.0.1/20_admin1.sql @@ -1,7 +1,6 @@ -- Response types for admin1 geocoder -- TODO: check if the types exist already in the db -CREATE TYPE geocode_admin_v1 AS (q TEXT, geom GEOMETRY, success BOOLEAN); CREATE TYPE geocode_admin_country_v1 AS (q TEXT, c TEXT, geom GEOMETRY, success BOOLEAN); diff --git a/geocoder/extension/sql/00_install_test.sql b/geocoder/extension/sql/00_install_test.sql new file mode 100644 index 0000000..6bbc3d4 --- /dev/null +++ b/geocoder/extension/sql/00_install_test.sql @@ -0,0 +1,16 @@ +-- Install dependencies +CREATE EXTENSION postgis; +CREATE EXTENSION schema_triggers; +CREATE EXTENSION plpythonu; +CREATE EXTENSION cartodb; + +-- Install the extension +CREATE EXTENSION cdb_geocoder; + +-- Mock the varnish invalidation function +CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$ +BEGIN + RETURN; +END +$$ +LANGUAGE plpgsql; diff --git a/geocoder/extension/sql/cdb_geocoder_admin0_test.sql b/geocoder/extension/sql/cdb_geocoder_admin0_test.sql index 4a9916f..3ce1a11 100644 --- a/geocoder/extension/sql/cdb_geocoder_admin0_test.sql +++ b/geocoder/extension/sql/cdb_geocoder_admin0_test.sql @@ -1,23 +1,9 @@ -CREATE EXTENSION postgis; -CREATE EXTENSION schema_triggers; -CREATE EXTENSION plpythonu; -CREATE EXTENSION cartodb; -CREATE EXTENSION cdb_geocoder_admin0; - -- Check that the synonym function is callable, should return NULL SELECT (admin0_synonym_lookup(Array['United States', 'ESP'])).*; -- Check that the geocoding function is callable, should return success = false SELECT (geocode_admin0_polygons(Array['Spain', 'USA', ''])).*; --- Mock the varnish invalidation function -CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$ -BEGIN - RETURN; -END -$$ -LANGUAGE plpgsql; - -- Add a few synonyms COPY admin0_synonyms (name, rank, created_at, updated_at, the_geom, the_geom_webmercator, cartodb_id, adm0_a3, name_) FROM stdin; United States 0 2014-09-30 10:13:28.383426+00 2014-09-30 10:13:28.383426+00 \N \N 29614 USA unitedstates diff --git a/geocoder/extension/sql/cdb_geocoder_admin1_test.sql b/geocoder/extension/sql/cdb_geocoder_admin1_test.sql index 8be88fc..5d64f54 100644 --- a/geocoder/extension/sql/cdb_geocoder_admin1_test.sql +++ b/geocoder/extension/sql/cdb_geocoder_admin1_test.sql @@ -1,22 +1,8 @@ -CREATE EXTENSION postgis; -CREATE EXTENSION schema_triggers; -CREATE EXTENSION plpythonu; -CREATE EXTENSION cartodb; -CREATE EXTENSION cdb_geocoder_admin1; - -- Check that the geocoding functions are callable, should return NULL SELECT (geocode_admin1_polygons(Array['TX','Cuidad Real', 'sevilla'])).*; SELECT (geocode_admin1_polygons(Array['NH', 'Vermont'], 'United States')).*; SELECT (geocode_admin1_polygons(Array['az', 'az'], Array['Ecuador', 'USA'])).*; --- Mock the varnish invalidation function -CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$ -BEGIN - RETURN; -END -$$ -LANGUAGE plpgsql; - -- Add a few data to the sources INSERT INTO global_province_polygons (the_geom, synonyms, iso3) VALUES ( '0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040', From 9d562b624af3547cfbce774c514cead05401d65c Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 3 Nov 2015 15:34:49 +0000 Subject: [PATCH 16/32] Fix ipaddr tests --- .../extension/expected/cdb_geocoder_ipaddr_test.out | 1 + .../extension/sql/cdb_geocoder_ipaddr_test.sql | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/geocoder/ip-addresses/extension/expected/cdb_geocoder_ipaddr_test.out b/geocoder/ip-addresses/extension/expected/cdb_geocoder_ipaddr_test.out index 1834323..03cb840 100644 --- a/geocoder/ip-addresses/extension/expected/cdb_geocoder_ipaddr_test.out +++ b/geocoder/ip-addresses/extension/expected/cdb_geocoder_ipaddr_test.out @@ -25,3 +25,4 @@ SELECT (geocode_ip(Array['2.235.35.0'])).*; ------------+----------------------------------------------------+--------- 2.235.35.0 | 0101000020E610000072F90FE9B7CF22405DFE43FAEDC34640 | t (1 row) + diff --git a/geocoder/ip-addresses/extension/sql/cdb_geocoder_ipaddr_test.sql b/geocoder/ip-addresses/extension/sql/cdb_geocoder_ipaddr_test.sql index 6334ee4..ba3071b 100644 --- a/geocoder/ip-addresses/extension/sql/cdb_geocoder_ipaddr_test.sql +++ b/geocoder/ip-addresses/extension/sql/cdb_geocoder_ipaddr_test.sql @@ -18,10 +18,10 @@ LANGUAGE plpgsql; -- Add a few IP sources COPY ip_address_locations (network_start_ip, the_geom, cartodb_id, created_at, updated_at, the_geom_webmercator) FROM stdin; ::ffff:2.235.35.0 0101000020E610000072F90FE9B7CF22405DFE43FAEDC34640 2821226 2014-08-25 10:35:51.665546+00 2014-08-25 10:35:51.665546+00 0101000020110F000010801778FBF32F4109868FF8BCC35541 -::ffff:31.7.187.0 \N 2783250 2014-08-25 10:35:51.665546+00 2014-08-25 10:35:51.665546+00 \N -::ffff:64.110.146.0 \N 2783251 2014-08-25 10:35:51.665546+00 2014-08-25 10:35:51.665546+00 \N -::ffff:72.5.198.0 \N 2783252 2014-08-25 10:35:51.665546+00 2014-08-25 10:35:51.665546+00 \N -::ffff:77.73.184.0 \N 2783253 2014-08-25 10:35:51.665546+00 2014-08-25 10:35:51.665546+00 \N +::ffff:31.7.187.0 \N 2783250 2014-08-25 10:35:51.665546+00 2014-08-25 10:35:51.665546+00 \N +::ffff:64.110.146.0 \N 2783251 2014-08-25 10:35:51.665546+00 2014-08-25 10:35:51.665546+00 \N +::ffff:72.5.198.0 \N 2783252 2014-08-25 10:35:51.665546+00 2014-08-25 10:35:51.665546+00 \N +::ffff:77.73.184.0 \N 2783253 2014-08-25 10:35:51.665546+00 2014-08-25 10:35:51.665546+00 \N \. -- Check that the geocoding function is callable, should return success = true From 006440ed521cf9bfde72aacd09b214fc03530f7c Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 3 Nov 2015 15:51:08 +0000 Subject: [PATCH 17/32] Fix test (missing trailing comma) --- .../namedplace/extension/sql/cdb_geocoder_namedplaces_test.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geocoder/namedplace/extension/sql/cdb_geocoder_namedplaces_test.sql b/geocoder/namedplace/extension/sql/cdb_geocoder_namedplaces_test.sql index 6f5c422..cbc2c91 100644 --- a/geocoder/namedplace/extension/sql/cdb_geocoder_namedplaces_test.sql +++ b/geocoder/namedplace/extension/sql/cdb_geocoder_namedplaces_test.sql @@ -10,7 +10,7 @@ SELECT (geocode_namedplace(Array['Elche', 'Granada', 'Madrid'], 'Spain')).*; SELECT (geocode_namedplace(Array['sunapee', 'sunapeeee', 'New York City', 'Madrid'], Array['', 'US', 'United States', NULL])).*; SELECT (geocode_namedplace(Array['Portland', 'Portland', 'New York City'], Array['Maine', 'Oregon', NULL], 'USA')).*; SELECT (geocode_namedplace(Array['Portland'], 'Oregon', 'USA')).*; -SELECT (geocode_namedplace(Array['Portland', 'Portland', 'New York City'], Array['Maine', 'Oregon', NULL], Array['USA'])).* +SELECT (geocode_namedplace(Array['Portland', 'Portland', 'New York City'], Array['Maine', 'Oregon', NULL], Array['USA'])).*; -- Mock the varnish invalidation function From 710db23d60982ec7ded18540ba7f7bf87f166933 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 3 Nov 2015 15:51:41 +0000 Subject: [PATCH 18/32] Fix expectation of namedplaces test --- .../expected/cdb_geocoder_namedplaces_test.out | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/geocoder/namedplace/extension/expected/cdb_geocoder_namedplaces_test.out b/geocoder/namedplace/extension/expected/cdb_geocoder_namedplaces_test.out index d09e4ce..53d0b30 100644 --- a/geocoder/namedplace/extension/expected/cdb_geocoder_namedplaces_test.out +++ b/geocoder/namedplace/extension/expected/cdb_geocoder_namedplaces_test.out @@ -11,13 +11,15 @@ SELECT (geocode_namedplace(Array['Madrid', 'New York City', 'sunapee'])).*; New York City | | f sunapee | | f (3 rows) + SELECT (geocode_namedplace(Array['Elche', 'Granada', 'Madrid'], 'Spain')).*; q | c | geom | success ---------+-------+------+--------- Elche | Spain | | f Granada | Spain | | f Madrid | Spain | | f - (3 rows) +(3 rows) + SELECT (geocode_namedplace(Array['sunapee', 'sunapeeee', 'New York City', 'Madrid'], Array['', 'US', 'United States', NULL])).*; q | c | geom | success ---------------+---------------+------+--------- @@ -26,6 +28,7 @@ SELECT (geocode_namedplace(Array['sunapee', 'sunapeeee', 'New York City', 'Madri sunapeeee | US | | f New York City | United States | | f (4 rows) + SELECT (geocode_namedplace(Array['Portland', 'Portland', 'New York City'], Array['Maine', 'Oregon', NULL], 'USA')).*; q | a1 | c | geom | success ---------------+--------+-----+------+--------- @@ -33,11 +36,13 @@ SELECT (geocode_namedplace(Array['Portland', 'Portland', 'New York City'], Array Portland | Maine | USA | | f Portland | Oregon | USA | | f (3 rows) + SELECT (geocode_namedplace(Array['Portland'], 'Oregon', 'USA')).*; q | a1 | c | geom | success ----------+--------+-----+------+--------- Portland | Oregon | USA | | f (1 row) + SELECT (geocode_namedplace(Array['Portland', 'Portland', 'New York City'], Array['Maine', 'Oregon', NULL], Array['USA'])).*; q | a1 | c | geom | success ---------------+--------+-----+------+--------- @@ -61,4 +66,5 @@ SELECT (geocode_namedplace(Array['Barcelona'])).* q | geom | success -----------+----------------------------------------------------+--------- Barcelona | 0101000020E6100000CA15DEE522E653C0A4C2D842902B4540 | t -(1 row) \ No newline at end of file +(1 row) + From 7152048707ac502628ee8393a7080d0dc794ebe1 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 3 Nov 2015 17:17:13 +0100 Subject: [PATCH 19/32] Move ipaddr into common geocoder --- geocoder/extension/Makefile | 3 +- .../expected/cdb_geocoder_ipaddr_test.out | 12 ------- .../sql/0.0.1/30_ipaddr.sql} | 3 -- .../sql/cdb_geocoder_ipaddr_test.sql | 14 -------- geocoder/ip-addresses/extension/.gitignore | 3 -- geocoder/ip-addresses/extension/Makefile | 8 ----- geocoder/ip-addresses/extension/README.md | 36 ------------------- .../extension/cdb_geocoder_ipaddr.control | 6 ---- 8 files changed, 2 insertions(+), 83 deletions(-) rename geocoder/{ip-addresses => }/extension/expected/cdb_geocoder_ipaddr_test.out (68%) rename geocoder/{ip-addresses/extension/cdb_geocoder_ipaddr--0.0.1.sql => extension/sql/0.0.1/30_ipaddr.sql} (95%) rename geocoder/{ip-addresses => }/extension/sql/cdb_geocoder_ipaddr_test.sql (74%) delete mode 100644 geocoder/ip-addresses/extension/.gitignore delete mode 100644 geocoder/ip-addresses/extension/Makefile delete mode 100644 geocoder/ip-addresses/extension/README.md delete mode 100644 geocoder/ip-addresses/extension/cdb_geocoder_ipaddr.control diff --git a/geocoder/extension/Makefile b/geocoder/extension/Makefile index 2985b9b..ac1778e 100644 --- a/geocoder/extension/Makefile +++ b/geocoder/extension/Makefile @@ -7,7 +7,8 @@ DATA = $(EXTENSION)--$(EXTVERSION).sql REGRESS = \ 00_install_test \ cdb_geocoder_admin0_test \ - cdb_geocoder_admin1_test + cdb_geocoder_admin1_test \ + cdb_geocoder_ipaddr_test # postgres build stuff PG_CONFIG = pg_config diff --git a/geocoder/ip-addresses/extension/expected/cdb_geocoder_ipaddr_test.out b/geocoder/extension/expected/cdb_geocoder_ipaddr_test.out similarity index 68% rename from geocoder/ip-addresses/extension/expected/cdb_geocoder_ipaddr_test.out rename to geocoder/extension/expected/cdb_geocoder_ipaddr_test.out index 03cb840..9b7bd57 100644 --- a/geocoder/ip-addresses/extension/expected/cdb_geocoder_ipaddr_test.out +++ b/geocoder/extension/expected/cdb_geocoder_ipaddr_test.out @@ -1,8 +1,3 @@ -CREATE EXTENSION postgis; -CREATE EXTENSION schema_triggers; -CREATE EXTENSION plpythonu; -CREATE EXTENSION cartodb; -CREATE EXTENSION cdb_geocoder_ipaddr; -- Check that the geocoding function is callable, should return success = false SELECT (geocode_ip(Array['100.0.24.0'])).*; q | geom | success @@ -10,13 +5,6 @@ SELECT (geocode_ip(Array['100.0.24.0'])).*; 100.0.24.0 | | f (1 row) --- Mock the varnish invalidation function -CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$ -BEGIN - RETURN; -END -$$ -LANGUAGE plpgsql; -- Add a few IP sources COPY ip_address_locations (network_start_ip, the_geom, cartodb_id, created_at, updated_at, the_geom_webmercator) FROM stdin; -- Check that the geocoding function is callable, should return success = true diff --git a/geocoder/ip-addresses/extension/cdb_geocoder_ipaddr--0.0.1.sql b/geocoder/extension/sql/0.0.1/30_ipaddr.sql similarity index 95% rename from geocoder/ip-addresses/extension/cdb_geocoder_ipaddr--0.0.1.sql rename to geocoder/extension/sql/0.0.1/30_ipaddr.sql index 7ce8c6b..8560aeb 100644 --- a/geocoder/ip-addresses/extension/cdb_geocoder_ipaddr--0.0.1.sql +++ b/geocoder/extension/sql/0.0.1/30_ipaddr.sql @@ -1,6 +1,3 @@ --- Complain if script is sourced in psql, rather than via CREATE EXTENSION -\echo Use "CREATE EXTENSION cdb_geocoder_ipaddr" to load this file. \quit - -- Response types for IP addresses geocoder CREATE TYPE geocode_ip_v1 AS (q text, geom geometry, success boolean); diff --git a/geocoder/ip-addresses/extension/sql/cdb_geocoder_ipaddr_test.sql b/geocoder/extension/sql/cdb_geocoder_ipaddr_test.sql similarity index 74% rename from geocoder/ip-addresses/extension/sql/cdb_geocoder_ipaddr_test.sql rename to geocoder/extension/sql/cdb_geocoder_ipaddr_test.sql index ba3071b..74ad5fa 100644 --- a/geocoder/ip-addresses/extension/sql/cdb_geocoder_ipaddr_test.sql +++ b/geocoder/extension/sql/cdb_geocoder_ipaddr_test.sql @@ -1,20 +1,6 @@ -CREATE EXTENSION postgis; -CREATE EXTENSION schema_triggers; -CREATE EXTENSION plpythonu; -CREATE EXTENSION cartodb; -CREATE EXTENSION cdb_geocoder_ipaddr; - -- Check that the geocoding function is callable, should return success = false SELECT (geocode_ip(Array['100.0.24.0'])).*; --- Mock the varnish invalidation function -CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$ -BEGIN - RETURN; -END -$$ -LANGUAGE plpgsql; - -- Add a few IP sources COPY ip_address_locations (network_start_ip, the_geom, cartodb_id, created_at, updated_at, the_geom_webmercator) FROM stdin; ::ffff:2.235.35.0 0101000020E610000072F90FE9B7CF22405DFE43FAEDC34640 2821226 2014-08-25 10:35:51.665546+00 2014-08-25 10:35:51.665546+00 0101000020110F000010801778FBF32F4109868FF8BCC35541 diff --git a/geocoder/ip-addresses/extension/.gitignore b/geocoder/ip-addresses/extension/.gitignore deleted file mode 100644 index e710f0e..0000000 --- a/geocoder/ip-addresses/extension/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -results/ -regression.diffs -regression.out diff --git a/geocoder/ip-addresses/extension/Makefile b/geocoder/ip-addresses/extension/Makefile deleted file mode 100644 index 922a44b..0000000 --- a/geocoder/ip-addresses/extension/Makefile +++ /dev/null @@ -1,8 +0,0 @@ -EXTENSION = cdb_geocoder_ipaddr -DATA = cdb_geocoder_ipaddr--0.0.1.sql -REGRESS = cdb_geocoder_ipaddr_test - -# postgres build stuff -PG_CONFIG = pg_config -PGXS := $(shell $(PG_CONFIG) --pgxs) -include $(PGXS) diff --git a/geocoder/ip-addresses/extension/README.md b/geocoder/ip-addresses/extension/README.md deleted file mode 100644 index 9944371..0000000 --- a/geocoder/ip-addresses/extension/README.md +++ /dev/null @@ -1,36 +0,0 @@ -# CartoDB IP addresses geocoder extension -Postgres extension for the CartoDB IP addresses geocoder. It is meant to contain the functions and related objects needed to geocode by IP addresses. It is not meant to contain the actual data used to geocode them. - -## Dependencies -This extension is thought to be used on top of CartoDB platform. Therefore a cartodb user is required to install the extension onto it. - -The following is a non-comprehensive list of dependencies: - -- Postgres 9.3+ -- Postgis extension -- Schema triggers extension -- CartoDB extension - -## Installation into the db cluster -This requires root privileges -``` -sudo make all install -``` - -## Execute tests -``` -PGUSER=postgres make installcheck -``` - -## Install onto a user's database -``` -psql -U cartodb_dev_user_367c0edc-b2ad-4bab-ad43-3d58a6179a93_db cartodb_dev_user_367c0edc-b2ad-4bab-ad43-3d58a6179a93_db -``` - -and then: - -```sql -CREATE EXTENSION cdb_geocoder_ipaddr; -``` - -The extension creation in the user's db does not require special privileges. It can be even created from the sql api. diff --git a/geocoder/ip-addresses/extension/cdb_geocoder_ipaddr.control b/geocoder/ip-addresses/extension/cdb_geocoder_ipaddr.control deleted file mode 100644 index af9e981..0000000 --- a/geocoder/ip-addresses/extension/cdb_geocoder_ipaddr.control +++ /dev/null @@ -1,6 +0,0 @@ -# cdb geocoder ipaddr extension -comment = 'CartoDB admin0 internal geocoder' -default_version = '0.0.1' -relocatable = true -requires = cartodb -superuser = false From 745c0a8b0a37b008768de50401ee0c705f2067d2 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 3 Nov 2015 17:35:40 +0100 Subject: [PATCH 20/32] Move namedplaces to cbd_geocoder extension --- geocoder/extension/Makefile | 3 +- .../cdb_geocoder_namedplaces_test.out | 12 ---- .../sql/0.0.1/15_country_decoder_table.sql | 54 +++++++++++++++++ geocoder/extension/sql/0.0.1/20_admin1.sql | 58 ------------------ .../sql/0.0.1/40_namedplaces.sql} | 60 ------------------- .../sql/cdb_geocoder_namedplaces_test.sql | 15 ----- geocoder/namedplace/extension/.gitignore | 3 - geocoder/namedplace/extension/Makefile | 8 --- geocoder/namedplace/extension/README.md | 36 ----------- .../cdb_geocoder_namedplaces.control | 6 -- 10 files changed, 56 insertions(+), 199 deletions(-) rename geocoder/{namedplace => }/extension/expected/cdb_geocoder_namedplaces_test.out (88%) create mode 100644 geocoder/extension/sql/0.0.1/15_country_decoder_table.sql rename geocoder/{namedplace/extension/cdb_geocoder_namedplaces--0.0.1.sql => extension/sql/0.0.1/40_namedplaces.sql} (89%) rename geocoder/{namedplace => }/extension/sql/cdb_geocoder_namedplaces_test.sql (82%) delete mode 100644 geocoder/namedplace/extension/.gitignore delete mode 100644 geocoder/namedplace/extension/Makefile delete mode 100644 geocoder/namedplace/extension/README.md delete mode 100644 geocoder/namedplace/extension/cdb_geocoder_namedplaces.control diff --git a/geocoder/extension/Makefile b/geocoder/extension/Makefile index ac1778e..d390592 100644 --- a/geocoder/extension/Makefile +++ b/geocoder/extension/Makefile @@ -8,7 +8,8 @@ REGRESS = \ 00_install_test \ cdb_geocoder_admin0_test \ cdb_geocoder_admin1_test \ - cdb_geocoder_ipaddr_test + cdb_geocoder_ipaddr_test \ + cdb_geocoder_namedplaces_test # postgres build stuff PG_CONFIG = pg_config diff --git a/geocoder/namedplace/extension/expected/cdb_geocoder_namedplaces_test.out b/geocoder/extension/expected/cdb_geocoder_namedplaces_test.out similarity index 88% rename from geocoder/namedplace/extension/expected/cdb_geocoder_namedplaces_test.out rename to geocoder/extension/expected/cdb_geocoder_namedplaces_test.out index 53d0b30..1e654fe 100644 --- a/geocoder/namedplace/extension/expected/cdb_geocoder_namedplaces_test.out +++ b/geocoder/extension/expected/cdb_geocoder_namedplaces_test.out @@ -1,8 +1,3 @@ -CREATE EXTENSION postgis; -CREATE EXTENSION schema_triggers; -CREATE EXTENSION plpythonu; -CREATE EXTENSION cartodb; -CREATE EXTENSION cdb_geocoder_namedplaces; -- Check that the different geocoding functions are callable, should return success = false SELECT (geocode_namedplace(Array['Madrid', 'New York City', 'sunapee'])).*; q | geom | success @@ -51,13 +46,6 @@ SELECT (geocode_namedplace(Array['Portland', 'Portland', 'New York City'], Array Portland | Maine | USA | | f (3 rows) --- Mock the varnish invalidation function -CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$ -BEGIN - RETURN; -END -$$ -LANGUAGE plpgsql; -- Add a named place source COPY global_cities_alternates_limited (geoname_id, name, the_geom, created_at, updated_at, the_geom_webmercator, preferred, lowername, cartodb_id, admin1_geonameid, iso2, admin1) FROM stdin; COPY global_cities_points_limited (geoname_id, name, asciiname, altnames, featclass, featcode, iso2, admin1, admin2, population, the_geom, created_at, updated_at, the_geom_webmercator, cartodb_id, lowername) FROM stdin; diff --git a/geocoder/extension/sql/0.0.1/15_country_decoder_table.sql b/geocoder/extension/sql/0.0.1/15_country_decoder_table.sql new file mode 100644 index 0000000..510fd3a --- /dev/null +++ b/geocoder/extension/sql/0.0.1/15_country_decoder_table.sql @@ -0,0 +1,54 @@ +CREATE TABLE country_decoder ( + name text, + nativename text, + tld text, + iso2 text, + ccn3 text, + iso3 text, + currency text, + callingcode text, + capital text, + altspellings text, + relevance text, + region text, + subregion text, + language text, + languagescodes text, + translations text, + population text, + latlng text, + demonym text, + borders text, + the_geom geometry(Geometry,4326), + cartodb_id integer NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL, + updated_at timestamp with time zone DEFAULT now() NOT NULL, + the_geom_webmercator geometry(Geometry,3857), + synbu text[], + synonyms text[], + users double precision +); + + +CREATE SEQUENCE country_decoder_cartodb_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; +ALTER SEQUENCE country_decoder_cartodb_id_seq OWNED BY country_decoder.cartodb_id; +ALTER TABLE ONLY country_decoder ALTER COLUMN cartodb_id SET DEFAULT nextval('country_decoder_cartodb_id_seq'::regclass); + +ALTER TABLE ONLY country_decoder + ADD CONSTRAINT country_decoder_cartodb_id_key UNIQUE (cartodb_id); +ALTER TABLE ONLY country_decoder + ADD CONSTRAINT country_decoder_pkey PRIMARY KEY (cartodb_id); +ALTER TABLE country_decoder CLUSTER ON country_decoder_pkey; + + +CREATE INDEX country_decoder_the_geom_idx ON country_decoder USING gist (the_geom); +CREATE INDEX country_decoder_the_geom_webmercator_idx ON country_decoder USING gist (the_geom_webmercator); + +CREATE TRIGGER track_updates AFTER INSERT OR DELETE OR UPDATE OR TRUNCATE ON country_decoder FOR EACH STATEMENT EXECUTE PROCEDURE cartodb.cdb_tablemetadata_trigger(); +CREATE TRIGGER update_the_geom_webmercator_trigger BEFORE INSERT OR UPDATE OF the_geom ON country_decoder FOR EACH ROW EXECUTE PROCEDURE cartodb._cdb_update_the_geom_webmercator(); +CREATE TRIGGER update_updated_at_trigger BEFORE UPDATE ON country_decoder FOR EACH ROW EXECUTE PROCEDURE cartodb._cdb_update_updated_at(); diff --git a/geocoder/extension/sql/0.0.1/20_admin1.sql b/geocoder/extension/sql/0.0.1/20_admin1.sql index f2a85f7..1d04c63 100644 --- a/geocoder/extension/sql/0.0.1/20_admin1.sql +++ b/geocoder/extension/sql/0.0.1/20_admin1.sql @@ -113,64 +113,6 @@ $$; -- Support tables -CREATE TABLE country_decoder ( - name text, - nativename text, - tld text, - iso2 text, - ccn3 text, - iso3 text, - currency text, - callingcode text, - capital text, - altspellings text, - relevance text, - region text, - subregion text, - language text, - languagescodes text, - translations text, - population text, - latlng text, - demonym text, - borders text, - the_geom geometry(Geometry,4326), - cartodb_id integer NOT NULL, - created_at timestamp with time zone DEFAULT now() NOT NULL, - updated_at timestamp with time zone DEFAULT now() NOT NULL, - the_geom_webmercator geometry(Geometry,3857), - synbu text[], - synonyms text[], - users double precision -); - -CREATE SEQUENCE country_decoder_cartodb_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; -ALTER SEQUENCE country_decoder_cartodb_id_seq OWNED BY country_decoder.cartodb_id; -ALTER TABLE ONLY country_decoder ALTER COLUMN cartodb_id SET DEFAULT nextval('country_decoder_cartodb_id_seq'::regclass); - - -ALTER TABLE ONLY country_decoder - ADD CONSTRAINT country_decoder_cartodb_id_key UNIQUE (cartodb_id); -ALTER TABLE ONLY country_decoder - ADD CONSTRAINT country_decoder_pkey PRIMARY KEY (cartodb_id); - -ALTER TABLE country_decoder CLUSTER ON country_decoder_pkey; - - -CREATE INDEX country_decoder_the_geom_idx ON country_decoder USING gist (the_geom); -CREATE INDEX country_decoder_the_geom_webmercator_idx ON country_decoder USING gist (the_geom_webmercator); - - -CREATE TRIGGER track_updates AFTER INSERT OR DELETE OR UPDATE OR TRUNCATE ON country_decoder FOR EACH STATEMENT EXECUTE PROCEDURE cartodb.cdb_tablemetadata_trigger(); -CREATE TRIGGER update_the_geom_webmercator_trigger BEFORE INSERT OR UPDATE OF the_geom ON country_decoder FOR EACH ROW EXECUTE PROCEDURE cartodb._cdb_update_the_geom_webmercator(); -CREATE TRIGGER update_updated_at_trigger BEFORE UPDATE ON country_decoder FOR EACH ROW EXECUTE PROCEDURE cartodb._cdb_update_updated_at(); - - CREATE TABLE global_province_polygons ( the_geom geometry(Geometry,4326), adm1_code text, diff --git a/geocoder/namedplace/extension/cdb_geocoder_namedplaces--0.0.1.sql b/geocoder/extension/sql/0.0.1/40_namedplaces.sql similarity index 89% rename from geocoder/namedplace/extension/cdb_geocoder_namedplaces--0.0.1.sql rename to geocoder/extension/sql/0.0.1/40_namedplaces.sql index 6dc537e..26e0f64 100644 --- a/geocoder/namedplace/extension/cdb_geocoder_namedplaces--0.0.1.sql +++ b/geocoder/extension/sql/0.0.1/40_namedplaces.sql @@ -1,11 +1,7 @@ --- Complain if script is sourced in psql, rather than via CREATE EXTENSION -\echo Use "CREATE EXTENSION cdb_geocoder_namedplaces" to load this file. \quit - -- Response types for namedplaces geocoder CREATE TYPE geocode_namedplace_country_v1 AS (q text, c text, geom geometry, success boolean); CREATE TYPE geocode_namedplace_v1 AS (q text, geom geometry, success boolean); CREATE TYPE geocode_admin1_country_v1 AS (q text, a1 text, c text, geom geometry, success boolean); -CREATE TYPE geocode_admin_country_v1 AS (q text, c text, geom geometry, success boolean); -- Public API functions -- --- Geocoding function --- @@ -346,62 +342,6 @@ CREATE TRIGGER update_the_geom_webmercator_trigger BEFORE INSERT OR UPDATE OF th CREATE TRIGGER update_updated_at_trigger BEFORE UPDATE ON admin1_decoder FOR EACH ROW EXECUTE PROCEDURE cartodb._cdb_update_updated_at(); -CREATE TABLE country_decoder ( - name text, - nativename text, - tld text, - iso2 text, - ccn3 text, - iso3 text, - currency text, - callingcode text, - capital text, - altspellings text, - relevance text, - region text, - subregion text, - language text, - languagescodes text, - translations text, - population text, - latlng text, - demonym text, - borders text, - the_geom geometry(Geometry,4326), - cartodb_id integer NOT NULL, - created_at timestamp with time zone DEFAULT now() NOT NULL, - updated_at timestamp with time zone DEFAULT now() NOT NULL, - the_geom_webmercator geometry(Geometry,3857), - synbu text[], - synonyms text[], - users double precision -); - - -CREATE SEQUENCE country_decoder_cartodb_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; -ALTER SEQUENCE country_decoder_cartodb_id_seq OWNED BY country_decoder.cartodb_id; -ALTER TABLE ONLY country_decoder ALTER COLUMN cartodb_id SET DEFAULT nextval('country_decoder_cartodb_id_seq'::regclass); - -ALTER TABLE ONLY country_decoder - ADD CONSTRAINT country_decoder_cartodb_id_key UNIQUE (cartodb_id); -ALTER TABLE ONLY country_decoder - ADD CONSTRAINT country_decoder_pkey PRIMARY KEY (cartodb_id); -ALTER TABLE country_decoder CLUSTER ON country_decoder_pkey; - - -CREATE INDEX country_decoder_the_geom_idx ON country_decoder USING gist (the_geom); -CREATE INDEX country_decoder_the_geom_webmercator_idx ON country_decoder USING gist (the_geom_webmercator); - -CREATE TRIGGER track_updates AFTER INSERT OR DELETE OR UPDATE OR TRUNCATE ON country_decoder FOR EACH STATEMENT EXECUTE PROCEDURE cartodb.cdb_tablemetadata_trigger(); -CREATE TRIGGER update_the_geom_webmercator_trigger BEFORE INSERT OR UPDATE OF the_geom ON country_decoder FOR EACH ROW EXECUTE PROCEDURE cartodb._cdb_update_the_geom_webmercator(); -CREATE TRIGGER update_updated_at_trigger BEFORE UPDATE ON country_decoder FOR EACH ROW EXECUTE PROCEDURE cartodb._cdb_update_updated_at(); - - CREATE TABLE global_cities_points_limited ( geoname_id integer, name text, diff --git a/geocoder/namedplace/extension/sql/cdb_geocoder_namedplaces_test.sql b/geocoder/extension/sql/cdb_geocoder_namedplaces_test.sql similarity index 82% rename from geocoder/namedplace/extension/sql/cdb_geocoder_namedplaces_test.sql rename to geocoder/extension/sql/cdb_geocoder_namedplaces_test.sql index cbc2c91..91f06e2 100644 --- a/geocoder/namedplace/extension/sql/cdb_geocoder_namedplaces_test.sql +++ b/geocoder/extension/sql/cdb_geocoder_namedplaces_test.sql @@ -1,9 +1,3 @@ -CREATE EXTENSION postgis; -CREATE EXTENSION schema_triggers; -CREATE EXTENSION plpythonu; -CREATE EXTENSION cartodb; -CREATE EXTENSION cdb_geocoder_namedplaces; - -- Check that the different geocoding functions are callable, should return success = false SELECT (geocode_namedplace(Array['Madrid', 'New York City', 'sunapee'])).*; SELECT (geocode_namedplace(Array['Elche', 'Granada', 'Madrid'], 'Spain')).*; @@ -12,15 +6,6 @@ SELECT (geocode_namedplace(Array['Portland', 'Portland', 'New York City'], Array SELECT (geocode_namedplace(Array['Portland'], 'Oregon', 'USA')).*; SELECT (geocode_namedplace(Array['Portland', 'Portland', 'New York City'], Array['Maine', 'Oregon', NULL], Array['USA'])).*; - --- Mock the varnish invalidation function -CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$ -BEGIN - RETURN; -END -$$ -LANGUAGE plpgsql; - -- Add a named place source COPY global_cities_alternates_limited (geoname_id, name, the_geom, created_at, updated_at, the_geom_webmercator, preferred, lowername, cartodb_id, admin1_geonameid, iso2, admin1) FROM stdin; 3128760 barcelona \N 2014-02-11 18:23:18.115612+00 2014-02-25 16:41:15.278786+00 \N t barcelona 7530944 409419 \N 56 diff --git a/geocoder/namedplace/extension/.gitignore b/geocoder/namedplace/extension/.gitignore deleted file mode 100644 index e710f0e..0000000 --- a/geocoder/namedplace/extension/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -results/ -regression.diffs -regression.out diff --git a/geocoder/namedplace/extension/Makefile b/geocoder/namedplace/extension/Makefile deleted file mode 100644 index ab1f0dc..0000000 --- a/geocoder/namedplace/extension/Makefile +++ /dev/null @@ -1,8 +0,0 @@ -EXTENSION = cdb_geocoder_namedplaces -DATA = cdb_geocoder_namedplaces--0.0.1.sql -REGRESS = cdb_geocoder_namedplaces_test - -# postgres build stuff -PG_CONFIG = pg_config -PGXS := $(shell $(PG_CONFIG) --pgxs) -include $(PGXS) diff --git a/geocoder/namedplace/extension/README.md b/geocoder/namedplace/extension/README.md deleted file mode 100644 index 478c892..0000000 --- a/geocoder/namedplace/extension/README.md +++ /dev/null @@ -1,36 +0,0 @@ -# CartoDB named places geocoder extension -Postgres extension for the CartoDB named places geocoder. It is meant to contain the functions and related objects needed to geocode by city names. It is not meant to contain the actual data used to geocode them. - -## Dependencies -This extension is thought to be used on top of CartoDB platform. Therefore a cartodb user is required to install the extension onto it. - -The following is a non-comprehensive list of dependencies: - -- Postgres 9.3+ -- Postgis extension -- Schema triggers extension -- CartoDB extension - -## Installation into the db cluster -This requires root privileges -``` -sudo make all install -``` - -## Execute tests -``` -PGUSER=postgres make installcheck -``` - -## Install onto a user's database -``` -psql -U cartodb_dev_user_367c0edc-b2ad-4bab-ad43-3d58a6179a93_db cartodb_dev_user_367c0edc-b2ad-4bab-ad43-3d58a6179a93_db -``` - -and then: - -```sql -CREATE EXTENSION cdb_geocoder_namedplaces; -``` - -The extension creation in the user's db does not require special privileges. It can be even created from the sql api. diff --git a/geocoder/namedplace/extension/cdb_geocoder_namedplaces.control b/geocoder/namedplace/extension/cdb_geocoder_namedplaces.control deleted file mode 100644 index 7da3164..0000000 --- a/geocoder/namedplace/extension/cdb_geocoder_namedplaces.control +++ /dev/null @@ -1,6 +0,0 @@ -# cdb geocoder namedplaces extension -comment = 'CartoDB named places internal geocoder' -default_version = '0.0.1' -relocatable = true -requires = cartodb -superuser = false From 13a94cd3cc397b787ff31842c34e95a9ea2ce56f Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 3 Nov 2015 18:37:58 +0100 Subject: [PATCH 21/32] Merge postalcodes into cdb_geocoder --- geocoder/extension/Makefile | 3 +- .../expected/cdb_geocoder_postalcode_test.out | 1 + geocoder/extension/sql/0.0.1/05_types.sql | 21 +++++++++++ geocoder/extension/sql/0.0.1/10_admin0.sql | 4 --- geocoder/extension/sql/0.0.1/20_admin1.sql | 6 ---- geocoder/extension/sql/0.0.1/30_ipaddr.sql | 3 -- .../extension/sql/0.0.1/40_namedplaces.sql | 5 --- .../sql/0.0.1/50_postalcodes.sql} | 13 ++----- .../sql/cdb_geocoder_postalcode_test.sql | 1 + geocoder/postal-codes/extension/.gitignore | 3 -- geocoder/postal-codes/extension/Makefile | 8 ----- geocoder/postal-codes/extension/README.md | 36 ------------------- .../extension/cdb_geocoder_postalcode.control | 6 ---- .../expected/cdb_geocoder_postalcode_test.out | 6 ---- .../sql/cdb_geocoder_postalcode_test.sql | 7 ---- .../setup/postal-codes/response-types.sql | 5 --- 16 files changed, 27 insertions(+), 101 deletions(-) create mode 100644 geocoder/extension/expected/cdb_geocoder_postalcode_test.out create mode 100644 geocoder/extension/sql/0.0.1/05_types.sql rename geocoder/{postal-codes/extension/cdb_geocoder_postalcode--0.0.1.sql => extension/sql/0.0.1/50_postalcodes.sql} (94%) create mode 100644 geocoder/extension/sql/cdb_geocoder_postalcode_test.sql delete mode 100644 geocoder/postal-codes/extension/.gitignore delete mode 100644 geocoder/postal-codes/extension/Makefile delete mode 100644 geocoder/postal-codes/extension/README.md delete mode 100644 geocoder/postal-codes/extension/cdb_geocoder_postalcode.control delete mode 100644 geocoder/postal-codes/extension/expected/cdb_geocoder_postalcode_test.out delete mode 100644 geocoder/postal-codes/extension/sql/cdb_geocoder_postalcode_test.sql delete mode 100644 geocoder/setup/postal-codes/response-types.sql diff --git a/geocoder/extension/Makefile b/geocoder/extension/Makefile index d390592..f5ec3b8 100644 --- a/geocoder/extension/Makefile +++ b/geocoder/extension/Makefile @@ -9,7 +9,8 @@ REGRESS = \ cdb_geocoder_admin0_test \ cdb_geocoder_admin1_test \ cdb_geocoder_ipaddr_test \ - cdb_geocoder_namedplaces_test + cdb_geocoder_namedplaces_test \ + cdb_geocoder_postalcode_test # postgres build stuff PG_CONFIG = pg_config diff --git a/geocoder/extension/expected/cdb_geocoder_postalcode_test.out b/geocoder/extension/expected/cdb_geocoder_postalcode_test.out new file mode 100644 index 0000000..4c317ad --- /dev/null +++ b/geocoder/extension/expected/cdb_geocoder_postalcode_test.out @@ -0,0 +1 @@ +-- PENDING diff --git a/geocoder/extension/sql/0.0.1/05_types.sql b/geocoder/extension/sql/0.0.1/05_types.sql new file mode 100644 index 0000000..360a926 --- /dev/null +++ b/geocoder/extension/sql/0.0.1/05_types.sql @@ -0,0 +1,21 @@ +-- Response types for admin0 geocoder +CREATE TYPE geocode_admin_v1 AS (q TEXT, geom GEOMETRY, success BOOLEAN); +CREATE TYPE synonym_lookup_v1 AS (q TEXT, adm0_a3 TEXT); + +-- Response types for admin1 geocoder +CREATE TYPE geocode_admin_country_v1 AS (q TEXT, c TEXT, geom GEOMETRY, success BOOLEAN); + +-- Response types for IP addresses geocoder +CREATE TYPE geocode_ip_v1 AS (q text, geom geometry, success boolean); + +-- Response types for namedplaces geocoder +CREATE TYPE geocode_namedplace_country_v1 AS (q TEXT, c TEXT, geom GEOMETRY, success BOOLEAN); +CREATE TYPE geocode_namedplace_v1 AS (q TEXT, geom GEOMETRY, success BOOLEAN); +CREATE TYPE geocode_admin1_country_v1 AS (q text, a1 text, c text, geom geometry, success boolean); + +-- Response types for postalcodes geocoder +CREATE TYPE geocode_postalint_country_v1 AS (q INT, c TEXT, geom GEOMETRY, success BOOLEAN); +CREATE TYPE geocode_place_country_iso_v1 AS (q TEXT, c TEXT, iso3 TEXT, geom GEOMETRY, success BOOLEAN); + + +CREATE TYPE available_services_v1 AS (q text, adm0_a3 text, postal_code_points boolean, postal_code_polygons boolean); diff --git a/geocoder/extension/sql/0.0.1/10_admin0.sql b/geocoder/extension/sql/0.0.1/10_admin0.sql index 100ff49..216d37e 100644 --- a/geocoder/extension/sql/0.0.1/10_admin0.sql +++ b/geocoder/extension/sql/0.0.1/10_admin0.sql @@ -1,7 +1,3 @@ --- Response types for admin0 geocoder -CREATE TYPE geocode_admin_v1 AS (q TEXT, geom GEOMETRY, success BOOLEAN); -CREATE TYPE synonym_lookup_v1 AS (q TEXT, adm0_a3 TEXT); - -- Public API functions -- --- Geocoding function --- -- TODO: deal with permissions diff --git a/geocoder/extension/sql/0.0.1/20_admin1.sql b/geocoder/extension/sql/0.0.1/20_admin1.sql index 1d04c63..283798e 100644 --- a/geocoder/extension/sql/0.0.1/20_admin1.sql +++ b/geocoder/extension/sql/0.0.1/20_admin1.sql @@ -1,9 +1,3 @@ --- Response types for admin1 geocoder --- TODO: check if the types exist already in the db - -CREATE TYPE geocode_admin_country_v1 AS (q TEXT, c TEXT, geom GEOMETRY, success BOOLEAN); - - -- Public API functions -- --- Geocoding function --- -- TODO: deal with permissions diff --git a/geocoder/extension/sql/0.0.1/30_ipaddr.sql b/geocoder/extension/sql/0.0.1/30_ipaddr.sql index 8560aeb..1018179 100644 --- a/geocoder/extension/sql/0.0.1/30_ipaddr.sql +++ b/geocoder/extension/sql/0.0.1/30_ipaddr.sql @@ -1,6 +1,3 @@ --- Response types for IP addresses geocoder -CREATE TYPE geocode_ip_v1 AS (q text, geom geometry, success boolean); - -- Public API functions -- --- Geocoding function --- -- TODO: deal with permissions diff --git a/geocoder/extension/sql/0.0.1/40_namedplaces.sql b/geocoder/extension/sql/0.0.1/40_namedplaces.sql index 26e0f64..39eed32 100644 --- a/geocoder/extension/sql/0.0.1/40_namedplaces.sql +++ b/geocoder/extension/sql/0.0.1/40_namedplaces.sql @@ -1,8 +1,3 @@ --- Response types for namedplaces geocoder -CREATE TYPE geocode_namedplace_country_v1 AS (q text, c text, geom geometry, success boolean); -CREATE TYPE geocode_namedplace_v1 AS (q text, geom geometry, success boolean); -CREATE TYPE geocode_admin1_country_v1 AS (q text, a1 text, c text, geom geometry, success boolean); - -- Public API functions -- --- Geocoding function --- -- TODO: deal with permissions diff --git a/geocoder/postal-codes/extension/cdb_geocoder_postalcode--0.0.1.sql b/geocoder/extension/sql/0.0.1/50_postalcodes.sql similarity index 94% rename from geocoder/postal-codes/extension/cdb_geocoder_postalcode--0.0.1.sql rename to geocoder/extension/sql/0.0.1/50_postalcodes.sql index be91673..c6b78ed 100644 --- a/geocoder/postal-codes/extension/cdb_geocoder_postalcode--0.0.1.sql +++ b/geocoder/extension/sql/0.0.1/50_postalcodes.sql @@ -1,12 +1,3 @@ --- Complain if script is sourced in psql, rather than via CREATE EXTENSION -\echo Use "CREATE EXTENSION cdb_geocoder_postalcode" to load this file. \quit - --- Response types for admin0 geocoder -CREATE TYPE geocode_namedplace_v1 AS (q TEXT, geom GEOMETRY, success BOOLEAN); -CREATE TYPE geocode_postalint_country_v1 AS (q TEXT, c TEXT, geom GEOMETRY, success BOOLEAN); -CREATE TYPE geocode_namedplace_country_v1 AS (q TEXT, c TEXT, geom GEOMETRY, success BOOLEAN); -CREATE TYPE available_services_v1 AS (q text, adm0_a3 text, postal_code_points boolean, postal_code_polygons boolean); - -- Public API functions -- --- Geocoding function --- -- TODO: deal with permissions @@ -391,8 +382,8 @@ CREATE SEQUENCE available_services_cartodb_id_seq NO MAXVALUE CACHE 1; -ALTER SEQUENCE available_services_cartodb_id_seq_cartodb_id_seq OWNED BY available_services.cartodb_id; -ALTER TABLE ONLY available_services ALTER COLUMN cartodb_id SET DEFAULT nextval('available_services_cartodb_id_seq_cartodb_id_seq'::regclass); +ALTER SEQUENCE available_services_cartodb_id_seq OWNED BY available_services.cartodb_id; +ALTER TABLE ONLY available_services ALTER COLUMN cartodb_id SET DEFAULT nextval('available_services_cartodb_id_seq'::regclass); ALTER TABLE ONLY available_services diff --git a/geocoder/extension/sql/cdb_geocoder_postalcode_test.sql b/geocoder/extension/sql/cdb_geocoder_postalcode_test.sql new file mode 100644 index 0000000..4c317ad --- /dev/null +++ b/geocoder/extension/sql/cdb_geocoder_postalcode_test.sql @@ -0,0 +1 @@ +-- PENDING diff --git a/geocoder/postal-codes/extension/.gitignore b/geocoder/postal-codes/extension/.gitignore deleted file mode 100644 index e710f0e..0000000 --- a/geocoder/postal-codes/extension/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -results/ -regression.diffs -regression.out diff --git a/geocoder/postal-codes/extension/Makefile b/geocoder/postal-codes/extension/Makefile deleted file mode 100644 index dcff338..0000000 --- a/geocoder/postal-codes/extension/Makefile +++ /dev/null @@ -1,8 +0,0 @@ -EXTENSION = cdb_geocoder_postalcode -DATA = cdb_geocoder_postalcode--0.0.1.sql -REGRESS = cdb_geocoder_postalcode_test - -# postgres build stuff -PG_CONFIG = pg_config -PGXS := $(shell $(PG_CONFIG) --pgxs) -include $(PGXS) diff --git a/geocoder/postal-codes/extension/README.md b/geocoder/postal-codes/extension/README.md deleted file mode 100644 index b2afd35..0000000 --- a/geocoder/postal-codes/extension/README.md +++ /dev/null @@ -1,36 +0,0 @@ -# CartoDB postal code geocoder extension -Postgres extension for the CartoDB postal code geocoder. It is meant to contain the functions and related objects needed to geocode by postal codes. It is not meant to contain the actual data used to geocode them. - -## Dependencies -This extension is thought to be used on top of CartoDB platform. Therefore a cartodb user is required to install the extension onto it. - -The following is a non-comprehensive list of dependencies: - -- Postgres 9.3+ -- Postgis extension -- Schema triggers extension -- CartoDB extension - -## Installation into the db cluster -This requires root privileges -``` -sudo make all install -``` - -## Execute tests -``` -PGUSER=postgres make installcheck -``` - -## Install onto a user's database -``` -psql -U development_cartodb_user_fe3b850a-01c0-48f9-8a26-a82f09e9b53f cartodb_dev_user_fe3b850a-01c0-48f9-8a26-a82f09e9b53f_db -``` - -and then: - -```sql -CREATE EXTENSION cdb_geocoder_postalcode; -``` - -The extension creation in the user's db does not require special privileges. It can be even created from the sql api. diff --git a/geocoder/postal-codes/extension/cdb_geocoder_postalcode.control b/geocoder/postal-codes/extension/cdb_geocoder_postalcode.control deleted file mode 100644 index 70f249c..0000000 --- a/geocoder/postal-codes/extension/cdb_geocoder_postalcode.control +++ /dev/null @@ -1,6 +0,0 @@ -# cdb geocoder postalcode extension -comment = 'CartoDB postalcode internal geocoder' -default_version = '0.0.1' -relocatable = true -requires = cartodb -superuser = false diff --git a/geocoder/postal-codes/extension/expected/cdb_geocoder_postalcode_test.out b/geocoder/postal-codes/extension/expected/cdb_geocoder_postalcode_test.out deleted file mode 100644 index 6824cae..0000000 --- a/geocoder/postal-codes/extension/expected/cdb_geocoder_postalcode_test.out +++ /dev/null @@ -1,6 +0,0 @@ -CREATE EXTENSION postgis; -CREATE EXTENSION schema_triggers; -CREATE EXTENSION plpythonu; -CREATE EXTENSION cartodb; -CREATE EXTENSION cdb_geocoder_postalcode; --- PENDING diff --git a/geocoder/postal-codes/extension/sql/cdb_geocoder_postalcode_test.sql b/geocoder/postal-codes/extension/sql/cdb_geocoder_postalcode_test.sql deleted file mode 100644 index 314eb72..0000000 --- a/geocoder/postal-codes/extension/sql/cdb_geocoder_postalcode_test.sql +++ /dev/null @@ -1,7 +0,0 @@ -CREATE EXTENSION postgis; -CREATE EXTENSION schema_triggers; -CREATE EXTENSION plpythonu; -CREATE EXTENSION cartodb; -CREATE EXTENSION cdb_geocoder_postalcode; - --- PENDING diff --git a/geocoder/setup/postal-codes/response-types.sql b/geocoder/setup/postal-codes/response-types.sql deleted file mode 100644 index 4a5bea5..0000000 --- a/geocoder/setup/postal-codes/response-types.sql +++ /dev/null @@ -1,5 +0,0 @@ --- Response types for postal codes geocoder -CREATE TYPE geocode_namedplace_v1 AS (q TEXT, geom GEOMETRY, success BOOLEAN); -CREATE TYPE geocode_place_country_iso_v1 AS (q TEXT, c TEXT, iso3 TEXT, geom GEOMETRY, success BOOLEAN); -CREATE TYPE geocode_namedplace_country_v1 AS (q TEXT, c TEXT, geom GEOMETRY, success BOOLEAN); -CREATE TYPE geocode_postalint_country_v1 AS (q INT, c TEXT, geom GEOMETRY, success BOOLEAN); From 44f007f558ee8178c4109ae0bbe47908bc9151f0 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 3 Nov 2015 18:41:17 +0100 Subject: [PATCH 22/32] Add tests for postalcodes (by iriberri) --- .../expected/cdb_geocoder_postalcode_test.out | 97 ++++++++++++++++++- .../sql/cdb_geocoder_postalcode_test.sql | 55 ++++++++++- 2 files changed, 150 insertions(+), 2 deletions(-) diff --git a/geocoder/extension/expected/cdb_geocoder_postalcode_test.out b/geocoder/extension/expected/cdb_geocoder_postalcode_test.out index 4c317ad..83256e6 100644 --- a/geocoder/extension/expected/cdb_geocoder_postalcode_test.out +++ b/geocoder/extension/expected/cdb_geocoder_postalcode_test.out @@ -1 +1,96 @@ --- PENDING +SELECT (geocode_postalcode_polygons(Array['03204'], Array['Spain'])).*; + q | c | geom | success +-------+-------+------+--------- + 03204 | Spain | | f +(1 row) + +SELECT (geocode_postalcode_polygons(Array['03204'], 'ESP')).*; + q | geom | success +-------+------+--------- + 03204 | | f +(1 row) + +SELECT (geocode_postalcode_polygons(Array['03204'])).*; + q | geom | success +-------+------+--------- + 03204 | | f +(1 row) + +SELECT (geocode_postalcode_points(Array['03204'], 'Spain')).*; + q | geom | success +-------+------+--------- + 03204 | | f +(1 row) + +SELECT (geocode_postalcode_points('{03204}', 'Spain')).*; + q | geom | success +-------+------+--------- + 03204 | | f +(1 row) + +SELECT (geocode_postalcode_points(Array['03204'])).*; + q | geom | success +-------+------+--------- + 03204 | | f +(1 row) + +SELECT (geocode_postalcode_points(Array['03204'], Array['Spain'])).*; + q | c | iso3 | geom | success +-------+-------+------+------+--------- + 03204 | Spain | | | f +(1 row) + +SELECT geocode_greatbritain_outward('YO1 721'); + geocode_greatbritain_outward +------------------------------ + +(1 row) + +SELECT (admin0_available_services(Array['Spain'])).*; + q | adm0_a3 | postal_code_points | postal_code_polygons +-------+---------+--------------------+---------------------- + Spain | | | +(1 row) + +-- Insert mock source data +INSERT INTO global_postal_code_points (the_geom, iso3, postal_code, postal_code_num) VALUES ( + '0101000020E61000000000000000E040408036B47414764840', + 'ESP', + '03204', + 3204 +); +INSERT INTO global_postal_code_polygons (the_geom, iso3, postal_code, postal_code_num) VALUES ( + '0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040', + 'ESP', + '03204', + 3204 +); +INSERT INTO country_decoder (iso3, synonyms) VALUES ( + 'ESP', + Array['spain', 'Spain', 'ESP'] +); +INSERT INTO available_services (adm0_a3, admin0, postal_code_points, postal_code_polygons) VALUES ( + 'ESP', + 't', + 't', + 't' +); +INSERT INTO admin0_synonyms (adm0_a3, name, name_, rank) VALUES ( + 'ESP', + 'Spain', + 'spain', + 3 +); +INSERT INTO admin0_synonyms (adm0_a3, name, name_, rank) VALUES ( + 'ESP', + 'ESP', + 'esp', + 4 +); +-- Check that the geocoding function is callable, should return success = true +SELECT (geocode_postalcode_polygons(Array['03204'], Array['Spain'])).geom; + geom +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + 0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040 +(1 row) + diff --git a/geocoder/extension/sql/cdb_geocoder_postalcode_test.sql b/geocoder/extension/sql/cdb_geocoder_postalcode_test.sql index 4c317ad..6a332c0 100644 --- a/geocoder/extension/sql/cdb_geocoder_postalcode_test.sql +++ b/geocoder/extension/sql/cdb_geocoder_postalcode_test.sql @@ -1 +1,54 @@ --- PENDING +SELECT (geocode_postalcode_polygons(Array['03204'], Array['Spain'])).*; +SELECT (geocode_postalcode_polygons(Array['03204'], 'ESP')).*; +SELECT (geocode_postalcode_polygons(Array['03204'])).*; +SELECT (geocode_postalcode_points(Array['03204'], 'Spain')).*; +SELECT (geocode_postalcode_points('{03204}', 'Spain')).*; +SELECT (geocode_postalcode_points(Array['03204'])).*; +SELECT (geocode_postalcode_points(Array['03204'], Array['Spain'])).*; +SELECT geocode_greatbritain_outward('YO1 721'); +SELECT (admin0_available_services(Array['Spain'])).*; + +-- Insert mock source data +INSERT INTO global_postal_code_points (the_geom, iso3, postal_code, postal_code_num) VALUES ( + '0101000020E61000000000000000E040408036B47414764840', + 'ESP', + '03204', + 3204 +); + +INSERT INTO global_postal_code_polygons (the_geom, iso3, postal_code, postal_code_num) VALUES ( + '0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040', + 'ESP', + '03204', + 3204 +); + +INSERT INTO country_decoder (iso3, synonyms) VALUES ( + 'ESP', + Array['spain', 'Spain', 'ESP'] +); + +INSERT INTO available_services (adm0_a3, admin0, postal_code_points, postal_code_polygons) VALUES ( + 'ESP', + 't', + 't', + 't' +); + +INSERT INTO admin0_synonyms (adm0_a3, name, name_, rank) VALUES ( + 'ESP', + 'Spain', + 'spain', + 3 +); + +INSERT INTO admin0_synonyms (adm0_a3, name, name_, rank) VALUES ( + 'ESP', + 'ESP', + 'esp', + 4 +); + +-- Check that the geocoding function is callable, should return success = true +SELECT (geocode_postalcode_polygons(Array['03204'], Array['Spain'])).geom; + From bdbbdbbf93328b0bd6b720e22f34a48d08827249 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 3 Nov 2015 18:32:57 +0000 Subject: [PATCH 23/32] Modify sequence names to match production --- geocoder/extension/sql/0.0.1/15_country_decoder_table.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/geocoder/extension/sql/0.0.1/15_country_decoder_table.sql b/geocoder/extension/sql/0.0.1/15_country_decoder_table.sql index 510fd3a..32d49cf 100644 --- a/geocoder/extension/sql/0.0.1/15_country_decoder_table.sql +++ b/geocoder/extension/sql/0.0.1/15_country_decoder_table.sql @@ -30,14 +30,14 @@ CREATE TABLE country_decoder ( ); -CREATE SEQUENCE country_decoder_cartodb_id_seq +CREATE SEQUENCE countries_cartodb_id_seq START WITH 1 INCREMENT BY 1 NO MINVALUE NO MAXVALUE CACHE 1; -ALTER SEQUENCE country_decoder_cartodb_id_seq OWNED BY country_decoder.cartodb_id; -ALTER TABLE ONLY country_decoder ALTER COLUMN cartodb_id SET DEFAULT nextval('country_decoder_cartodb_id_seq'::regclass); +ALTER SEQUENCE countries_cartodb_id_seq OWNED BY country_decoder.cartodb_id; +ALTER TABLE ONLY country_decoder ALTER COLUMN cartodb_id SET DEFAULT nextval('countries_cartodb_id_seq'::regclass); ALTER TABLE ONLY country_decoder ADD CONSTRAINT country_decoder_cartodb_id_key UNIQUE (cartodb_id); From e3caf7db56b02a53689d8b63228f8be2e6d41279 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 3 Nov 2015 18:35:49 +0000 Subject: [PATCH 24/32] Modify sequence names to match production --- geocoder/extension/sql/0.0.1/20_admin1.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/geocoder/extension/sql/0.0.1/20_admin1.sql b/geocoder/extension/sql/0.0.1/20_admin1.sql index 283798e..8d4310a 100644 --- a/geocoder/extension/sql/0.0.1/20_admin1.sql +++ b/geocoder/extension/sql/0.0.1/20_admin1.sql @@ -176,14 +176,14 @@ CREATE TABLE global_province_polygons ( frequency double precision ); -CREATE SEQUENCE global_province_polygons_cartodb_id_seq +CREATE SEQUENCE ne_10m_admin_1_states_provinces_cartodb_id_seq START WITH 1 INCREMENT BY 1 NO MINVALUE NO MAXVALUE CACHE 1; -ALTER SEQUENCE global_province_polygons_cartodb_id_seq OWNED BY global_province_polygons.cartodb_id; -ALTER TABLE ONLY global_province_polygons ALTER COLUMN cartodb_id SET DEFAULT nextval('global_province_polygons_cartodb_id_seq'::regclass); +ALTER SEQUENCE ne_10m_admin_1_states_provinces_cartodb_id_seq OWNED BY global_province_polygons.cartodb_id; +ALTER TABLE ONLY global_province_polygons ALTER COLUMN cartodb_id SET DEFAULT nextval('ne_10m_admin_1_states_provinces_cartodb_id_seq'::regclass); ALTER TABLE ONLY global_province_polygons ADD CONSTRAINT global_province_polygons_cartodb_id_key UNIQUE (cartodb_id); ALTER TABLE ONLY global_province_polygons From 8ff3e81b5f55fb56cb0ebedc52816d20a54993b0 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 3 Nov 2015 19:39:06 +0100 Subject: [PATCH 25/32] Modify sequence names to match production --- geocoder/extension/sql/0.0.1/30_ipaddr.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/geocoder/extension/sql/0.0.1/30_ipaddr.sql b/geocoder/extension/sql/0.0.1/30_ipaddr.sql index 1018179..879be99 100644 --- a/geocoder/extension/sql/0.0.1/30_ipaddr.sql +++ b/geocoder/extension/sql/0.0.1/30_ipaddr.sql @@ -48,14 +48,14 @@ CREATE TABLE ip_address_locations ( the_geom_webmercator geometry(Geometry,3857) ); -CREATE SEQUENCE ip_address_locations_cartodb_id_seq +CREATE SEQUENCE geolite2_city_blocks_cartodb_id_seq START WITH 1 INCREMENT BY 1 NO MINVALUE NO MAXVALUE CACHE 1; -ALTER SEQUENCE ip_address_locations_cartodb_id_seq OWNED BY ip_address_locations.cartodb_id; -ALTER TABLE ONLY ip_address_locations ALTER COLUMN cartodb_id SET DEFAULT nextval('ip_address_locations_cartodb_id_seq'::regclass); +ALTER SEQUENCE geolite2_city_blocks_cartodb_id_seq OWNED BY ip_address_locations.cartodb_id; +ALTER TABLE ONLY ip_address_locations ALTER COLUMN cartodb_id SET DEFAULT nextval('geolite2_city_blocks_cartodb_id_seq'::regclass); ALTER TABLE ONLY ip_address_locations From 0bc00a7f76e283b22dcbed0a6bf3246f33a2259c Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 4 Nov 2015 15:31:08 +0100 Subject: [PATCH 26/32] Update README.md --- geocoder/extension/README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/geocoder/extension/README.md b/geocoder/extension/README.md index bee0de1..a55400c 100644 --- a/geocoder/extension/README.md +++ b/geocoder/extension/README.md @@ -2,7 +2,7 @@ Postgres extension for the CartoDB geocoder. It is meant to contain the functions and related objects needed to provide a geocoding service for administrative areas of level 0, administrative areas of level 1, postal codes, IP addresses and city names. It is not meant to contain the actual data used to geocode them. ## Dependencies -This extension is thought to be used on top of CartoDB platform. Therefore a cartodb user is required to install the extension onto it. +This extension is thought to be used on top of CartoDB platform. Therefore **a cartodb user is required** to install the extension onto it. The following is a non-comprehensive list of dependencies: @@ -28,7 +28,10 @@ One-liner: sudo PGUSER=postgres make all install installcheck ``` -## Install onto a user's database +## Install onto a cartodb user's database + +Remember that **is mandatory to install into a cartodb user's database** + ``` psql -U development_cartodb_user_fe3b850a-01c0-48f9-8a26-a82f09e9b53f cartodb_dev_user_fe3b850a-01c0-48f9-8a26-a82f09e9b53f_db ``` From 0f4754aa24ba9858c209c5094139a1f61c323fc2 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 4 Nov 2015 15:02:34 +0100 Subject: [PATCH 27/32] Modify sequence name to match dumps, table global_cities_points_limited --- geocoder/extension/sql/0.0.1/40_namedplaces.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/geocoder/extension/sql/0.0.1/40_namedplaces.sql b/geocoder/extension/sql/0.0.1/40_namedplaces.sql index 39eed32..381ff8d 100644 --- a/geocoder/extension/sql/0.0.1/40_namedplaces.sql +++ b/geocoder/extension/sql/0.0.1/40_namedplaces.sql @@ -361,14 +361,14 @@ CREATE TABLE global_cities_points_limited ( ); -CREATE SEQUENCE global_cities_points_limited_cartodb_id_seq +CREATE SEQUENCE points_cities_le_cartodb_id_seq START WITH 1 INCREMENT BY 1 NO MINVALUE NO MAXVALUE CACHE 1; -ALTER SEQUENCE global_cities_points_limited_cartodb_id_seq OWNED BY global_cities_points_limited.cartodb_id; -ALTER TABLE ONLY global_cities_points_limited ALTER COLUMN cartodb_id SET DEFAULT nextval('global_cities_points_limited_cartodb_id_seq'::regclass); +ALTER SEQUENCE points_cities_le_cartodb_id_seq OWNED BY global_cities_points_limited.cartodb_id; +ALTER TABLE ONLY global_cities_points_limited ALTER COLUMN cartodb_id SET DEFAULT nextval('points_cities_le_cartodb_id_seq'::regclass); ALTER TABLE ONLY global_cities_points_limited ADD CONSTRAINT global_cities_points_limited_cartodb_id_key UNIQUE (cartodb_id); From f06029cfd7daad46da54d2e2066364365991a073 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 4 Nov 2015 15:09:45 +0100 Subject: [PATCH 28/32] Modify sequence name to match dumps, table global_postal_code_points --- geocoder/extension/sql/0.0.1/50_postalcodes.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/geocoder/extension/sql/0.0.1/50_postalcodes.sql b/geocoder/extension/sql/0.0.1/50_postalcodes.sql index c6b78ed..ceca4a8 100644 --- a/geocoder/extension/sql/0.0.1/50_postalcodes.sql +++ b/geocoder/extension/sql/0.0.1/50_postalcodes.sql @@ -334,14 +334,14 @@ CREATE TABLE global_postal_code_points ( ); -CREATE SEQUENCE global_postal_code_points_cartodb_id_seq +CREATE SEQUENCE allcountries_cartodb_id_seq START WITH 1 INCREMENT BY 1 NO MINVALUE NO MAXVALUE CACHE 1; -ALTER SEQUENCE global_postal_code_points_cartodb_id_seq OWNED BY global_postal_code_points.cartodb_id; -ALTER TABLE ONLY global_postal_code_points ALTER COLUMN cartodb_id SET DEFAULT nextval('global_postal_code_points_cartodb_id_seq'::regclass); +ALTER SEQUENCE allcountries_cartodb_id_seq OWNED BY global_postal_code_points.cartodb_id; +ALTER TABLE ONLY global_postal_code_points ALTER COLUMN cartodb_id SET DEFAULT nextval('allcountries_cartodb_id_seq'::regclass); ALTER TABLE ONLY global_postal_code_points From 016e5af85ac41f84fb6df20b2c3daddff1a24843 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 4 Nov 2015 15:12:40 +0100 Subject: [PATCH 29/32] Modify sequence name to match dumps, table global_postal_code_polygons --- geocoder/extension/sql/0.0.1/50_postalcodes.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/geocoder/extension/sql/0.0.1/50_postalcodes.sql b/geocoder/extension/sql/0.0.1/50_postalcodes.sql index ceca4a8..9dd264f 100644 --- a/geocoder/extension/sql/0.0.1/50_postalcodes.sql +++ b/geocoder/extension/sql/0.0.1/50_postalcodes.sql @@ -283,14 +283,14 @@ CREATE TABLE global_postal_code_polygons ( ); -CREATE SEQUENCE global_postal_code_polygons_cartodb_id_seq +CREATE SEQUENCE tl_2013_us_zcta510_cartodb_id_seq START WITH 1 INCREMENT BY 1 NO MINVALUE NO MAXVALUE CACHE 1; -ALTER SEQUENCE global_postal_code_polygons_cartodb_id_seq OWNED BY global_postal_code_polygons.cartodb_id; -ALTER TABLE ONLY global_postal_code_polygons ALTER COLUMN cartodb_id SET DEFAULT nextval('global_postal_code_polygons_cartodb_id_seq'::regclass); +ALTER SEQUENCE tl_2013_us_zcta510_cartodb_id_seq OWNED BY global_postal_code_polygons.cartodb_id; +ALTER TABLE ONLY global_postal_code_polygons ALTER COLUMN cartodb_id SET DEFAULT nextval('tl_2013_us_zcta510_cartodb_id_seq'::regclass); ALTER TABLE ONLY global_postal_code_polygons From dae0fa38abada8888617544b1085cc1f73c8a90a Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 5 Nov 2015 15:56:27 +0000 Subject: [PATCH 30/32] Rename tests and improve Makefile --- geocoder/extension/Makefile | 9 ++------- .../{cdb_geocoder_admin0_test.out => 10_admin0_test.out} | 0 .../{cdb_geocoder_admin1_test.out => 20_admin1_test.out} | 0 .../{cdb_geocoder_ipaddr_test.out => 30_ipaddr_test.out} | 0 ...oder_namedplaces_test.out => 40_namedplaces_test.out} | 0 ...ocoder_postalcode_test.out => 50_postalcode_test.out} | 0 .../{cdb_geocoder_admin0_test.sql => 10_admin0_test.sql} | 0 .../{cdb_geocoder_admin1_test.sql => 20_admin1_test.sql} | 0 .../{cdb_geocoder_ipaddr_test.sql => 30_ipaddr_test.sql} | 0 ...oder_namedplaces_test.sql => 40_namedplaces_test.sql} | 0 ...ocoder_postalcode_test.sql => 50_postalcode_test.sql} | 0 11 files changed, 2 insertions(+), 7 deletions(-) rename geocoder/extension/expected/{cdb_geocoder_admin0_test.out => 10_admin0_test.out} (100%) rename geocoder/extension/expected/{cdb_geocoder_admin1_test.out => 20_admin1_test.out} (100%) rename geocoder/extension/expected/{cdb_geocoder_ipaddr_test.out => 30_ipaddr_test.out} (100%) rename geocoder/extension/expected/{cdb_geocoder_namedplaces_test.out => 40_namedplaces_test.out} (100%) rename geocoder/extension/expected/{cdb_geocoder_postalcode_test.out => 50_postalcode_test.out} (100%) rename geocoder/extension/sql/{cdb_geocoder_admin0_test.sql => 10_admin0_test.sql} (100%) rename geocoder/extension/sql/{cdb_geocoder_admin1_test.sql => 20_admin1_test.sql} (100%) rename geocoder/extension/sql/{cdb_geocoder_ipaddr_test.sql => 30_ipaddr_test.sql} (100%) rename geocoder/extension/sql/{cdb_geocoder_namedplaces_test.sql => 40_namedplaces_test.sql} (100%) rename geocoder/extension/sql/{cdb_geocoder_postalcode_test.sql => 50_postalcode_test.sql} (100%) diff --git a/geocoder/extension/Makefile b/geocoder/extension/Makefile index f5ec3b8..f93f488 100644 --- a/geocoder/extension/Makefile +++ b/geocoder/extension/Makefile @@ -4,13 +4,8 @@ EXTENSION = cdb_geocoder EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/") DATA = $(EXTENSION)--$(EXTVERSION).sql -REGRESS = \ - 00_install_test \ - cdb_geocoder_admin0_test \ - cdb_geocoder_admin1_test \ - cdb_geocoder_ipaddr_test \ - cdb_geocoder_namedplaces_test \ - cdb_geocoder_postalcode_test + +REGRESS = $(notdir $(basename $(wildcard sql/*test.sql))) # postgres build stuff PG_CONFIG = pg_config diff --git a/geocoder/extension/expected/cdb_geocoder_admin0_test.out b/geocoder/extension/expected/10_admin0_test.out similarity index 100% rename from geocoder/extension/expected/cdb_geocoder_admin0_test.out rename to geocoder/extension/expected/10_admin0_test.out diff --git a/geocoder/extension/expected/cdb_geocoder_admin1_test.out b/geocoder/extension/expected/20_admin1_test.out similarity index 100% rename from geocoder/extension/expected/cdb_geocoder_admin1_test.out rename to geocoder/extension/expected/20_admin1_test.out diff --git a/geocoder/extension/expected/cdb_geocoder_ipaddr_test.out b/geocoder/extension/expected/30_ipaddr_test.out similarity index 100% rename from geocoder/extension/expected/cdb_geocoder_ipaddr_test.out rename to geocoder/extension/expected/30_ipaddr_test.out diff --git a/geocoder/extension/expected/cdb_geocoder_namedplaces_test.out b/geocoder/extension/expected/40_namedplaces_test.out similarity index 100% rename from geocoder/extension/expected/cdb_geocoder_namedplaces_test.out rename to geocoder/extension/expected/40_namedplaces_test.out diff --git a/geocoder/extension/expected/cdb_geocoder_postalcode_test.out b/geocoder/extension/expected/50_postalcode_test.out similarity index 100% rename from geocoder/extension/expected/cdb_geocoder_postalcode_test.out rename to geocoder/extension/expected/50_postalcode_test.out diff --git a/geocoder/extension/sql/cdb_geocoder_admin0_test.sql b/geocoder/extension/sql/10_admin0_test.sql similarity index 100% rename from geocoder/extension/sql/cdb_geocoder_admin0_test.sql rename to geocoder/extension/sql/10_admin0_test.sql diff --git a/geocoder/extension/sql/cdb_geocoder_admin1_test.sql b/geocoder/extension/sql/20_admin1_test.sql similarity index 100% rename from geocoder/extension/sql/cdb_geocoder_admin1_test.sql rename to geocoder/extension/sql/20_admin1_test.sql diff --git a/geocoder/extension/sql/cdb_geocoder_ipaddr_test.sql b/geocoder/extension/sql/30_ipaddr_test.sql similarity index 100% rename from geocoder/extension/sql/cdb_geocoder_ipaddr_test.sql rename to geocoder/extension/sql/30_ipaddr_test.sql diff --git a/geocoder/extension/sql/cdb_geocoder_namedplaces_test.sql b/geocoder/extension/sql/40_namedplaces_test.sql similarity index 100% rename from geocoder/extension/sql/cdb_geocoder_namedplaces_test.sql rename to geocoder/extension/sql/40_namedplaces_test.sql diff --git a/geocoder/extension/sql/cdb_geocoder_postalcode_test.sql b/geocoder/extension/sql/50_postalcode_test.sql similarity index 100% rename from geocoder/extension/sql/cdb_geocoder_postalcode_test.sql rename to geocoder/extension/sql/50_postalcode_test.sql From faf5514d2eba884fe828edb74e7847b2900beed1 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 5 Nov 2015 18:37:21 +0100 Subject: [PATCH 31/32] Add a script to load dumps into db --- geocoder/geocoder_restore_dump | 43 ++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100755 geocoder/geocoder_restore_dump diff --git a/geocoder/geocoder_restore_dump b/geocoder/geocoder_restore_dump new file mode 100755 index 0000000..eb55ce7 --- /dev/null +++ b/geocoder/geocoder_restore_dump @@ -0,0 +1,43 @@ +#!/bin/bash + +function usage() { + cat < Date: Thu, 5 Nov 2015 19:44:14 +0100 Subject: [PATCH 32/32] Script to download dumps from S3 --- geocoder/.gitignore | 1 + geocoder/geocoder_download_dumps | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 geocoder/.gitignore create mode 100755 geocoder/geocoder_download_dumps diff --git a/geocoder/.gitignore b/geocoder/.gitignore new file mode 100644 index 0000000..a4ee109 --- /dev/null +++ b/geocoder/.gitignore @@ -0,0 +1 @@ +db_dumps/ diff --git a/geocoder/geocoder_download_dumps b/geocoder/geocoder_download_dumps new file mode 100755 index 0000000..1e8cb72 --- /dev/null +++ b/geocoder/geocoder_download_dumps @@ -0,0 +1,25 @@ +#!/bin/bash + +TARGET_DIR=db_dumps +BASE_URL=https://s3.amazonaws.com/data.cartodb.net/geocoding/dumps +VERSION=0.0.1 + +DUMP_LIST="admin0_synonyms.sql +available_services.sql +country_decoder.sql +admin1_decoder.sql +global_cities_alternates_limited.sql +global_cities_points_limited.sql +global_postal_code_points.sql +global_province_polygons.sql +ip_address_locations.sql +ne_admin0_v3.sql +global_postal_code_polygons.sql" + +mkdir -p $TARGET_DIR + +for file in $DUMP_LIST; do + url="${BASE_URL}/${VERSION}/$file" + + wget --directory-prefix=$TARGET_DIR $url +done