From de0ced9434416a4b1ac9e734d1b0700a4cff5fe1 Mon Sep 17 00:00:00 2001 From: Carla Iriberri Date: Fri, 23 Oct 2015 18:44:26 +0200 Subject: [PATCH 1/4] First files --- geocoder/ip-addresses/extension/.gitignore | 3 + geocoder/ip-addresses/extension/Makefile | 8 ++ .../extension/cdb_geocoder_ipaddr--0.0.1.sql | 80 +++++++++++++++++++ .../extension/cdb_geocoder_ipaddr.control | 6 ++ 4 files changed, 97 insertions(+) create mode 100644 geocoder/ip-addresses/extension/.gitignore create mode 100644 geocoder/ip-addresses/extension/Makefile create mode 100644 geocoder/ip-addresses/extension/cdb_geocoder_ipaddr--0.0.1.sql create mode 100644 geocoder/ip-addresses/extension/cdb_geocoder_ipaddr.control diff --git a/geocoder/ip-addresses/extension/.gitignore b/geocoder/ip-addresses/extension/.gitignore new file mode 100644 index 0000000..e710f0e --- /dev/null +++ b/geocoder/ip-addresses/extension/.gitignore @@ -0,0 +1,3 @@ +results/ +regression.diffs +regression.out diff --git a/geocoder/ip-addresses/extension/Makefile b/geocoder/ip-addresses/extension/Makefile new file mode 100644 index 0000000..922a44b --- /dev/null +++ b/geocoder/ip-addresses/extension/Makefile @@ -0,0 +1,8 @@ +EXTENSION = cdb_geocoder_ipaddr +DATA = cdb_geocoder_ipaddr--0.0.1.sql +REGRESS = cdb_geocoder_ipaddr_test + +# postgres build stuff +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) diff --git a/geocoder/ip-addresses/extension/cdb_geocoder_ipaddr--0.0.1.sql b/geocoder/ip-addresses/extension/cdb_geocoder_ipaddr--0.0.1.sql new file mode 100644 index 0000000..7ce8c6b --- /dev/null +++ b/geocoder/ip-addresses/extension/cdb_geocoder_ipaddr--0.0.1.sql @@ -0,0 +1,80 @@ +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_geocoder_ipaddr" to load this file. \quit + +-- Response types for IP addresses geocoder +CREATE TYPE geocode_ip_v1 AS (q text, geom geometry, success boolean); + +-- Public API functions -- +--- Geocoding function --- +-- TODO: deal with permissions + +CREATE OR REPLACE FUNCTION geocode_ip(ip text[]) RETURNS SETOF geocode_ip_v1 + LANGUAGE plpgsql SECURITY DEFINER + AS $$ + DECLARE + ret geocode_ip_v1%rowtype; + n TEXT; + new_ips INET[]; + old_ips TEXT[]; + BEGIN + FOR n IN SELECT unnest(ip) LOOP + BEGIN + IF family(n::inet)=6 THEN + new_ips := array_append(new_ips, n::inet); + old_ips := array_append(old_ips, n); + ELSE + new_ips := array_append(new_ips, ('::ffff:'||n)::inet); + old_ips := array_append(old_ips, n); + END IF; + EXCEPTION WHEN OTHERS THEN + SELECT n AS q, NULL as geom, FALSE as success INTO ret; + RETURN NEXT ret; + END; + END LOOP; + FOR ret IN WITH ips AS (SELECT unnest(old_ips) s, unnest(new_ips) net), + matches AS (SELECT s, (SELECT the_geom FROM ip_address_locations WHERE network_start_ip <= ips.net ORDER BY network_start_ip DESC LIMIT 1) geom FROM ips) + SELECT s, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM matches + LOOP + RETURN NEXT ret; + END LOOP; + RETURN; +END +$$; + +-------------------------------------------------------------------------------- + +-- Support tables + +CREATE TABLE ip_address_locations ( + network_start_ip inet, + the_geom geometry(Geometry,4326), + cartodb_id integer NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL, + updated_at timestamp with time zone DEFAULT now() NOT NULL, + the_geom_webmercator geometry(Geometry,3857) +); + +CREATE SEQUENCE ip_address_locations_cartodb_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; +ALTER SEQUENCE ip_address_locations_cartodb_id_seq OWNED BY ip_address_locations.cartodb_id; +ALTER TABLE ONLY ip_address_locations ALTER COLUMN cartodb_id SET DEFAULT nextval('ip_address_locations_cartodb_id_seq'::regclass); + + +ALTER TABLE ONLY ip_address_locations + ADD CONSTRAINT ip_address_locations_cartodb_id_key UNIQUE (cartodb_id); +ALTER TABLE ONLY ip_address_locations + ADD CONSTRAINT ip_address_locations_pkey PRIMARY KEY (cartodb_id); + + +CREATE INDEX ip_address_locations_the_geom_idx ON ip_address_locations USING gist (the_geom); +CREATE INDEX ip_address_locations_the_geom_webmercator_idx ON ip_address_locations USING gist (the_geom_webmercator); +CREATE INDEX ip_address_locations_startip_idx ON ip_address_locations USING btree (network_start_ip); + +CREATE TRIGGER track_updates AFTER INSERT OR DELETE OR UPDATE OR TRUNCATE ON ip_address_locations FOR EACH STATEMENT EXECUTE PROCEDURE cartodb.cdb_tablemetadata_trigger(); +CREATE TRIGGER update_the_geom_webmercator_trigger BEFORE INSERT OR UPDATE OF the_geom ON ip_address_locations FOR EACH ROW EXECUTE PROCEDURE cartodb._cdb_update_the_geom_webmercator(); +CREATE TRIGGER update_updated_at_trigger BEFORE UPDATE ON ip_address_locations FOR EACH ROW EXECUTE PROCEDURE cartodb._cdb_update_updated_at(); + diff --git a/geocoder/ip-addresses/extension/cdb_geocoder_ipaddr.control b/geocoder/ip-addresses/extension/cdb_geocoder_ipaddr.control new file mode 100644 index 0000000..af9e981 --- /dev/null +++ b/geocoder/ip-addresses/extension/cdb_geocoder_ipaddr.control @@ -0,0 +1,6 @@ +# cdb geocoder ipaddr extension +comment = 'CartoDB admin0 internal geocoder' +default_version = '0.0.1' +relocatable = true +requires = cartodb +superuser = false From 21b5949d35d6f6d7dad62da3c1325b87c53921c8 Mon Sep 17 00:00:00 2001 From: Carla Iriberri Date: Mon, 26 Oct 2015 10:13:55 +0100 Subject: [PATCH 2/4] Adding more files --- geocoder/ip-addresses/extension/README.md | 36 +++++++++++++++++++ .../sql/cdb_geocoder_ipaddr_test.sql | 28 +++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 geocoder/ip-addresses/extension/README.md create mode 100644 geocoder/ip-addresses/extension/sql/cdb_geocoder_ipaddr_test.sql diff --git a/geocoder/ip-addresses/extension/README.md b/geocoder/ip-addresses/extension/README.md new file mode 100644 index 0000000..b814d22 --- /dev/null +++ b/geocoder/ip-addresses/extension/README.md @@ -0,0 +1,36 @@ +# CartoDB IP addresses geocoder extension +Postgres extension for the CartoDB admin0 geocoder. It is meant to contain the functions and related objects needed to geocode by IP addresses. It is not meant to contain the actual data used to geocode them. + +## Dependencies +This extension is thought to be used on top of CartoDB platform. Therefore a cartodb user is required to install the extension onto it. + +The following is a non-comprehensive list of dependencies: + +- Postgres 9.3+ +- Postgis extension +- Schema triggers extension +- CartoDB extension + +## Installation into the db cluster +This requires root privileges +``` +sudo make all install +``` + +## Execute tests +``` +PGUSER=postgres make installcheck +``` + +## Install onto a user's database +``` +psql -U development_cartodb_user_fe3b850a-01c0-48f9-8a26-a82f09e9b53f cartodb_dev_user_fe3b850a-01c0-48f9-8a26-a82f09e9b53f_db +``` + +and then: + +```sql +CREATE EXTENSION cdb_geocoder_ipaddr; +``` + +The extension creation in the user's db does not require special privileges. It can be even created from the sql api. diff --git a/geocoder/ip-addresses/extension/sql/cdb_geocoder_ipaddr_test.sql b/geocoder/ip-addresses/extension/sql/cdb_geocoder_ipaddr_test.sql new file mode 100644 index 0000000..6334ee4 --- /dev/null +++ b/geocoder/ip-addresses/extension/sql/cdb_geocoder_ipaddr_test.sql @@ -0,0 +1,28 @@ +CREATE EXTENSION postgis; +CREATE EXTENSION schema_triggers; +CREATE EXTENSION plpythonu; +CREATE EXTENSION cartodb; +CREATE EXTENSION cdb_geocoder_ipaddr; + +-- Check that the geocoding function is callable, should return success = false +SELECT (geocode_ip(Array['100.0.24.0'])).*; + +-- Mock the varnish invalidation function +CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$ +BEGIN + RETURN; +END +$$ +LANGUAGE plpgsql; + +-- Add a few IP sources +COPY ip_address_locations (network_start_ip, the_geom, cartodb_id, created_at, updated_at, the_geom_webmercator) FROM stdin; +::ffff:2.235.35.0 0101000020E610000072F90FE9B7CF22405DFE43FAEDC34640 2821226 2014-08-25 10:35:51.665546+00 2014-08-25 10:35:51.665546+00 0101000020110F000010801778FBF32F4109868FF8BCC35541 +::ffff:31.7.187.0 \N 2783250 2014-08-25 10:35:51.665546+00 2014-08-25 10:35:51.665546+00 \N +::ffff:64.110.146.0 \N 2783251 2014-08-25 10:35:51.665546+00 2014-08-25 10:35:51.665546+00 \N +::ffff:72.5.198.0 \N 2783252 2014-08-25 10:35:51.665546+00 2014-08-25 10:35:51.665546+00 \N +::ffff:77.73.184.0 \N 2783253 2014-08-25 10:35:51.665546+00 2014-08-25 10:35:51.665546+00 \N +\. + +-- Check that the geocoding function is callable, should return success = true +SELECT (geocode_ip(Array['2.235.35.0'])).*; From 1739000bdd2fb799306799c86a45923c5f561a93 Mon Sep 17 00:00:00 2001 From: Carla Iriberri Date: Mon, 26 Oct 2015 12:31:20 +0100 Subject: [PATCH 3/4] Fixes tests --- geocoder/ip-addresses/extension/README.md | 2 +- .../expected/cdb_geocoder_ipaddr_test.out | 27 +++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 geocoder/ip-addresses/extension/expected/cdb_geocoder_ipaddr_test.out diff --git a/geocoder/ip-addresses/extension/README.md b/geocoder/ip-addresses/extension/README.md index b814d22..908d8a3 100644 --- a/geocoder/ip-addresses/extension/README.md +++ b/geocoder/ip-addresses/extension/README.md @@ -24,7 +24,7 @@ PGUSER=postgres make installcheck ## Install onto a user's database ``` -psql -U development_cartodb_user_fe3b850a-01c0-48f9-8a26-a82f09e9b53f cartodb_dev_user_fe3b850a-01c0-48f9-8a26-a82f09e9b53f_db +psql -U cartodb_dev_user_367c0edc-b2ad-4bab-ad43-3d58a6179a93_db cartodb_dev_user_367c0edc-b2ad-4bab-ad43-3d58a6179a93_db ``` and then: diff --git a/geocoder/ip-addresses/extension/expected/cdb_geocoder_ipaddr_test.out b/geocoder/ip-addresses/extension/expected/cdb_geocoder_ipaddr_test.out new file mode 100644 index 0000000..1834323 --- /dev/null +++ b/geocoder/ip-addresses/extension/expected/cdb_geocoder_ipaddr_test.out @@ -0,0 +1,27 @@ +CREATE EXTENSION postgis; +CREATE EXTENSION schema_triggers; +CREATE EXTENSION plpythonu; +CREATE EXTENSION cartodb; +CREATE EXTENSION cdb_geocoder_ipaddr; +-- Check that the geocoding function is callable, should return success = false +SELECT (geocode_ip(Array['100.0.24.0'])).*; + q | geom | success +------------+------+--------- + 100.0.24.0 | | f +(1 row) + +-- Mock the varnish invalidation function +CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$ +BEGIN + RETURN; +END +$$ +LANGUAGE plpgsql; +-- Add a few IP sources +COPY ip_address_locations (network_start_ip, the_geom, cartodb_id, created_at, updated_at, the_geom_webmercator) FROM stdin; +-- Check that the geocoding function is callable, should return success = true +SELECT (geocode_ip(Array['2.235.35.0'])).*; + q | geom | success +------------+----------------------------------------------------+--------- + 2.235.35.0 | 0101000020E610000072F90FE9B7CF22405DFE43FAEDC34640 | t +(1 row) From 3734ab3dbaf3fbea91fd7f6ad30a6ecebae2e679 Mon Sep 17 00:00:00 2001 From: Carla Date: Mon, 26 Oct 2015 12:49:36 +0100 Subject: [PATCH 4/4] Update README.md --- geocoder/ip-addresses/extension/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geocoder/ip-addresses/extension/README.md b/geocoder/ip-addresses/extension/README.md index 908d8a3..9944371 100644 --- a/geocoder/ip-addresses/extension/README.md +++ b/geocoder/ip-addresses/extension/README.md @@ -1,5 +1,5 @@ # CartoDB IP addresses geocoder extension -Postgres extension for the CartoDB admin0 geocoder. It is meant to contain the functions and related objects needed to geocode by IP addresses. It is not meant to contain the actual data used to geocode them. +Postgres extension for the CartoDB IP addresses geocoder. It is meant to contain the functions and related objects needed to geocode by IP addresses. It is not meant to contain the actual data used to geocode them. ## Dependencies This extension is thought to be used on top of CartoDB platform. Therefore a cartodb user is required to install the extension onto it.