Merge remote-tracking branch 'origin/extension_ipaddr' into merge-into-one-extension

geocoder/ip-addresses/extension/.gitignore

@@ -0,0 +1,3 @@
+results/
+regression.diffs
+regression.out

geocoder/ip-addresses/extension/Makefile

@@ -0,0 +1,8 @@
+EXTENSION = cdb_geocoder_ipaddr
+DATA = cdb_geocoder_ipaddr--0.0.1.sql
+REGRESS = cdb_geocoder_ipaddr_test
+
+# postgres build stuff
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)

geocoder/ip-addresses/extension/README.md

@@ -0,0 +1,36 @@
+# CartoDB IP addresses geocoder extension
+Postgres extension for the CartoDB IP addresses geocoder. It is meant to contain the functions and related objects needed to geocode by IP addresses. It is not meant to contain the actual data used to geocode them.
+
+## Dependencies
+This extension is thought to be used on top of CartoDB platform. Therefore a cartodb user is required to install the extension onto it.
+
+The following is a non-comprehensive list of dependencies:
+
+- Postgres 9.3+
+- Postgis extension
+- Schema triggers extension
+- CartoDB extension
+
+## Installation into the db cluster
+This requires root privileges
+```
+sudo make all install
+```
+
+## Execute tests
+```
+PGUSER=postgres make installcheck
+```
+
+## Install onto a user's database
+```
+psql -U cartodb_dev_user_367c0edc-b2ad-4bab-ad43-3d58a6179a93_db cartodb_dev_user_367c0edc-b2ad-4bab-ad43-3d58a6179a93_db
+```
+
+and then:
+
+```sql
+CREATE EXTENSION cdb_geocoder_ipaddr;
+```
+
+The extension creation in the user's db does not require special privileges. It can be even created from the sql api.

geocoder/ip-addresses/extension/cdb_geocoder_ipaddr--0.0.1.sql

@@ -0,0 +1,80 @@
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "CREATE EXTENSION cdb_geocoder_ipaddr" to load this file. \quit
+
+-- Response types for IP addresses geocoder
+CREATE TYPE geocode_ip_v1 AS (q text, geom geometry, success boolean);
+
+-- Public API functions --
+--- Geocoding function ---
+-- TODO: deal with permissions
+
+CREATE OR REPLACE FUNCTION geocode_ip(ip text[]) RETURNS SETOF geocode_ip_v1
+    LANGUAGE plpgsql SECURITY DEFINER
+    AS $$
+  DECLARE 
+    ret geocode_ip_v1%rowtype;
+    n TEXT;
+    new_ips INET[];
+    old_ips TEXT[];
+  BEGIN
+  FOR n IN SELECT unnest(ip) LOOP
+    BEGIN
+      IF family(n::inet)=6 THEN
+        new_ips := array_append(new_ips, n::inet);
+        old_ips := array_append(old_ips, n);
+      ELSE
+        new_ips := array_append(new_ips, ('::ffff:'||n)::inet);
+        old_ips := array_append(old_ips, n);
+      END IF;
+    EXCEPTION WHEN OTHERS THEN
+      SELECT n AS q, NULL as geom, FALSE as success INTO ret;
+      RETURN NEXT ret;
+    END;
+  END LOOP;
+  FOR ret IN WITH ips AS (SELECT unnest(old_ips) s, unnest(new_ips) net),
+    matches AS (SELECT s, (SELECT the_geom FROM ip_address_locations WHERE network_start_ip <= ips.net ORDER BY network_start_ip DESC LIMIT 1) geom FROM ips)
+    SELECT s, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM matches
+    LOOP
+    RETURN NEXT ret;
+  END LOOP;
+  RETURN;
+END
+$$;
+
+--------------------------------------------------------------------------------
+
+-- Support tables
+
+CREATE TABLE ip_address_locations (
+    network_start_ip inet,
+    the_geom geometry(Geometry,4326),
+    cartodb_id integer NOT NULL,
+    created_at timestamp with time zone DEFAULT now() NOT NULL,
+    updated_at timestamp with time zone DEFAULT now() NOT NULL,
+    the_geom_webmercator geometry(Geometry,3857)
+);
+
+CREATE SEQUENCE ip_address_locations_cartodb_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+ALTER SEQUENCE ip_address_locations_cartodb_id_seq OWNED BY ip_address_locations.cartodb_id;
+ALTER TABLE ONLY ip_address_locations ALTER COLUMN cartodb_id SET DEFAULT nextval('ip_address_locations_cartodb_id_seq'::regclass);
+
+
+ALTER TABLE ONLY ip_address_locations
+    ADD CONSTRAINT ip_address_locations_cartodb_id_key UNIQUE (cartodb_id);
+ALTER TABLE ONLY ip_address_locations
+    ADD CONSTRAINT ip_address_locations_pkey PRIMARY KEY (cartodb_id);
+
+
+CREATE INDEX ip_address_locations_the_geom_idx ON ip_address_locations USING gist (the_geom);
+CREATE INDEX ip_address_locations_the_geom_webmercator_idx ON ip_address_locations USING gist (the_geom_webmercator);
+CREATE INDEX ip_address_locations_startip_idx ON ip_address_locations USING btree (network_start_ip);
+
+CREATE TRIGGER track_updates AFTER INSERT OR DELETE OR UPDATE OR TRUNCATE ON ip_address_locations FOR EACH STATEMENT EXECUTE PROCEDURE cartodb.cdb_tablemetadata_trigger();
+CREATE TRIGGER update_the_geom_webmercator_trigger BEFORE INSERT OR UPDATE OF the_geom ON ip_address_locations FOR EACH ROW EXECUTE PROCEDURE cartodb._cdb_update_the_geom_webmercator();
+CREATE TRIGGER update_updated_at_trigger BEFORE UPDATE ON ip_address_locations FOR EACH ROW EXECUTE PROCEDURE cartodb._cdb_update_updated_at();
+

geocoder/ip-addresses/extension/cdb_geocoder_ipaddr.control

@@ -0,0 +1,6 @@
+# cdb geocoder ipaddr extension
+comment = 'CartoDB admin0 internal geocoder'
+default_version = '0.0.1'
+relocatable = true
+requires = cartodb
+superuser = false

geocoder/ip-addresses/extension/expected/cdb_geocoder_ipaddr_test.out

@@ -0,0 +1,27 @@
+CREATE EXTENSION postgis;
+CREATE EXTENSION schema_triggers;
+CREATE EXTENSION plpythonu;
+CREATE EXTENSION cartodb;
+CREATE EXTENSION cdb_geocoder_ipaddr;
+-- Check that the geocoding function is callable, should return success = false
+SELECT (geocode_ip(Array['100.0.24.0'])).*;
+     q      | geom | success 
+------------+------+---------
+ 100.0.24.0 |      | f
+(1 row)
+
+-- Mock the varnish invalidation function
+CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$
+BEGIN
+  RETURN;
+END
+$$
+LANGUAGE plpgsql;
+-- Add a few IP sources
+COPY ip_address_locations (network_start_ip, the_geom, cartodb_id, created_at, updated_at, the_geom_webmercator) FROM stdin;
+-- Check that the geocoding function is callable, should return success = true
+SELECT (geocode_ip(Array['2.235.35.0'])).*;
+     q      |                        geom                        | success 
+------------+----------------------------------------------------+---------
+ 2.235.35.0 | 0101000020E610000072F90FE9B7CF22405DFE43FAEDC34640 | t
+(1 row)

geocoder/ip-addresses/extension/sql/cdb_geocoder_ipaddr_test.sql

@@ -0,0 +1,28 @@
+CREATE EXTENSION postgis;
+CREATE EXTENSION schema_triggers;
+CREATE EXTENSION plpythonu;
+CREATE EXTENSION cartodb;
+CREATE EXTENSION cdb_geocoder_ipaddr;
+
+-- Check that the geocoding function is callable, should return success = false
+SELECT (geocode_ip(Array['100.0.24.0'])).*;
+
+-- Mock the varnish invalidation function
+CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$
+BEGIN
+  RETURN;
+END
+$$
+LANGUAGE plpgsql;
+
+-- Add a few IP sources
+COPY ip_address_locations (network_start_ip, the_geom, cartodb_id, created_at, updated_at, the_geom_webmercator) FROM stdin;
+::ffff:2.235.35.0	0101000020E610000072F90FE9B7CF22405DFE43FAEDC34640	2821226	2014-08-25 10:35:51.665546+00	2014-08-25 10:35:51.665546+00	0101000020110F000010801778FBF32F4109868FF8BCC35541
+::ffff:31.7.187.0       \N      2783250 2014-08-25 10:35:51.665546+00   2014-08-25 10:35:51.665546+00   \N
+::ffff:64.110.146.0     \N      2783251 2014-08-25 10:35:51.665546+00   2014-08-25 10:35:51.665546+00   \N
+::ffff:72.5.198.0       \N      2783252 2014-08-25 10:35:51.665546+00   2014-08-25 10:35:51.665546+00   \N
+::ffff:77.73.184.0      \N      2783253 2014-08-25 10:35:51.665546+00   2014-08-25 10:35:51.665546+00   \N
+\.
+
+-- Check that the geocoding function is callable, should return success = true
+SELECT (geocode_ip(Array['2.235.35.0'])).*;