Merge pull request #235 from CartoDB/fix_pg10_issues

Fix pg10 issues

NEWS.md

@@ -0,0 +1,4 @@
+0.0.2 (19/02/2018)
+------------------
+
+Compatibility with PostgreSQL 10.

geocoder/extension/Makefile

@@ -3,9 +3,13 @@
 EXTENSION = cdb_geocoder
 EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/")
 
-DATA = $(EXTENSION)--$(EXTVERSION).sql
+NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql
+DATA = $(NEW_EXTENSION_ARTIFACT) \
+         $(EXTENSION)--*--*.sql
 
-REGRESS = $(notdir $(basename $(sort $(wildcard sql/*test.sql))))
+REGRESS = $(notdir $(basename $(sort $(wildcard test/sql/*test.sql))))
+TEST_DIR = test/
+REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)' --user='postgres'
 
 # postgres build stuff
 PG_CONFIG = pg_config
@@ -13,10 +17,10 @@ PGXS := $(shell $(PG_CONFIG) --pgxs)
 include $(PGXS)
 
 
-SOURCES_DATA = $(wildcard sql/$(EXTVERSION)/*.sql)
+SOURCES_DATA = $(wildcard sql/*.sql)
 
-$(DATA): $(SOURCES_DATA)
+$(NEW_EXTENSION_ARTIFACT): $(SOURCES_DATA)
-	rm -f $@
+	rm -f $(EXTENSION)--*.sql
 	cat $(SOURCES_DATA) >> $@
 
 all: $(DATA)

geocoder/extension/cdb_geocoder--0.0.1--0.0.2.sql

@@ -0,0 +1,58 @@
+CREATE OR REPLACE FUNCTION geocode_admin0_polygons(name text[])
+  RETURNS SETOF geocode_admin_v1 AS $$
+  DECLARE
+    ret geocode_admin_v1%rowtype;
+  BEGIN
+  -- FOR ret IN
+  RETURN QUERY
+    SELECT q, n.the_geom as geom, CASE WHEN s.adm0_a3 IS NULL then FALSE ELSE TRUE END AS success
+    FROM unnest(name) WITH ORDINALITY q
+    LEFT OUTER JOIN admin0_synonyms s ON name_ = lower(geocode_clean_name(q))::text
+    LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3
+    GROUP BY q, n.the_geom, s.adm0_a3, q.ordinality
+    ORDER BY q.ordinality;
+END
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
+
+CREATE OR REPLACE FUNCTION admin0_synonym_lookup(name text[])
+ RETURNS SETOF synonym_lookup_v1 AS $$
+ DECLARE
+    ret synonym_lookup_v1%rowtype;
+ BEGIN  RETURN QUERY
+    SELECT q, s.adm0_a3
+    FROM unnest(name) WITH ORDINALITY q
+    LEFT OUTER JOIN admin0_synonyms s ON name_ = lower(geocode_clean_name(q))::text
+    GROUP BY q, s.adm0_a3, q.ordinality
+    ORDER BY q.ordinality;
+END
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
+
+CREATE OR REPLACE FUNCTION geocode_namedplace(places text[], admin1s text[], inputcountry text[]) RETURNS SETOF geocode_admin1_country_v1
+    LANGUAGE plpgsql IMMUTABLE SECURITY DEFINER
+    AS $$
+  DECLARE
+    ret geocode_admin1_country_v1%rowtype;
+  BEGIN
+    IF admin1s IS NULL THEN
+      FOR ret IN SELECT g.q as q, NULL as a1, g.c as c, g.geom as geom, g.success as success FROM (SELECT (geocode_namedplace(places, inputcountry)).*) g LOOP
+          RETURN NEXT ret;
+      END LOOP;
+    ELSE
+      FOR ret IN WITH clean AS (
+          SELECT array_agg(p) p, array_agg(a) a, c
+          FROM (SELECT p, a, c
+                FROM (SELECT p, a, c, nest.ordinality as ord FROM unnest(places, admin1s) with ordinality nest (p, a), LATERAL unnest(inputcountry) with ordinality c) z
+                GROUP BY p, a, c, z.ord
+                ORDER BY z.ord
+              ) y
+          GROUP BY c
+        )
+        SELECT (geocode_namedplace(p, a, c)).* FROM clean
+      LOOP
+        RETURN NEXT ret;
+      END LOOP;
+    END IF;
+  RETURN;
+END
+$$;
+

geocoder/extension/cdb_geocoder--0.0.2--0.0.1.sql

@@ -0,0 +1,46 @@
+CREATE OR REPLACE FUNCTION geocode_admin0_polygons(name text[])
+  RETURNS SETOF geocode_admin_v1 AS $$
+  DECLARE
+    ret geocode_admin_v1%rowtype;
+  BEGIN
+  -- FOR ret IN
+  RETURN QUERY
+    SELECT d.q, n.the_geom as geom, CASE WHEN s.adm0_a3 IS NULL then FALSE ELSE TRUE END AS success
+      FROM (SELECT q, lower(geocode_clean_name(q))::text x
+        FROM (SELECT unnest(name) q) g) d
+      LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x
+      LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.the_geom, s.adm0_a3;
+END
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
+
+CREATE OR REPLACE FUNCTION admin0_synonym_lookup(name text[])
+ RETURNS SETOF synonym_lookup_v1 AS $$
+ DECLARE
+    ret synonym_lookup_v1%rowtype;
+ BEGIN  RETURN QUERY
+    SELECT d.q, s.adm0_a3
+      FROM (SELECT q, lower(geocode_clean_name(q))::text x
+        FROM (SELECT unnest(name) q) g) d
+      LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x GROUP BY d.q, s.adm0_a3;
+END
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
+
+CREATE OR REPLACE FUNCTION geocode_namedplace(places text[], admin1s text[], inputcountry text[]) RETURNS SETOF geocode_admin1_country_v1
+    LANGUAGE plpgsql IMMUTABLE SECURITY DEFINER
+    AS $$
+  DECLARE
+    ret geocode_admin1_country_v1%rowtype;
+  BEGIN
+    IF admin1s IS NULL THEN
+      FOR ret IN SELECT g.q as q, NULL as a1, g.c as c, g.geom as geom, g.success as success FROM (SELECT (geocode_namedplace(places, inputcountry)).*) g LOOP
+          RETURN NEXT ret;
+      END LOOP;
+    ELSE
+      FOR ret IN WITH clean AS (SELECT array_agg(p) p, array_agg(a) a, c FROM (SELECT p, a, c FROM (SELECT unnest(places) p, unnest(admin1s) a, unnest(inputcountry) c) z GROUP BY p, a, c) y GROUP BY c)
+        SELECT (geocode_namedplace(p, a, c)).* FROM clean LOOP
+          RETURN NEXT ret;
+      END LOOP;
+    END IF;
+  RETURN;
+END
+$$;

geocoder/extension/cdb_geocoder.control

@@ -1,6 +1,6 @@
 # cdb geocoder extension
 comment = 'CartoDB internal geocoder'
-default_version = '0.0.1'
+default_version = '0.0.2'
 relocatable = true
 requires = cartodb
 superuser = false

geocoder/extension/sql/20_admin0.sql

@@ -8,11 +8,12 @@ CREATE OR REPLACE FUNCTION geocode_admin0_polygons(name text[])
   BEGIN
   -- FOR ret IN
   RETURN QUERY
-    SELECT d.q, n.the_geom as geom, CASE WHEN s.adm0_a3 IS NULL then FALSE ELSE TRUE END AS success
+    SELECT q, n.the_geom as geom, CASE WHEN s.adm0_a3 IS NULL then FALSE ELSE TRUE END AS success
-      FROM (SELECT q, lower(geocode_clean_name(q))::text x
+    FROM unnest(name) WITH ORDINALITY q
-        FROM (SELECT unnest(name) q) g) d
+    LEFT OUTER JOIN admin0_synonyms s ON name_ = lower(geocode_clean_name(q))::text
-      LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x
+    LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3
-      LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.the_geom, s.adm0_a3;
+    GROUP BY q, n.the_geom, s.adm0_a3, q.ordinality
+    ORDER BY q.ordinality;
 END
 $$ LANGUAGE 'plpgsql' SECURITY DEFINER;
 
@@ -23,10 +24,11 @@ CREATE OR REPLACE FUNCTION admin0_synonym_lookup(name text[])
  DECLARE
     ret synonym_lookup_v1%rowtype;
  BEGIN  RETURN QUERY
-    SELECT d.q, s.adm0_a3
+    SELECT q, s.adm0_a3
-      FROM (SELECT q, lower(geocode_clean_name(q))::text x
+    FROM unnest(name) WITH ORDINALITY q
-        FROM (SELECT unnest(name) q) g) d
+    LEFT OUTER JOIN admin0_synonyms s ON name_ = lower(geocode_clean_name(q))::text
-      LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x GROUP BY d.q, s.adm0_a3;
+    GROUP BY q, s.adm0_a3, q.ordinality
+    ORDER BY q.ordinality;
 END
 $$ LANGUAGE 'plpgsql' SECURITY DEFINER;
 

geocoder/extension/sql/50_namedplaces.sql

@@ -225,7 +225,7 @@ CREATE OR REPLACE FUNCTION geocode_namedplace(places text[], admin1s text[], inp
       next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL))
       SELECT q, a1, inputcountry as c, geom, TRUE AS success FROM best WHERE geom IS NOT NULL
       UNION ALL
-      SELECT q, a1, inputcountry as c, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next 
+      SELECT q, a1, inputcountry as c, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next
       LOOP
       RETURN NEXT ret;
     END LOOP;
@@ -260,9 +260,18 @@ CREATE OR REPLACE FUNCTION geocode_namedplace(places text[], admin1s text[], inp
           RETURN NEXT ret;
       END LOOP;
     ELSE
-      FOR ret IN WITH clean AS (SELECT array_agg(p) p, array_agg(a) a, c FROM (SELECT p, a, c FROM (SELECT unnest(places) p, unnest(admin1s) a, unnest(inputcountry) c) z GROUP BY p, a, c) y GROUP BY c)
+      FOR ret IN WITH clean AS (
-        SELECT (geocode_namedplace(p, a, c)).* FROM clean LOOP
+          SELECT array_agg(p) p, array_agg(a) a, c
-          RETURN NEXT ret;
+          FROM (SELECT p, a, c
+                FROM (SELECT p, a, c, nest.ordinality as ord FROM unnest(places, admin1s) with ordinality nest (p, a), LATERAL unnest(inputcountry) with ordinality c) z
+                GROUP BY p, a, c, z.ord
+                ORDER BY z.ord
+              ) y
+          GROUP BY c
+        )
+        SELECT (geocode_namedplace(p, a, c)).* FROM clean
+      LOOP
+        RETURN NEXT ret;
       END LOOP;
     END IF;
   RETURN;

geocoder/extension/test/expected/20_admin0_test.out

@@ -10,9 +10,9 @@ SELECT (admin0_synonym_lookup(Array['United States', 'ESP'])).*;
 SELECT (geocode_admin0_polygons(Array['Spain', 'USA', ''])).*;
    q   | geom | success 
 -------+------+---------
-       |      | f
  Spain |      | f
  USA   |      | f
+       |      | f
 (3 rows)
 
 -- Add a few synonyms
@@ -21,7 +21,7 @@ COPY admin0_synonyms (name, rank, created_at, updated_at, the_geom, the_geom_web
 SELECT (admin0_synonym_lookup(Array['United States', 'ESP'])).*;
        q       | adm0_a3 
 ---------------+---------
- ESP           | ESP
  United States | USA
+ ESP           | ESP
 (2 rows)
 

geocoder/extension/test/expected/50_namedplaces_test.out

@@ -42,8 +42,8 @@ SELECT (geocode_namedplace(Array['Portland', 'Portland', 'New York City'], Array
        q       |   a1   |  c  | geom | success 
 ---------------+--------+-----+------+---------
  New York City |        | USA |      | f
- Portland      | Oregon | USA |      | f
  Portland      | Maine  | USA |      | f
+ Portland      | Oregon | USA |      | f
 (3 rows)
 
 SELECT namedplace_guess_country(Array['granada', 'jaen', 'cordoba', 'madrid', 'valladolid']);