Merge pull request #195 from CartoDB/194-sanitize_country_inputs

Sanitize country inputs

geocoder/extension/sql/0.0.1/10_aux_functions.sql

@@ -0,0 +1,8 @@
+-- Cleaning function
+CREATE OR REPLACE FUNCTION geocode_clean_name(name text) RETURNS text
+    LANGUAGE plpgsql
+    AS $$
+  BEGIN
+    RETURN regexp_replace(name, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g');
+  END
+$$;

geocoder/extension/sql/0.0.1/20_admin0.sql

@@ -9,7 +9,7 @@ CREATE OR REPLACE FUNCTION geocode_admin0_polygons(name text[])
   -- FOR ret IN
   RETURN QUERY
     SELECT d.q, n.the_geom as geom, CASE WHEN s.adm0_a3 IS NULL then FALSE ELSE TRUE END AS success
-      FROM (SELECT q, lower(regexp_replace(q, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text x
+      FROM (SELECT q, lower(geocode_clean_name(q))::text x
         FROM (SELECT unnest(name) q) g) d
       LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x
       LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.the_geom, s.adm0_a3;
@@ -24,7 +24,7 @@ CREATE OR REPLACE FUNCTION admin0_synonym_lookup(name text[])
     ret synonym_lookup_v1%rowtype;
  BEGIN  RETURN QUERY
     SELECT d.q, s.adm0_a3
-      FROM (SELECT q, lower(regexp_replace(q, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text x
+      FROM (SELECT q, lower(geocode_clean_name(q))::text x
         FROM (SELECT unnest(name) q) g) d
       LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x GROUP BY d.q, s.adm0_a3;
 END
@@ -74,7 +74,7 @@ CREATE INDEX idx_admin0_synonyms_rank ON admin0_synonyms USING btree (rank);
 -- create trigger function. used in both admin0 and admin1 synonym tables
 CREATE OR REPLACE FUNCTION alpha_numeric_identifiers() RETURNS trigger AS $alpha_numeric_identifiers$
     BEGIN
-        NEW.name_ := lower(regexp_replace(NEW.name, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'));
+        NEW.name_ := lower(geocode_clean_name(NEW.name));
         RETURN NEW;
     END;
 $alpha_numeric_identifiers$ LANGUAGE plpgsql;

geocoder/extension/sql/0.0.1/30_admin1.sql

@@ -4,23 +4,23 @@
 CREATE OR REPLACE FUNCTION geocode_admin1_polygons(name text[]) RETURNS SETOF geocode_admin_v1
     LANGUAGE plpgsql SECURITY DEFINER
     AS $$
-  DECLARE 
+  DECLARE
     ret geocode_admin_v1%rowtype;
   BEGIN
   FOR ret IN
     SELECT
        q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success
     FROM (
-      SELECT 
+      SELECT
         q, (
-          SELECT the_geom 
+          SELECT the_geom
           FROM global_province_polygons
-          WHERE d.c = ANY (synonyms) 
+          WHERE d.c = ANY (synonyms)
           ORDER BY frequency DESC LIMIT 1
         ) geom
       FROM (SELECT trim(replace(lower(unnest(name)),'.',' ')) c, unnest(name) q) d
     ) v
-  LOOP 
+  LOOP
     RETURN NEXT ret;
   END LOOP;
   RETURN;
@@ -31,23 +31,23 @@ $$;
 CREATE OR REPLACE FUNCTION geocode_admin1_polygons(name text[], inputcountry text) RETURNS SETOF geocode_admin_v1
     LANGUAGE plpgsql SECURITY DEFINER
     AS $$
-  DECLARE 
+  DECLARE
     ret geocode_admin_v1%rowtype;
   BEGIN
 
-  FOR ret IN WITH 
+  FOR ret IN WITH
-    p AS (SELECT r.c, r.q, (SELECT iso3 FROM country_decoder WHERE lower(inputcountry) = ANY (synonyms)) i FROM (SELECT  trim(replace(lower(unnest(name)),'.',' ')) c, unnest(name) q) r)
+    p AS (SELECT r.c, r.q, (SELECT iso3 FROM country_decoder WHERE lower(geocode_clean_name(inputcountry))::text = ANY (synonyms)) i FROM (SELECT  trim(replace(lower(unnest(name)),'.',' ')) c, unnest(name) q) r)
     SELECT
        q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success
     FROM (
-      SELECT 
+      SELECT
         q, (
-          SELECT the_geom 
+          SELECT the_geom
           FROM global_province_polygons
-          WHERE p.c = ANY (synonyms) 
+          WHERE p.c = ANY (synonyms)
           AND iso3 = p.i
           -- To calculate frequency, I simply counted the number of users
-          -- we had signed up in each country. Countries with more users, 
+          -- we had signed up in each country. Countries with more users,
           -- we favor higher in the geocoder :)
           ORDER BY frequency DESC LIMIT 1
         ) geom
@@ -63,7 +63,7 @@ $$;
 CREATE OR REPLACE FUNCTION geocode_admin1_polygons(names text[], country text[]) RETURNS SETOF geocode_admin_country_v1
     LANGUAGE plpgsql SECURITY DEFINER
     AS $$
-  DECLARE 
+  DECLARE
     ret geocode_admin_country_v1%rowtype;
     nans TEXT[];
   BEGIN
@@ -79,19 +79,19 @@ CREATE OR REPLACE FUNCTION geocode_admin1_polygons(names text[], country text[])
   END IF;
 
 
-  FOR ret IN WITH 
+  FOR ret IN WITH
-    p AS (SELECT r.p, r.q, c, (SELECT iso3 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT  trim(replace(lower(unnest(names)),'.',' ')) p, unnest(names) q, unnest(country) c) r)
+    p AS (SELECT r.p, r.q, c, (SELECT iso3 FROM country_decoder WHERE lower(geocode_clean_name(r.c))::text = ANY (synonyms)) i FROM (SELECT  trim(replace(lower(unnest(names)),'.',' ')) p, unnest(names) q, unnest(country) c) r)
     SELECT
        q, c, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success
     FROM (
-      SELECT 
+      SELECT
         q, c, (
-          SELECT the_geom 
+          SELECT the_geom
           FROM global_province_polygons
-          WHERE p.p = ANY (synonyms) 
+          WHERE p.p = ANY (synonyms)
           AND iso3 = p.i
           -- To calculate frequency, I simply counted the number of users
-          -- we had signed up in each country. Countries with more users, 
+          -- we had signed up in each country. Countries with more users,
           -- we favor higher in the geocoder :)
           ORDER BY frequency DESC LIMIT 1
         ) geom

geocoder/extension/sql/0.0.1/50_namedplaces.sql

@@ -5,7 +5,7 @@
 CREATE OR REPLACE FUNCTION geocode_namedplace(places text[], country text[]) RETURNS SETOF geocode_namedplace_country_v1
     LANGUAGE plpgsql IMMUTABLE SECURITY DEFINER
     AS $$
-  DECLARE 
+  DECLARE
     ret geocode_namedplace_country_v1%rowtype;
     nans TEXT[];
   BEGIN
@@ -27,13 +27,13 @@ CREATE OR REPLACE FUNCTION geocode_namedplace(places text[], country text[]) RET
     END LOOP;
   END IF;
 
-  FOR ret IN WITH 
+  FOR ret IN WITH
-    p AS (SELECT r.s, r.c, (SELECT iso2 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT unnest(places) AS s, unnest(country)::text AS c) r),
+    p AS (SELECT r.s, r.c, (SELECT iso2 FROM country_decoder WHERE lower(geocode_clean_name(r.c))::text = ANY (synonyms)) i FROM (SELECT unnest(places) AS s, unnest(country)::text AS c) r),
     best AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p),
     next AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE c = p.c AND geom IS NOT NULL))
     SELECT q, c, geom, TRUE AS success FROM best WHERE geom IS NOT NULL
     UNION ALL
-    SELECT q, c, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next 
+    SELECT q, c, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next
     LOOP
     RETURN NEXT ret;
   END LOOP;
@@ -45,7 +45,7 @@ $$;
 CREATE OR REPLACE FUNCTION geocode_namedplace(places text[], inputcountry text) RETURNS SETOF geocode_admin_country_v1
     LANGUAGE plpgsql IMMUTABLE SECURITY DEFINER
     AS $$
-  DECLARE 
+  DECLARE
     ret geocode_admin_country_v1%rowtype;
     isoTwo TEXT := NULL;
     has_country BOOLEAN;
@@ -59,22 +59,22 @@ CREATE OR REPLACE FUNCTION geocode_namedplace(places text[], inputcountry text)
   END IF;
 
   IF has_country THEN
-    SELECT iso2 INTO isoTwo FROM country_decoder WHERE lower(inputcountry) = ANY (synonyms) LIMIT 1;
+    SELECT iso2 INTO isoTwo FROM country_decoder WHERE lower(geocode_clean_name(inputcountry))::text = ANY (synonyms) LIMIT 1;
-    FOR ret IN WITH 
+    FOR ret IN WITH
       best AS (SELECT p.s AS q, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = isoTwo ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT unnest(places) AS s) p),
       next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = isoTwo AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT unnest(places) AS s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL))
       SELECT q, inputcountry c, geom, TRUE AS success FROM best WHERE geom IS NOT NULL
       UNION ALL
-      SELECT q, inputcountry c, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next 
+      SELECT q, inputcountry c, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next
       LOOP
       RETURN NEXT ret;
     END LOOP;
   -- no country included, or iso interpretation found
-  ELSE 
+  ELSE
-    FOR ret IN 
+    FOR ret IN
-      SELECT g.q as q, inputcountry as c, g.geom as geom, g.success as success FROM (SELECT (geocode_namedplace(places)).*) g 
+      SELECT g.q as q, inputcountry as c, g.geom as geom, g.success as success FROM (SELECT (geocode_namedplace(places)).*) g
     LOOP
-      RETURN NEXT ret; 
+      RETURN NEXT ret;
     END LOOP;
   END IF;
   RETURN;
@@ -85,7 +85,7 @@ $$;
 CREATE OR REPLACE FUNCTION geocode_namedplace(places text[], admin1s text, inputcountry text) RETURNS SETOF geocode_admin1_country_v1
     LANGUAGE plpgsql IMMUTABLE SECURITY DEFINER
     AS $$
-  DECLARE 
+  DECLARE
     ret geocode_admin1_country_v1%rowtype;
     has_country BOOLEAN;
     has_admin1s BOOLEAN;
@@ -115,12 +115,12 @@ CREATE OR REPLACE FUNCTION geocode_namedplace(places text[], admin1s text, input
         RETURN NEXT ret;
       END LOOP;
     -- no country, has admin1 value
-    ELSE 
+    ELSE
-      FOR ret IN 
+      FOR ret IN
           SELECT g.q, admin1s AS a1, inputcountry as c, g.geom, g.success FROM (
           SELECT (
             geocode_namedplace(
-              places, 
+              places,
               (SELECT array_agg(a) FROM (SELECT admin1s a FROM GENERATE_SERIES(1, Array_Length(places, 1)) s) r),
               NULL
             )
@@ -136,12 +136,12 @@ CREATE OR REPLACE FUNCTION geocode_namedplace(places text[], admin1s text, input
         RETURN NEXT ret;
       END LOOP;
     -- has country, has admin1 value
-    ELSE 
+    ELSE
-      FOR ret IN 
+      FOR ret IN
           SELECT g.q, admin1s AS a1, inputcountry as c, g.geom, g.success FROM (
           SELECT (
             geocode_namedplace(
-              places, 
+              places,
               (SELECT array_agg(a) FROM (SELECT admin1s a FROM GENERATE_SERIES(1, Array_Length(places, 1)) s) r),
               inputcountry
             )
@@ -158,7 +158,7 @@ $$;
 CREATE OR REPLACE FUNCTION geocode_namedplace(places text[], admin1s text[], inputcountry text) RETURNS SETOF geocode_admin1_country_v1
     LANGUAGE plpgsql IMMUTABLE SECURITY DEFINER
     AS $$
-  DECLARE 
+  DECLARE
     ret geocode_admin1_country_v1%rowtype;
     nans TEXT[];
     isoTwo TEXT := NULL;
@@ -172,7 +172,7 @@ CREATE OR REPLACE FUNCTION geocode_namedplace(places text[], admin1s text[], inp
     has_country := FALSE;
   END IF;
   IF has_country THEN
-    SELECT iso2 INTO isoTwo FROM country_decoder WHERE lower(inputcountry) = ANY (synonyms) LIMIT 1;
+    SELECT iso2 INTO isoTwo FROM country_decoder WHERE lower(geocode_clean_name(inputcountry))::text = ANY (synonyms) LIMIT 1;
   END IF;
 
   -- find all cases where admin1 is NULL
@@ -200,15 +200,15 @@ CREATE OR REPLACE FUNCTION geocode_namedplace(places text[], admin1s text[], inp
     SELECT array_agg(p), array_agg(c) INTO places, admin1s FROM (SELECT unnest(places) p, unnest(admin1s) c) g WHERE c!='';
     IF has_country THEN
       -- geocode our named place without admin1 but with our iso2
-      FOR ret IN 
+      FOR ret IN
-        SELECT g.q, '' AS a1, inputcountry as c, g.geom, g.success FROM (SELECT (geocode_namedplace(nans, inputcountry)).*) g 
+        SELECT g.q, '' AS a1, inputcountry as c, g.geom, g.success FROM (SELECT (geocode_namedplace(nans, inputcountry)).*) g
       LOOP
         RETURN NEXT ret;
       END LOOP;
     ELSE
       -- geocode our named place without admin1 and without iso2
-      FOR ret IN 
+      FOR ret IN
-        SELECT g.q, '' AS a1, inputcountry as c, g.geom, g.success FROM (SELECT (geocode_namedplace(nans)).*) g 
+        SELECT g.q, '' AS a1, inputcountry as c, g.geom, g.success FROM (SELECT (geocode_namedplace(nans)).*) g
       LOOP
         RETURN NEXT ret;
       END LOOP;
@@ -217,8 +217,8 @@ CREATE OR REPLACE FUNCTION geocode_namedplace(places text[], admin1s text[], inp
 
   -- geocode all the cases where admin1 is available
   IF has_country THEN
-    FOR ret IN WITH 
+    FOR ret IN WITH
-      -- return c=iso2 and search without country 
+      -- return c=iso2 and search without country
       p AS (
         SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder WHERE lower(r.a1) = ANY (synonyms) AND admin1_decoder.iso2 = isoTwo LIMIT 1) i FROM (SELECT unnest(places) AS s, unnest(admin1s)::text AS a1) r),
       best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p),
@@ -230,15 +230,15 @@ CREATE OR REPLACE FUNCTION geocode_namedplace(places text[], admin1s text[], inp
       RETURN NEXT ret;
     END LOOP;
   ELSE
-    -- return c=NULL and search without country 
+    -- return c=NULL and search without country
-    FOR ret IN WITH 
+    FOR ret IN WITH
       p AS (
         SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder WHERE lower(r.a1) = ANY (synonyms) LIMIT 1) i FROM (SELECT unnest(places) AS s, unnest(admin1s)::text AS a1) r WHERE a1 IS NOT NULL and a1 != ''),
       best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p),
       next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL))
       SELECT q, a1, inputcountry as c, geom, TRUE AS success FROM best WHERE geom IS NOT NULL
       UNION ALL
-      SELECT q, a1, inputcountry as c, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next 
+      SELECT q, a1, inputcountry as c, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next
       LOOP
       RETURN NEXT ret;
     END LOOP;
@@ -252,14 +252,14 @@ $$;
 CREATE OR REPLACE FUNCTION geocode_namedplace(places text[], admin1s text[], inputcountry text[]) RETURNS SETOF geocode_admin1_country_v1
     LANGUAGE plpgsql IMMUTABLE SECURITY DEFINER
     AS $$
-  DECLARE 
+  DECLARE
     ret geocode_admin1_country_v1%rowtype;
   BEGIN
     IF admin1s IS NULL THEN
       FOR ret IN SELECT g.q as q, NULL as a1, g.c as c, g.geom as geom, g.success as success FROM (SELECT (geocode_namedplace(places, inputcountry)).*) g LOOP
           RETURN NEXT ret;
       END LOOP;
-    ELSE 
+    ELSE
       FOR ret IN WITH clean AS (SELECT array_agg(p) p, array_agg(a) a, c FROM (SELECT p, a, c FROM (SELECT unnest(places) p, unnest(admin1s) a, unnest(inputcountry) c) z GROUP BY p, a, c) y GROUP BY c)
         SELECT (geocode_namedplace(p, a, c)).* FROM clean LOOP
           RETURN NEXT ret;
@@ -273,14 +273,14 @@ $$;
 CREATE OR REPLACE FUNCTION geocode_namedplace(places text[]) RETURNS SETOF geocode_namedplace_v1
     LANGUAGE plpgsql IMMUTABLE SECURITY DEFINER
     AS $$
-  DECLARE 
+  DECLARE
     ret geocode_namedplace_v1%rowtype;
   BEGIN
   FOR ret IN WITH best AS (SELECT s AS q, (SELECT the_geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT unnest(places) as s) p),
         next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT unnest(places) as s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL))
         SELECT q, geom, TRUE AS success FROM best WHERE geom IS NOT NULL
         UNION ALL
-        SELECT q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next 
+        SELECT q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next
     LOOP
     RETURN NEXT ret;
   END LOOP;

geocoder/extension/sql/0.0.1/60_postalcodes.sql

@@ -5,18 +5,18 @@
 CREATE FUNCTION geocode_postalcode_polygons(code text[], inputcountries text[]) RETURNS SETOF geocode_namedplace_country_v1
     LANGUAGE plpgsql SECURITY DEFINER
     AS $$
-  DECLARE 
+  DECLARE
     ret geocode_namedplace_country_v1%rowtype;
     adm text[];
   BEGIN
-  
+
-  SELECT INTO adm array_agg((SELECT adm0_a3 FROM admin0_synonyms WHERE name_ = lower(regexp_replace(b.c, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text LIMIT 1)) FROM (SELECT UNNEST(inputcountries) c) b;
+  SELECT INTO adm array_agg((SELECT adm0_a3 FROM admin0_synonyms WHERE name_ = lower(geocode_clean_name(b.c))::text LIMIT 1)) FROM (SELECT UNNEST(inputcountries) c) b;
 
   FOR ret IN
     SELECT
       q, c, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success
     FROM (
-      SELECT 
+      SELECT
         q, c, (
           SELECT the_geom
           FROM global_postal_code_polygons
@@ -25,7 +25,7 @@ CREATE FUNCTION geocode_postalcode_polygons(code text[], inputcountries text[])
         ) geom
       FROM (SELECT unnest(code) q, unnest(inputcountries) c, unnest(adm) a) d
     ) v
-  LOOP 
+  LOOP
     RETURN NEXT ret;
   END LOOP;
   RETURN;
@@ -38,26 +38,26 @@ $$;
 CREATE FUNCTION geocode_postalcode_polygons(code text[], inputcountry text) RETURNS SETOF geocode_namedplace_v1
     LANGUAGE plpgsql SECURITY DEFINER
     AS $$
-  DECLARE 
+  DECLARE
     ret geocode_namedplace_v1%rowtype;
   BEGIN
   FOR ret IN
     SELECT
       q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success
     FROM (
-      SELECT 
+      SELECT
         q, (
-          SELECT the_geom 
+          SELECT the_geom
           FROM global_postal_code_polygons
           WHERE postal_code = upper(d.q)
             AND iso3 = (
                 SELECT iso3 FROM country_decoder WHERE
-                lower(inputcountry) = ANY (synonyms) LIMIT 1
+                lower(geocode_clean_name(inputcountry))::text = ANY (synonyms) LIMIT 1
             )
         ) geom
       FROM (SELECT unnest(code) q) d
     ) v
-  LOOP 
+  LOOP
     RETURN NEXT ret;
   END LOOP;
   RETURN;
@@ -69,23 +69,23 @@ $$;
 CREATE FUNCTION geocode_postalcode_polygons(code text[]) RETURNS SETOF geocode_namedplace_v1
     LANGUAGE plpgsql SECURITY DEFINER
     AS $$
-  DECLARE 
+  DECLARE
     ret geocode_namedplace_v1%rowtype;
   BEGIN
   FOR ret IN
     SELECT
       q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success
     FROM (
-      SELECT 
+      SELECT
         q, (
-          SELECT the_geom 
+          SELECT the_geom
           FROM global_postal_code_polygons
-          WHERE postal_code = upper(d.q) 
+          WHERE postal_code = upper(d.q)
           LIMIT 1
         ) geom
       FROM (SELECT unnest(code) q) d
     ) v
-  LOOP 
+  LOOP
     RETURN NEXT ret;
   END LOOP;
   RETURN;
@@ -98,27 +98,27 @@ $$;
 CREATE FUNCTION geocode_postalcode_points(code text[], inputcountry text) RETURNS SETOF geocode_namedplace_v1
     LANGUAGE plpgsql SECURITY DEFINER
     AS $$
-  DECLARE 
+  DECLARE
     ret geocode_namedplace_v1%rowtype;
   BEGIN
   FOR ret IN
     SELECT
       q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success
     FROM (
-      SELECT 
+      SELECT
         q, (
-          SELECT the_geom 
+          SELECT the_geom
           FROM global_postal_code_points
           WHERE postal_code = upper(d.q)
             AND iso3 = (
                 SELECT iso3 FROM country_decoder WHERE
-                lower(inputcountry) = ANY (synonyms) LIMIT 1
+                lower(geocode_clean_name(inputcountry))::text = ANY (synonyms) LIMIT 1
             )
           LIMIT 1
         ) geom
       FROM (SELECT unnest(code) q) d
     ) v
-  LOOP 
+  LOOP
     RETURN NEXT ret;
   END LOOP;
   RETURN;
@@ -130,27 +130,27 @@ $$;
 CREATE FUNCTION geocode_postalcode_points(code integer[], inputcountries text[]) RETURNS SETOF geocode_postalint_country_v1
     LANGUAGE plpgsql SECURITY DEFINER
     AS $$
-  DECLARE 
+  DECLARE
     ret geocode_postalint_country_v1%rowtype;
   BEGIN
   FOR ret IN
     SELECT
       q, c, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success
     FROM (
-      SELECT 
+      SELECT
         q, c, (
           SELECT the_geom
           FROM global_postal_code_points
           WHERE postal_code_num = d.q
             AND iso3 = (
                 SELECT iso3 FROM country_decoder WHERE
-                lower(d.c) = ANY (synonyms) LIMIT 1
+                lower(geocode_clean_name(d.c))::text = ANY (synonyms) LIMIT 1
             )
           LIMIT 1
         ) geom
       FROM (SELECT unnest(code) q, unnest(inputcountries) c) d
     ) v
-  LOOP 
+  LOOP
     RETURN NEXT ret;
   END LOOP;
   RETURN;
@@ -162,23 +162,23 @@ $$;
 CREATE FUNCTION geocode_postalcode_points(code text[]) RETURNS SETOF geocode_namedplace_v1
     LANGUAGE plpgsql SECURITY DEFINER
     AS $$
-  DECLARE 
+  DECLARE
     ret geocode_namedplace_v1%rowtype;
   BEGIN
   FOR ret IN
     SELECT
       q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success
     FROM (
-      SELECT 
+      SELECT
         q, (
-          SELECT the_geom 
+          SELECT the_geom
           FROM global_postal_code_points
-          WHERE postal_code = upper(d.q) 
+          WHERE postal_code = upper(d.q)
           LIMIT 1
         ) geom
       FROM (SELECT unnest(code) q) d
     ) v
-  LOOP 
+  LOOP
     RETURN NEXT ret;
   END LOOP;
   RETURN;
@@ -190,7 +190,7 @@ $$;
 CREATE FUNCTION geocode_postalcode_points(code text[], inputcountries text[]) RETURNS SETOF geocode_place_country_iso_v1
     LANGUAGE plpgsql SECURITY DEFINER
     AS $$
-  DECLARE 
+  DECLARE
     ret geocode_place_country_iso_v1%rowtype;
     geo GEOMETRY;
   BEGIN
@@ -199,21 +199,21 @@ CREATE FUNCTION geocode_postalcode_points(code text[], inputcountries text[]) RE
     SELECT
       q, c, iso3, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success
     FROM (
-      SELECT 
+      SELECT
         q, c, (SELECT iso3 FROM country_decoder WHERE
-                lower(d.c) = ANY (synonyms) LIMIT 1) iso3, (
+                lower(geocode_clean_name(d.c))::text = ANY (synonyms) LIMIT 1) iso3, (
           SELECT the_geom
           FROM global_postal_code_points
           WHERE postal_code = upper(d.q)
             AND iso3 = (
                 SELECT iso3 FROM country_decoder WHERE
-                lower(d.c) = ANY (synonyms) LIMIT 1
+                lower(geocode_clean_name(d.c))::text = ANY (synonyms) LIMIT 1
             )
           LIMIT 1
         ) geom
       FROM (SELECT unnest(code) q, unnest(inputcountries) c) d
     ) v
-  LOOP 
+  LOOP
     IF ret.geom IS NULL AND ret.iso3 = 'GBR' THEN
       geo := geocode_greatbritain_outward(ret.q);
       IF geo IS NOT NULL THEN
@@ -239,7 +239,7 @@ CREATE FUNCTION geocode_greatbritain_outward(code text) RETURNS geometry
   geom := NULL;
   IF array_length(string_to_array(code,' '),1) = 2 THEN
     code := split_part(code, ' ', 1) || ' ' || rpad(substring(split_part(code, ' ', 2), 1, 1), 3, '#');
-    SELECT the_geom INTO geom FROM global_postal_code_points WHERE 
+    SELECT the_geom INTO geom FROM global_postal_code_points WHERE
       postal_code = code
       AND iso3 = 'GBR'
       LIMIT 1;
@@ -252,15 +252,15 @@ $$;
 CREATE FUNCTION admin0_available_services(name text[]) RETURNS SETOF available_services_v1
     LANGUAGE plpgsql SECURITY DEFINER
     AS $$
-  DECLARE 
+  DECLARE
     ret available_services_v1%rowtype;
   BEGIN  RETURN QUERY
-    SELECT d.q, n.adm0_a3, n.postal_code_points, n.postal_code_polygons FROM 
+    SELECT d.q, n.adm0_a3, n.postal_code_points, n.postal_code_polygons FROM
       (
-        SELECT q, lower(regexp_replace(q, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text x FROM 
+        SELECT q, lower(geocode_clean_name(q))::text x FROM
           (
             SELECT unnest(name) q
-          ) 
+          )
       g) d LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x LEFT OUTER JOIN available_services n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.adm0_a3, n.postal_code_points, n.postal_code_polygons;
 END
 $$;