From 46d7027cdb1731fd15c96212750f194ce42de24a Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Fri, 4 May 2018 15:18:55 +0200
Subject: [PATCH 01/45] Created MVT returning function for DO

---
 src/pg/sql/45_observatory_mvt.sql | 221 ++++++++++++++++++++++++++++++
 1 file changed, 221 insertions(+)
 create mode 100644 src/pg/sql/45_observatory_mvt.sql

diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql
new file mode 100644
index 0000000..d78f750
--- /dev/null
+++ b/src/pg/sql/45_observatory_mvt.sql
@@ -0,0 +1,221 @@
+CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetTileBounds(z INTEGER, x INTEGER, y INTEGER)
+RETURNS NUMERIC[] AS $$
+import math
+def tile2lnglat(z, x, y):
+    n = 2.0 ** z
+
+    lon = x / n * 360.0 - 180.0
+    lat_rad = math.atan(math.sinh(math.pi * (1 - 2 * y / n)))
+    lat = - math.degrees(lat_rad)
+
+    return lon, lat
+
+lon0, lat0 = tile2lnglat(z, x, y)
+lon1, lat1 = tile2lnglat(z, x+1, y+1)
+
+return [lon0, lat0, lon1, lat1]
+$$ LANGUAGE plpythonu;
+
+DROP FUNCTION IF EXISTS cdb_observatory.OBS_GetMVT(z INTEGER, x INTEGER, y INTEGER, params JSONB);
+CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetMVT(z INTEGER, x INTEGER, y INTEGER,
+params JSON DEFAULT NULL,
+extent INTEGER DEFAULT 4096, buf INTEGER DEFAULT 256, clip_geom BOOLEAN DEFAULT True)
+RETURNS TABLE (mvt BYTEA)
+AS $$
+DECLARE
+  bounds NUMERIC[];
+  geom GEOMETRY;
+  ext BOX2D;
+  meta JSON;
+
+  procgeom_clauses TEXT;
+  val_clauses TEXT;
+  json_clause TEXT;
+BEGIN
+  bounds := cdb_observatory.OBS_GetTileBounds(z, x, y);
+  geom := ST_MakeEnvelope(bounds[1], bounds[2], bounds[3], bounds[4], 4326);
+  ext := ST_Extent(geom);
+  meta := cdb_observatory.obs_getmeta(geom, params::json, 1::integer, 1::integer, 1::integer);
+
+  /* Read metadata to generate clauses for query */
+  EXECUTE $query$
+    WITH _meta AS (SELECT
+      row_number() over () colid, *
+      FROM json_to_recordset($1)
+      AS x(id TEXT, numer_id TEXT, numer_aggregate TEXT, numer_colname TEXT,
+            numer_geomref_colname TEXT, numer_tablename TEXT, numer_type TEXT,
+            denom_id TEXT, denom_aggregate TEXT, denom_colname TEXT,
+            denom_geomref_colname TEXT, denom_tablename TEXT, denom_type TEXT,
+            denom_reltype TEXT, geom_id TEXT, geom_colname TEXT,
+            geom_geomref_colname TEXT, geom_tablename TEXT, geom_type TEXT,
+            numer_timespan TEXT, geom_timespan TEXT, normalization TEXT,
+            api_method TEXT, api_args JSON)
+    ),
+
+  -- Generate procgeom clauses.
+  -- These join the users' geoms to the relevant geometries for the
+  -- asked-for measures in the Observatory.
+  _procgeom_clauses AS (
+    SELECT
+      '_procgeoms_' || Coalesce(geom_tablename || '_' || geom_geomref_colname, api_method) || ' AS (' ||
+          'SELECT ' ||
+            'ST_AsMVTGeom(st_intersection(' || geom_tablename || '.' || geom_colname || ', _geoms.geom), $2, $3, $4, $5) AS mvtgeom, ' ||
+            geom_tablename || '.' || geom_geomref_colname || ' AS geomref, ' ||
+            'CASE WHEN ST_Within(_geoms.geom, ' || geom_tablename || '.' || geom_colname || ')
+                    THEN ST_Area(_geoms.geom) / Nullif(ST_Area(' || geom_tablename || '.' || geom_colname || '), 0)
+                    WHEN ST_Within(' || geom_tablename || '.' || geom_colname || ', _geoms.geom)
+                    THEN 1
+                    ELSE ST_Area(cdb_observatory.safe_intersection(_geoms.geom, ' || geom_tablename || '.' || geom_colname || ')) /
+                      Nullif(ST_Area(' || geom_tablename || '.' || geom_colname || '), 0)
+              END pct_obs' || '
+          FROM _geoms, observatory.' || geom_tablename || '
+          WHERE ST_Intersects(_geoms.geom, ' || geom_tablename || '.' || geom_colname || '))'
+      AS procgeom_clause
+    FROM _meta
+    GROUP BY api_method, geom_tablename, geom_geomref_colname, geom_colname
+  ),
+
+  -- Generate val clauses.
+  -- These perform interpolations or other necessary calculations to
+  -- provide values according to users geometries.
+  _val_clauses AS (
+    SELECT
+      '_vals_' || Coalesce(geom_tablename || '_' || geom_geomref_colname, api_method) || ' AS (
+        SELECT _procgeoms.geomref, _procgeoms.mvtgeom, ' ||
+          String_Agg('json_build_object(' || CASE
+            -- api-delivered values
+            WHEN api_method IS NOT NULL THEN
+            '''value'', ' ||
+              'ARRAY_AGG( ' ||
+                api_method || '.' || numer_colname || ')::' || numer_type || '[]'
+            -- numeric internal values
+            WHEN cdb_observatory.isnumeric(numer_type) THEN
+            '''value'', ' || CASE
+              -- denominated
+              WHEN LOWER(normalization) LIKE 'denom%'
+                THEN CASE
+                WHEN denom_tablename IS NULL THEN ' NULL '
+                -- denominated polygon interpolation
+                -- SUM (numer * (% OBS geom in user geom)) / SUM (denom * (% OBS geom in user geom))
+                ELSE
+                ' ROUND(CAST(SUM(' || numer_tablename || '.' || numer_colname || ' ' ||
+                ' * _procgeoms.pct_obs ' ||
+                ' ) / NULLIF(SUM(' || denom_tablename || '.' || denom_colname || ' ' ||
+                '            * _procgeoms.pct_obs), 0) AS NUMERIC), 4) '
+                END
+              -- areaNormalized
+              WHEN LOWER(normalization) LIKE 'area%'
+                THEN 
+                -- areaNormalized polygon interpolation
+                -- SUM (numer * (% OBS geom in user geom)) / area of big geom
+                ' ROUND(CAST(SUM(' || numer_tablename || '.' || numer_colname || ' ' ||
+                ' * _procgeoms.pct_obs' ||
+                ' ) / (Nullif(ST_Area(cdb_observatory.FIRST(_procgeoms.geom)::Geography), 0) / 1000000) AS NUMERIC), 4) '
+              -- median/average measures with universe
+              WHEN LOWER(numer_aggregate) IN ('median', 'average') AND
+                  denom_reltype ILIKE 'universe' AND LOWER(normalization) LIKE 'pre%'
+                THEN
+                -- predenominated polygon interpolation weighted by universe
+                -- SUM (numer * denom * (% user geom in OBS geom)) / SUM (denom * (% user geom in OBS geom))
+                --     (10 * 1000 * 1) / (1000 * 1) = 10
+                --     (10 * 1000 * 1 + 50 * 10 * 1) / (1000 + 10) = 10500 / 10000 = 10.5
+                ' ROUND(CAST(SUM(' || numer_tablename || '.' || numer_colname ||
+                ' * ' || denom_tablename || '.' || denom_colname ||
+                ' * _procgeoms.pct_obs ' ||
+                ' ) / Nullif(SUM(' || denom_tablename || '.' || denom_colname ||
+                ' * _procgeoms.pct_obs ' || '), 0)AS NUMERIC), 4) '
+              -- prenormalized for summable measures. point or summable only!
+              WHEN numer_aggregate ILIKE 'sum' AND LOWER(normalization) LIKE 'pre%'
+                THEN
+                -- predenominated polygon interpolation
+                -- SUM (numer * (% user geom in OBS geom))
+                ' ROUND(CAST(SUM(' || numer_tablename || '.' || numer_colname || ' ' ||
+                ' * _procgeoms.pct_obs) AS NUMERIC), 4) '
+              -- Everything else. Point only!
+              ELSE
+                ' cdb_observatory._OBS_RaiseNotice(''Cannot perform calculation over polygon for ' ||
+                    numer_id || '/' || coalesce(denom_id, '') || '/' || geom_id || '/' || numer_timespan || ''')::Numeric '
+              END || '::' || numer_type
+
+            -- categorical/text
+            WHEN LOWER(numer_type) LIKE 'text' THEN
+              '''value'', ' || 'MODE() WITHIN GROUP (ORDER BY ' || numer_tablename || '.' || numer_colname || ') '
+              -- geometry
+            WHEN numer_id IS NULL THEN
+              '''geomref'', _procgeoms.geomref, ' ||
+              '''value'', ' || 'cdb_observatory.FIRST(_procgeoms.geom)::TEXT'
+              -- code below will return the intersection of the user's geom and the
+              -- OBS geom
+              --'''value'', ' || 'ST_Union(cdb_observatory.safe_intersection(_geoms.geom, ' || geom_tablename ||
+              --    '.' || geom_colname || '))::TEXT'
+            ELSE ''
+            END
+          || ') val_' || colid, ', ')
+        || '
+        FROM _procgeoms_' || Coalesce(geom_tablename || '_' || geom_geomref_colname, api_method) || ' _procgeoms ' ||
+          Coalesce(String_Agg(DISTINCT
+              Coalesce('LEFT JOIN observatory.' || numer_tablename || ' ON _procgeoms.geomref = observatory.' || numer_tablename || '.' || numer_geomref_colname,
+                ', LATERAL (SELECT * FROM cdb_observatory.' || api_method || '(_procgeoms.geom' || Coalesce(', ' ||
+                    (SELECT STRING_AGG(REPLACE(val::text, '"', ''''), ', ')
+                      FROM (SELECT JSON_Array_Elements(api_args) as val) as vals),
+                    '') || ')) AS ' || api_method)
+          , ' '), '') ||
+                                E'\n GROUP BY _procgeoms.geomref, _procgeoms.mvtgeom
+                                    ORDER BY _procgeoms.geomref'
+      || ')'
+      AS val_clause,
+      '_vals_' || Coalesce(geom_tablename || '_' || geom_geomref_colname, api_method) AS cte_name
+    FROM _meta
+    GROUP BY geom_tablename, geom_geomref_colname, geom_colname, api_method
+  ),
+
+  -- Generate clauses necessary to join together val_clauses
+  _val_joins AS (
+    SELECT String_Agg(a.cte_name || '.geomref = ' || b.cte_name || '.geomref ', ' AND ') val_joins
+    FROM _val_clauses a, _val_clauses b
+    WHERE a.cte_name != b.cte_name
+      AND a.cte_name < b.cte_name
+  ),
+
+  -- Generate JSON clause.  This puts together vals from val_clauses
+  _json_clause AS (SELECT
+    'SELECT ST_AsMVT(q, ''data'', $3) FROM (SELECT ' || cdb_observatory.FIRST(cte_name) || '.geomref::TEXT id, '
+        || cdb_observatory.FIRST(cte_name) || '.mvtgeom geom, 
+        to_JSONB(ARRAY[' || (SELECT String_Agg('val_' || colid, ', ') FROM _meta) || '])
+      FROM ' || String_Agg(cte_name, ', ') ||
+    Coalesce(' WHERE ' || val_joins, ') q')
+    AS json_clause
+    FROM _val_clauses, _val_joins
+    GROUP BY val_joins
+  )
+
+  SELECT (SELECT String_Agg(procgeom_clause, E',\n     ') FROM _procgeom_clauses),
+          (SELECT String_Agg(val_clause, E',\n     ') FROM _val_clauses),
+          json_clause
+    FROM _json_clause
+  $query$ INTO
+    procgeom_clauses,
+    val_clauses,
+    json_clause
+  USING meta;
+
+  /* Execute query */
+  RETURN QUERY EXECUTE format($query$
+    WITH _geoms AS (%s),
+    -- procgeom_clauses
+    %s,
+
+    -- val_clauses
+    %s
+
+    -- json_clause
+    %s
+  $query$, 'SELECT $1::geometry as geom',
+            String_Agg(procgeom_clauses, E',\n       '),
+            String_Agg(val_clauses, E',\n       '),
+            json_clause)
+  USING geom, ext, extent, buf, clip_geom;
+  RETURN;
+
+END
+$$ LANGUAGE plpgsql;

From fafd4a364e2a66347282fe60f08adc242c9e4e3c Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Tue, 8 May 2018 15:52:30 +0200
Subject: [PATCH 02/45] Fixed naming issue

---
 src/pg/sql/45_observatory_mvt.sql | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql
index d78f750..789cf8a 100644
--- a/src/pg/sql/45_observatory_mvt.sql
+++ b/src/pg/sql/45_observatory_mvt.sql
@@ -110,7 +110,7 @@ BEGIN
                 -- SUM (numer * (% OBS geom in user geom)) / area of big geom
                 ' ROUND(CAST(SUM(' || numer_tablename || '.' || numer_colname || ' ' ||
                 ' * _procgeoms.pct_obs' ||
-                ' ) / (Nullif(ST_Area(cdb_observatory.FIRST(_procgeoms.geom)::Geography), 0) / 1000000) AS NUMERIC), 4) '
+                ' ) / (Nullif(ST_Area(cdb_observatory.FIRST(_procgeoms.mvtgeom)::Geography), 0) / 1000000) AS NUMERIC), 4) '
               -- median/average measures with universe
               WHEN LOWER(numer_aggregate) IN ('median', 'average') AND
                   denom_reltype ILIKE 'universe' AND LOWER(normalization) LIKE 'pre%'
@@ -143,7 +143,7 @@ BEGIN
               -- geometry
             WHEN numer_id IS NULL THEN
               '''geomref'', _procgeoms.geomref, ' ||
-              '''value'', ' || 'cdb_observatory.FIRST(_procgeoms.geom)::TEXT'
+              '''value'', ' || 'cdb_observatory.FIRST(_procgeoms.mvtgeom)::TEXT'
               -- code below will return the intersection of the user's geom and the
               -- OBS geom
               --'''value'', ' || 'ST_Union(cdb_observatory.safe_intersection(_geoms.geom, ' || geom_tablename ||
@@ -155,7 +155,7 @@ BEGIN
         FROM _procgeoms_' || Coalesce(geom_tablename || '_' || geom_geomref_colname, api_method) || ' _procgeoms ' ||
           Coalesce(String_Agg(DISTINCT
               Coalesce('LEFT JOIN observatory.' || numer_tablename || ' ON _procgeoms.geomref = observatory.' || numer_tablename || '.' || numer_geomref_colname,
-                ', LATERAL (SELECT * FROM cdb_observatory.' || api_method || '(_procgeoms.geom' || Coalesce(', ' ||
+                ', LATERAL (SELECT * FROM cdb_observatory.' || api_method || '(_procgeoms.mvtgeom' || Coalesce(', ' ||
                     (SELECT STRING_AGG(REPLACE(val::text, '"', ''''), ', ')
                       FROM (SELECT JSON_Array_Elements(api_args) as val) as vals),
                     '') || ')) AS ' || api_method)

From 3fba8a309109951db524be4298aa159cab0d882a Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Tue, 8 May 2018 16:54:53 +0200
Subject: [PATCH 03/45] Inverting y coordinate

---
 src/pg/sql/45_observatory_mvt.sql | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql
index 789cf8a..da678e5 100644
--- a/src/pg/sql/45_observatory_mvt.sql
+++ b/src/pg/sql/45_observatory_mvt.sql
@@ -3,6 +3,7 @@ RETURNS NUMERIC[] AS $$
 import math
 def tile2lnglat(z, x, y):
     n = 2.0 ** z
+    y = (1 << z) - y - 1
 
     lon = x / n * 360.0 - 180.0
     lat_rad = math.atan(math.sinh(math.pi * (1 - 2 * y / n)))

From 5f9f77ded14d300120f2980697064d8ec9846b2e Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Tue, 8 May 2018 17:50:01 +0200
Subject: [PATCH 04/45] Fixed errors

---
 src/pg/sql/45_observatory_mvt.sql | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql
index da678e5..5c3bd82 100644
--- a/src/pg/sql/45_observatory_mvt.sql
+++ b/src/pg/sql/45_observatory_mvt.sql
@@ -35,7 +35,6 @@ DECLARE
 BEGIN
   bounds := cdb_observatory.OBS_GetTileBounds(z, x, y);
   geom := ST_MakeEnvelope(bounds[1], bounds[2], bounds[3], bounds[4], 4326);
-  ext := ST_Extent(geom);
   meta := cdb_observatory.obs_getmeta(geom, params::json, 1::integer, 1::integer, 1::integer);
 
   /* Read metadata to generate clauses for query */
@@ -60,7 +59,7 @@ BEGIN
     SELECT
       '_procgeoms_' || Coalesce(geom_tablename || '_' || geom_geomref_colname, api_method) || ' AS (' ||
           'SELECT ' ||
-            'ST_AsMVTGeom(st_intersection(' || geom_tablename || '.' || geom_colname || ', _geoms.geom), $2, $3, $4, $5) AS mvtgeom, ' ||
+            'ST_AsMVTGeom(st_intersection(' || geom_tablename || '.' || geom_colname || ', _geoms.geom), ST_MakeBox2D(ST_Point(0, 0), ST_Point($2, $2)), $2, $3, $4) AS mvtgeom, ' ||
             geom_tablename || '.' || geom_geomref_colname || ' AS geomref, ' ||
             'CASE WHEN ST_Within(_geoms.geom, ' || geom_tablename || '.' || geom_colname || ')
                     THEN ST_Area(_geoms.geom) / Nullif(ST_Area(' || geom_tablename || '.' || geom_colname || '), 0)
@@ -180,7 +179,8 @@ BEGIN
 
   -- Generate JSON clause.  This puts together vals from val_clauses
   _json_clause AS (SELECT
-    'SELECT ST_AsMVT(q, ''data'', $3) FROM (SELECT ' || cdb_observatory.FIRST(cte_name) || '.geomref::TEXT id, '
+    'SELECT ST_AsMVT(q, ''data'', $2) FROM (' ||
+    'SELECT ' || cdb_observatory.FIRST(cte_name) || '.geomref::TEXT id, '
         || cdb_observatory.FIRST(cte_name) || '.mvtgeom geom, 
         to_JSONB(ARRAY[' || (SELECT String_Agg('val_' || colid, ', ') FROM _meta) || '])
       FROM ' || String_Agg(cte_name, ', ') ||
@@ -215,7 +215,7 @@ BEGIN
             String_Agg(procgeom_clauses, E',\n       '),
             String_Agg(val_clauses, E',\n       '),
             json_clause)
-  USING geom, ext, extent, buf, clip_geom;
+  USING geom, extent, buf, clip_geom;
   RETURN;
 
 END

From 50008ea149c4da33f57335ee25a65783410500e4 Mon Sep 17 00:00:00 2001
From: Mario de Frutos <ethervoid@gmail.com>
Date: Wed, 9 May 2018 09:39:20 +0200
Subject: [PATCH 05/45] WIP

---
 src/pg/sql/45_observatory_mvt.sql | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql
index 5c3bd82..cd54b64 100644
--- a/src/pg/sql/45_observatory_mvt.sql
+++ b/src/pg/sql/45_observatory_mvt.sql
@@ -57,7 +57,7 @@ BEGIN
   -- asked-for measures in the Observatory.
   _procgeom_clauses AS (
     SELECT
-      '_procgeoms_' || Coalesce(geom_tablename || '_' || geom_geomref_colname, api_method) || ' AS (' ||
+      '_procgeoms_' || Coalesce(left(geom_tablename,10) || '_' || geom_geomref_colname, api_method) || ' AS (' ||
           'SELECT ' ||
             'ST_AsMVTGeom(st_intersection(' || geom_tablename || '.' || geom_colname || ', _geoms.geom), ST_MakeBox2D(ST_Point(0, 0), ST_Point($2, $2)), $2, $3, $4) AS mvtgeom, ' ||
             geom_tablename || '.' || geom_geomref_colname || ' AS geomref, ' ||
@@ -80,7 +80,7 @@ BEGIN
   -- provide values according to users geometries.
   _val_clauses AS (
     SELECT
-      '_vals_' || Coalesce(geom_tablename || '_' || geom_geomref_colname, api_method) || ' AS (
+      '_vals_' || Coalesce(left(geom_tablename,10) || '_' || geom_geomref_colname, api_method) || ' AS (
         SELECT _procgeoms.geomref, _procgeoms.mvtgeom, ' ||
           String_Agg('json_build_object(' || CASE
             -- api-delivered values
@@ -105,7 +105,7 @@ BEGIN
                 END
               -- areaNormalized
               WHEN LOWER(normalization) LIKE 'area%'
-                THEN 
+                THEN
                 -- areaNormalized polygon interpolation
                 -- SUM (numer * (% OBS geom in user geom)) / area of big geom
                 ' ROUND(CAST(SUM(' || numer_tablename || '.' || numer_colname || ' ' ||
@@ -152,7 +152,7 @@ BEGIN
             END
           || ') val_' || colid, ', ')
         || '
-        FROM _procgeoms_' || Coalesce(geom_tablename || '_' || geom_geomref_colname, api_method) || ' _procgeoms ' ||
+        FROM _procgeoms_' || Coalesce(left(geom_tablename,10) || '_' || geom_geomref_colname, api_method) || ' _procgeoms ' ||
           Coalesce(String_Agg(DISTINCT
               Coalesce('LEFT JOIN observatory.' || numer_tablename || ' ON _procgeoms.geomref = observatory.' || numer_tablename || '.' || numer_geomref_colname,
                 ', LATERAL (SELECT * FROM cdb_observatory.' || api_method || '(_procgeoms.mvtgeom' || Coalesce(', ' ||
@@ -164,7 +164,7 @@ BEGIN
                                     ORDER BY _procgeoms.geomref'
       || ')'
       AS val_clause,
-      '_vals_' || Coalesce(geom_tablename || '_' || geom_geomref_colname, api_method) AS cte_name
+      '_vals_' || Coalesce(left(geom_tablename, 10) || '_' || geom_geomref_colname, api_method) AS cte_name
     FROM _meta
     GROUP BY geom_tablename, geom_geomref_colname, geom_colname, api_method
   ),
@@ -180,9 +180,8 @@ BEGIN
   -- Generate JSON clause.  This puts together vals from val_clauses
   _json_clause AS (SELECT
     'SELECT ST_AsMVT(q, ''data'', $2) FROM (' ||
-    'SELECT ' || cdb_observatory.FIRST(cte_name) || '.geomref::TEXT id, '
-        || cdb_observatory.FIRST(cte_name) || '.mvtgeom geom, 
-        to_JSONB(ARRAY[' || (SELECT String_Agg('val_' || colid, ', ') FROM _meta) || '])
+    'SELECT ' || cdb_observatory.FIRST(cte_name) || '.geomref::TEXT id, ' || (SELECT String_Agg('val_' || colid || '->>''value'' as val_' || colid, ', ') FROM _meta) || ', '
+        || cdb_observatory.FIRST(cte_name) || '.mvtgeom geom
       FROM ' || String_Agg(cte_name, ', ') ||
     Coalesce(' WHERE ' || val_joins, ') q')
     AS json_clause

From d309a25379f9bf6a59edff1a242292ed5cf1cfa4 Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Wed, 9 May 2018 10:14:59 +0200
Subject: [PATCH 06/45] Fixed MVT data

---
 src/pg/sql/45_observatory_mvt.sql | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql
index cd54b64..881de4d 100644
--- a/src/pg/sql/45_observatory_mvt.sql
+++ b/src/pg/sql/45_observatory_mvt.sql
@@ -180,8 +180,8 @@ BEGIN
   -- Generate JSON clause.  This puts together vals from val_clauses
   _json_clause AS (SELECT
     'SELECT ST_AsMVT(q, ''data'', $2) FROM (' ||
-    'SELECT ' || cdb_observatory.FIRST(cte_name) || '.geomref::TEXT id, ' || (SELECT String_Agg('val_' || colid || '->>''value'' as val_' || colid, ', ') FROM _meta) || ', '
-        || cdb_observatory.FIRST(cte_name) || '.mvtgeom geom
+    'SELECT ' || cdb_observatory.FIRST(cte_name) || '.mvtgeom geom,'
+              || (SELECT String_Agg('val_' || colid || '->>''value'' as val_' || colid, ', ') FROM _meta) || ', 
       FROM ' || String_Agg(cte_name, ', ') ||
     Coalesce(' WHERE ' || val_joins, ') q')
     AS json_clause

From 93790e9f0df808f46108a252d66218e29ea6aad5 Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Wed, 9 May 2018 10:16:03 +0200
Subject: [PATCH 07/45] Fixed MVT data

---
 src/pg/sql/45_observatory_mvt.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql
index 881de4d..bead56d 100644
--- a/src/pg/sql/45_observatory_mvt.sql
+++ b/src/pg/sql/45_observatory_mvt.sql
@@ -181,7 +181,7 @@ BEGIN
   _json_clause AS (SELECT
     'SELECT ST_AsMVT(q, ''data'', $2) FROM (' ||
     'SELECT ' || cdb_observatory.FIRST(cte_name) || '.mvtgeom geom,'
-              || (SELECT String_Agg('val_' || colid || '->>''value'' as val_' || colid, ', ') FROM _meta) || ', 
+              || (SELECT String_Agg('val_' || colid || '->>''value'' as val_' || colid, ', ') FROM _meta) || ' 
       FROM ' || String_Agg(cte_name, ', ') ||
     Coalesce(' WHERE ' || val_joins, ') q')
     AS json_clause

From 4b128ea3e8175bdea96fec9dc52265d7d3afb9ed Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Wed, 9 May 2018 11:20:17 +0200
Subject: [PATCH 08/45] Fixing bounds

---
 src/pg/sql/45_observatory_mvt.sql | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql
index bead56d..3c17e43 100644
--- a/src/pg/sql/45_observatory_mvt.sql
+++ b/src/pg/sql/45_observatory_mvt.sql
@@ -35,6 +35,7 @@ DECLARE
 BEGIN
   bounds := cdb_observatory.OBS_GetTileBounds(z, x, y);
   geom := ST_MakeEnvelope(bounds[1], bounds[2], bounds[3], bounds[4], 4326);
+  ext := ST_MakeBox2D(ST_Point(bounds[1], bounds[2]), ST_Point(bounds[3], bounds[4]));
   meta := cdb_observatory.obs_getmeta(geom, params::json, 1::integer, 1::integer, 1::integer);
 
   /* Read metadata to generate clauses for query */
@@ -57,9 +58,9 @@ BEGIN
   -- asked-for measures in the Observatory.
   _procgeom_clauses AS (
     SELECT
-      '_procgeoms_' || Coalesce(left(geom_tablename,10) || '_' || geom_geomref_colname, api_method) || ' AS (' ||
+      '_procgeoms_' || Coalesce(left(geom_tablename,40) || '_' || geom_geomref_colname, api_method) || ' AS (' ||
           'SELECT ' ||
-            'ST_AsMVTGeom(st_intersection(' || geom_tablename || '.' || geom_colname || ', _geoms.geom), ST_MakeBox2D(ST_Point(0, 0), ST_Point($2, $2)), $2, $3, $4) AS mvtgeom, ' ||
+            'ST_AsMVTGeom(st_intersection(' || geom_tablename || '.' || geom_colname || ', _geoms.geom), $2, $3, $4, $5) AS mvtgeom, ' ||
             geom_tablename || '.' || geom_geomref_colname || ' AS geomref, ' ||
             'CASE WHEN ST_Within(_geoms.geom, ' || geom_tablename || '.' || geom_colname || ')
                     THEN ST_Area(_geoms.geom) / Nullif(ST_Area(' || geom_tablename || '.' || geom_colname || '), 0)
@@ -80,7 +81,7 @@ BEGIN
   -- provide values according to users geometries.
   _val_clauses AS (
     SELECT
-      '_vals_' || Coalesce(left(geom_tablename,10) || '_' || geom_geomref_colname, api_method) || ' AS (
+      '_vals_' || Coalesce(left(geom_tablename,40) || '_' || geom_geomref_colname, api_method) || ' AS (
         SELECT _procgeoms.geomref, _procgeoms.mvtgeom, ' ||
           String_Agg('json_build_object(' || CASE
             -- api-delivered values
@@ -152,7 +153,7 @@ BEGIN
             END
           || ') val_' || colid, ', ')
         || '
-        FROM _procgeoms_' || Coalesce(left(geom_tablename,10) || '_' || geom_geomref_colname, api_method) || ' _procgeoms ' ||
+        FROM _procgeoms_' || Coalesce(left(geom_tablename,40) || '_' || geom_geomref_colname, api_method) || ' _procgeoms ' ||
           Coalesce(String_Agg(DISTINCT
               Coalesce('LEFT JOIN observatory.' || numer_tablename || ' ON _procgeoms.geomref = observatory.' || numer_tablename || '.' || numer_geomref_colname,
                 ', LATERAL (SELECT * FROM cdb_observatory.' || api_method || '(_procgeoms.mvtgeom' || Coalesce(', ' ||
@@ -164,7 +165,7 @@ BEGIN
                                     ORDER BY _procgeoms.geomref'
       || ')'
       AS val_clause,
-      '_vals_' || Coalesce(left(geom_tablename, 10) || '_' || geom_geomref_colname, api_method) AS cte_name
+      '_vals_' || Coalesce(left(geom_tablename, 40) || '_' || geom_geomref_colname, api_method) AS cte_name
     FROM _meta
     GROUP BY geom_tablename, geom_geomref_colname, geom_colname, api_method
   ),
@@ -179,9 +180,9 @@ BEGIN
 
   -- Generate JSON clause.  This puts together vals from val_clauses
   _json_clause AS (SELECT
-    'SELECT ST_AsMVT(q, ''data'', $2) FROM (' ||
-    'SELECT ' || cdb_observatory.FIRST(cte_name) || '.mvtgeom geom,'
-              || (SELECT String_Agg('val_' || colid || '->>''value'' as val_' || colid, ', ') FROM _meta) || ' 
+    'SELECT ST_AsMVT(q, ''data'', $3) FROM (' ||
+    'SELECT ' || cdb_observatory.FIRST(cte_name) || '.mvtgeom geom,
+        to_JSONB(ARRAY[' || (SELECT String_Agg('val_' || colid, ', ') FROM _meta) || '])
       FROM ' || String_Agg(cte_name, ', ') ||
     Coalesce(' WHERE ' || val_joins, ') q')
     AS json_clause
@@ -214,7 +215,7 @@ BEGIN
             String_Agg(procgeom_clauses, E',\n       '),
             String_Agg(val_clauses, E',\n       '),
             json_clause)
-  USING geom, extent, buf, clip_geom;
+  USING geom, ext, extent, buf, clip_geom;
   RETURN;
 
 END

From db637f6e52654e5b61f09c55181831b212420e11 Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Wed, 9 May 2018 11:36:24 +0200
Subject: [PATCH 09/45] Returning simple JSON

---
 src/pg/sql/45_observatory_mvt.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql
index 3c17e43..29558ec 100644
--- a/src/pg/sql/45_observatory_mvt.sql
+++ b/src/pg/sql/45_observatory_mvt.sql
@@ -182,7 +182,7 @@ BEGIN
   _json_clause AS (SELECT
     'SELECT ST_AsMVT(q, ''data'', $3) FROM (' ||
     'SELECT ' || cdb_observatory.FIRST(cte_name) || '.mvtgeom geom,
-        to_JSONB(ARRAY[' || (SELECT String_Agg('val_' || colid, ', ') FROM _meta) || '])
+        to_JSONB(' || (SELECT String_Agg('val_' || colid, ', ') FROM _meta) || ')
       FROM ' || String_Agg(cte_name, ', ') ||
     Coalesce(' WHERE ' || val_joins, ') q')
     AS json_clause

From dcf10edd9ef8694e36b84fb8cca22d7545935ee4 Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Wed, 9 May 2018 12:29:42 +0200
Subject: [PATCH 10/45] Added tolerance for the areas

---
 src/pg/sql/45_observatory_mvt.sql | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql
index 29558ec..f4c1560 100644
--- a/src/pg/sql/45_observatory_mvt.sql
+++ b/src/pg/sql/45_observatory_mvt.sql
@@ -24,6 +24,7 @@ extent INTEGER DEFAULT 4096, buf INTEGER DEFAULT 256, clip_geom BOOLEAN DEFAULT
 RETURNS TABLE (mvt BYTEA)
 AS $$
 DECLARE
+  tolerance NUMERIC DEFAULT 100;
   bounds NUMERIC[];
   geom GEOMETRY;
   ext BOX2D;
@@ -70,7 +71,9 @@ BEGIN
                       Nullif(ST_Area(' || geom_tablename || '.' || geom_colname || '), 0)
               END pct_obs' || '
           FROM _geoms, observatory.' || geom_tablename || '
-          WHERE ST_Intersects(_geoms.geom, ' || geom_tablename || '.' || geom_colname || '))'
+          WHERE ST_Intersects(_geoms.geom, ' || geom_tablename || '.' || geom_colname || ')'
+          || ' AND ST_Area(st_intersection(' || geom_tablename || '.' || geom_colname || ', _geoms.geom)) > ST_Area($1) / $6 '
+          || ')'
       AS procgeom_clause
     FROM _meta
     GROUP BY api_method, geom_tablename, geom_geomref_colname, geom_colname
@@ -215,7 +218,7 @@ BEGIN
             String_Agg(procgeom_clauses, E',\n       '),
             String_Agg(val_clauses, E',\n       '),
             json_clause)
-  USING geom, ext, extent, buf, clip_geom;
+  USING geom, ext, extent, buf, clip_geom, tolerance;
   RETURN;
 
 END

From b8d00c49cb25de5a1f02c4265f33e3557dd44f24 Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Wed, 9 May 2018 12:46:00 +0200
Subject: [PATCH 11/45] Fixed areas

---
 src/pg/sql/45_observatory_mvt.sql | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql
index f4c1560..586b305 100644
--- a/src/pg/sql/45_observatory_mvt.sql
+++ b/src/pg/sql/45_observatory_mvt.sql
@@ -61,6 +61,7 @@ BEGIN
     SELECT
       '_procgeoms_' || Coalesce(left(geom_tablename,40) || '_' || geom_geomref_colname, api_method) || ' AS (' ||
           'SELECT ' ||
+            'st_intersection(' || geom_tablename || '.' || geom_colname || ', _geoms.geom) AS geom, ' ||
             'ST_AsMVTGeom(st_intersection(' || geom_tablename || '.' || geom_colname || ', _geoms.geom), $2, $3, $4, $5) AS mvtgeom, ' ||
             geom_tablename || '.' || geom_geomref_colname || ' AS geomref, ' ||
             'CASE WHEN ST_Within(_geoms.geom, ' || geom_tablename || '.' || geom_colname || ')
@@ -72,7 +73,7 @@ BEGIN
               END pct_obs' || '
           FROM _geoms, observatory.' || geom_tablename || '
           WHERE ST_Intersects(_geoms.geom, ' || geom_tablename || '.' || geom_colname || ')'
-          || ' AND ST_Area(st_intersection(' || geom_tablename || '.' || geom_colname || ', _geoms.geom)) > ST_Area($1) / $6 '
+          --|| ' AND ST_Area(st_intersection(' || geom_tablename || '.' |QUERY| geom_colname || ', _geoms.geom)) > ST_Area($1) / $6 '
           || ')'
       AS procgeom_clause
     FROM _meta
@@ -114,7 +115,7 @@ BEGIN
                 -- SUM (numer * (% OBS geom in user geom)) / area of big geom
                 ' ROUND(CAST(SUM(' || numer_tablename || '.' || numer_colname || ' ' ||
                 ' * _procgeoms.pct_obs' ||
-                ' ) / (Nullif(ST_Area(cdb_observatory.FIRST(_procgeoms.mvtgeom)::Geography), 0) / 1000000) AS NUMERIC), 4) '
+                ' ) / (Nullif(ST_Area(cdb_observatory.FIRST(_procgeoms.geom)::Geography), 0) / 1000000) AS NUMERIC), 4) '
               -- median/average measures with universe
               WHEN LOWER(numer_aggregate) IN ('median', 'average') AND
                   denom_reltype ILIKE 'universe' AND LOWER(normalization) LIKE 'pre%'

From 5c4bb80f5775a9c1ee0658dee8bd17c1afc0305f Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Wed, 9 May 2018 13:18:19 +0200
Subject: [PATCH 12/45] Fixed area/perimeter relation

---
 src/pg/sql/45_observatory_mvt.sql | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql
index 586b305..32fcf63 100644
--- a/src/pg/sql/45_observatory_mvt.sql
+++ b/src/pg/sql/45_observatory_mvt.sql
@@ -188,7 +188,8 @@ BEGIN
     'SELECT ' || cdb_observatory.FIRST(cte_name) || '.mvtgeom geom,
         to_JSONB(' || (SELECT String_Agg('val_' || colid, ', ') FROM _meta) || ')
       FROM ' || String_Agg(cte_name, ', ') ||
-    Coalesce(' WHERE ' || val_joins, ') q')
+    ' WHERE ST_Area(' || cdb_observatory.FIRST(cte_name) || '.mvtgeom) / ST_Perimeter(' || cdb_observatory.FIRST(cte_name) || '.mvtgeom)' || ' > 0.001' ||
+    Coalesce(' AND ' || val_joins, ') q')
     AS json_clause
     FROM _val_clauses, _val_joins
     GROUP BY val_joins

From 92155b6b33d44a58e249dffad717037efec74178 Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Wed, 9 May 2018 13:27:03 +0200
Subject: [PATCH 13/45] Fixed area > 0

---
 src/pg/sql/45_observatory_mvt.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql
index 32fcf63..9ba25fe 100644
--- a/src/pg/sql/45_observatory_mvt.sql
+++ b/src/pg/sql/45_observatory_mvt.sql
@@ -188,7 +188,7 @@ BEGIN
     'SELECT ' || cdb_observatory.FIRST(cte_name) || '.mvtgeom geom,
         to_JSONB(' || (SELECT String_Agg('val_' || colid, ', ') FROM _meta) || ')
       FROM ' || String_Agg(cte_name, ', ') ||
-    ' WHERE ST_Area(' || cdb_observatory.FIRST(cte_name) || '.mvtgeom) / ST_Perimeter(' || cdb_observatory.FIRST(cte_name) || '.mvtgeom)' || ' > 0.001' ||
+    ' WHERE ST_Area(' || cdb_observatory.FIRST(cte_name) || '.mvtgeom) > 0' ||
     Coalesce(' AND ' || val_joins, ') q')
     AS json_clause
     FROM _val_clauses, _val_joins

From 78a7ccbe4b67a07cb7774d1be446cf2e9d5de26f Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Wed, 9 May 2018 13:40:26 +0200
Subject: [PATCH 14/45] Fix y coordinate

---
 src/pg/sql/45_observatory_mvt.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql
index 9ba25fe..a81cca7 100644
--- a/src/pg/sql/45_observatory_mvt.sql
+++ b/src/pg/sql/45_observatory_mvt.sql
@@ -12,7 +12,7 @@ def tile2lnglat(z, x, y):
     return lon, lat
 
 lon0, lat0 = tile2lnglat(z, x, y)
-lon1, lat1 = tile2lnglat(z, x+1, y+1)
+lon1, lat1 = tile2lnglat(z, x+1, y-1)
 
 return [lon0, lat0, lon1, lat1]
 $$ LANGUAGE plpythonu;

From d050b968a426156d116b61f8b4ad1702d5e57ed7 Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Wed, 9 May 2018 14:47:00 +0200
Subject: [PATCH 15/45] Fixed field naming

---
 src/pg/sql/45_observatory_mvt.sql | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql
index a81cca7..85190af 100644
--- a/src/pg/sql/45_observatory_mvt.sql
+++ b/src/pg/sql/45_observatory_mvt.sql
@@ -90,12 +90,12 @@ BEGIN
           String_Agg('json_build_object(' || CASE
             -- api-delivered values
             WHEN api_method IS NOT NULL THEN
-            '''value'', ' ||
+            '''' || numer_colname || ''', ' ||
               'ARRAY_AGG( ' ||
                 api_method || '.' || numer_colname || ')::' || numer_type || '[]'
             -- numeric internal values
             WHEN cdb_observatory.isnumeric(numer_type) THEN
-            '''value'', ' || CASE
+            '''' || numer_colname || ''', ' || CASE
               -- denominated
               WHEN LOWER(normalization) LIKE 'denom%'
                 THEN CASE
@@ -144,11 +144,11 @@ BEGIN
 
             -- categorical/text
             WHEN LOWER(numer_type) LIKE 'text' THEN
-              '''value'', ' || 'MODE() WITHIN GROUP (ORDER BY ' || numer_tablename || '.' || numer_colname || ') '
+              '''' || numer_colname || ''', ' || 'MODE() WITHIN GROUP (ORDER BY ' || numer_tablename || '.' || numer_colname || ') '
               -- geometry
             WHEN numer_id IS NULL THEN
               '''geomref'', _procgeoms.geomref, ' ||
-              '''value'', ' || 'cdb_observatory.FIRST(_procgeoms.mvtgeom)::TEXT'
+              '''' || numer_colname || ''', ' || 'cdb_observatory.FIRST(_procgeoms.mvtgeom)::TEXT'
               -- code below will return the intersection of the user's geom and the
               -- OBS geom
               --'''value'', ' || 'ST_Union(cdb_observatory.safe_intersection(_geoms.geom, ' || geom_tablename ||

From 9f62a8c12733a8f1e029497d66e2f8b57844847d Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Fri, 11 May 2018 15:30:37 +0200
Subject: [PATCH 16/45] Fixing errors. Request multiple measurements is now
 allowed

---
 src/pg/sql/45_observatory_mvt.sql | 20 ++++++--------------
 1 file changed, 6 insertions(+), 14 deletions(-)

diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql
index 85190af..d8d33a5 100644
--- a/src/pg/sql/45_observatory_mvt.sql
+++ b/src/pg/sql/45_observatory_mvt.sql
@@ -24,7 +24,6 @@ extent INTEGER DEFAULT 4096, buf INTEGER DEFAULT 256, clip_geom BOOLEAN DEFAULT
 RETURNS TABLE (mvt BYTEA)
 AS $$
 DECLARE
-  tolerance NUMERIC DEFAULT 100;
   bounds NUMERIC[];
   geom GEOMETRY;
   ext BOX2D;
@@ -73,7 +72,6 @@ BEGIN
               END pct_obs' || '
           FROM _geoms, observatory.' || geom_tablename || '
           WHERE ST_Intersects(_geoms.geom, ' || geom_tablename || '.' || geom_colname || ')'
-          --|| ' AND ST_Area(st_intersection(' || geom_tablename || '.' |QUERY| geom_colname || ', _geoms.geom)) > ST_Area($1) / $6 '
           || ')'
       AS procgeom_clause
     FROM _meta
@@ -101,7 +99,6 @@ BEGIN
                 THEN CASE
                 WHEN denom_tablename IS NULL THEN ' NULL '
                 -- denominated polygon interpolation
-                -- SUM (numer * (% OBS geom in user geom)) / SUM (denom * (% OBS geom in user geom))
                 ELSE
                 ' ROUND(CAST(SUM(' || numer_tablename || '.' || numer_colname || ' ' ||
                 ' * _procgeoms.pct_obs ' ||
@@ -112,7 +109,6 @@ BEGIN
               WHEN LOWER(normalization) LIKE 'area%'
                 THEN
                 -- areaNormalized polygon interpolation
-                -- SUM (numer * (% OBS geom in user geom)) / area of big geom
                 ' ROUND(CAST(SUM(' || numer_tablename || '.' || numer_colname || ' ' ||
                 ' * _procgeoms.pct_obs' ||
                 ' ) / (Nullif(ST_Area(cdb_observatory.FIRST(_procgeoms.geom)::Geography), 0) / 1000000) AS NUMERIC), 4) '
@@ -121,9 +117,6 @@ BEGIN
                   denom_reltype ILIKE 'universe' AND LOWER(normalization) LIKE 'pre%'
                 THEN
                 -- predenominated polygon interpolation weighted by universe
-                -- SUM (numer * denom * (% user geom in OBS geom)) / SUM (denom * (% user geom in OBS geom))
-                --     (10 * 1000 * 1) / (1000 * 1) = 10
-                --     (10 * 1000 * 1 + 50 * 10 * 1) / (1000 + 10) = 10500 / 10000 = 10.5
                 ' ROUND(CAST(SUM(' || numer_tablename || '.' || numer_colname ||
                 ' * ' || denom_tablename || '.' || denom_colname ||
                 ' * _procgeoms.pct_obs ' ||
@@ -133,7 +126,6 @@ BEGIN
               WHEN numer_aggregate ILIKE 'sum' AND LOWER(normalization) LIKE 'pre%'
                 THEN
                 -- predenominated polygon interpolation
-                -- SUM (numer * (% user geom in OBS geom))
                 ' ROUND(CAST(SUM(' || numer_tablename || '.' || numer_colname || ' ' ||
                 ' * _procgeoms.pct_obs) AS NUMERIC), 4) '
               -- Everything else. Point only!
@@ -149,10 +141,6 @@ BEGIN
             WHEN numer_id IS NULL THEN
               '''geomref'', _procgeoms.geomref, ' ||
               '''' || numer_colname || ''', ' || 'cdb_observatory.FIRST(_procgeoms.mvtgeom)::TEXT'
-              -- code below will return the intersection of the user's geom and the
-              -- OBS geom
-              --'''value'', ' || 'ST_Union(cdb_observatory.safe_intersection(_geoms.geom, ' || geom_tablename ||
-              --    '.' || geom_colname || '))::TEXT'
             ELSE ''
             END
           || ') val_' || colid, ', ')
@@ -186,7 +174,7 @@ BEGIN
   _json_clause AS (SELECT
     'SELECT ST_AsMVT(q, ''data'', $3) FROM (' ||
     'SELECT ' || cdb_observatory.FIRST(cte_name) || '.mvtgeom geom,
-        to_JSONB(' || (SELECT String_Agg('val_' || colid, ', ') FROM _meta) || ')
+        replace(' || (SELECT String_Agg('val_' || colid, '::TEXT || ') FROM _meta) || ', ''}{'', '', '')::jsonb
       FROM ' || String_Agg(cte_name, ', ') ||
     ' WHERE ST_Area(' || cdb_observatory.FIRST(cte_name) || '.mvtgeom) > 0' ||
     Coalesce(' AND ' || val_joins, ') q')
@@ -205,6 +193,10 @@ BEGIN
     json_clause
   USING meta;
 
+  IF procgeom_clauses IS NULL OR val_clauses IS NULL OR json_clause IS NULL THEN
+    RETURN;
+  END IF;
+
   /* Execute query */
   RETURN QUERY EXECUTE format($query$
     WITH _geoms AS (%s),
@@ -220,7 +212,7 @@ BEGIN
             String_Agg(procgeom_clauses, E',\n       '),
             String_Agg(val_clauses, E',\n       '),
             json_clause)
-  USING geom, ext, extent, buf, clip_geom, tolerance;
+  USING geom, ext, extent, buf, clip_geom;
   RETURN;
 
 END

From 915e67c061c526d338d4bb7a25d30a2e560404c2 Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Wed, 16 May 2018 13:34:32 +0200
Subject: [PATCH 17/45] Added DO+MasterCard MVT returning function

---
 src/pg/sql/45_observatory_mvt.sql | 143 ++++++++++++++++++++++++++++++
 1 file changed, 143 insertions(+)

diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql
index d8d33a5..b6bb121 100644
--- a/src/pg/sql/45_observatory_mvt.sql
+++ b/src/pg/sql/45_observatory_mvt.sql
@@ -217,3 +217,146 @@ BEGIN
 
 END
 $$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetMCDOMVT(z INTEGER, x INTEGER, y INTEGER,
+geography_level TEXT,
+do_measurements TEXT[],
+mastercard_measurements TEXT[],
+mastercard_category TEXT DEFAULT 'Total Retail',
+extent INTEGER DEFAULT 4096, buf INTEGER DEFAULT 256, clip_geom BOOLEAN DEFAULT True)
+RETURNS TABLE (mvt BYTEA)
+AS $$
+DECLARE
+  mastercard_schema CONSTANT TEXT DEFAULT 'us.mastercard';
+  mastercard_geoid CONSTANT TEXT DEFAULT 'region_id';
+  mastercard_category_column CONSTANT TEXT DEFAULT 'category';
+  mastercard_table TEXT;
+
+  bounds NUMERIC[];
+  geom GEOMETRY;
+  ext BOX2D;
+
+  measurement TEXT;
+  getmeta_parameters TEXT;
+  meta JSON;
+
+  numer_tablenames_do TEXT;
+  numer_tablenames_mc TEXT;
+  numer_colnames_do TEXT;
+  numer_colnames_do_normalized TEXT;
+  numer_colnames_mc TEXT;
+  numer_colnames_mc_normalized TEXT;
+  geom_tablenames TEXT;
+  geom_colnames TEXT;
+  geom_geomref_colnames TEXT;
+  geom_geomref_colnames_qualified TEXT;
+  geom_relations_do TEXT;
+  geom_relations_mc TEXT;
+BEGIN
+  bounds := cdb_observatory.OBS_GetTileBounds(z, x, y);
+  geom := ST_MakeEnvelope(bounds[1], bounds[2], bounds[3], bounds[4], 4326);
+  ext := ST_MakeBox2D(ST_Point(bounds[1], bounds[2]), ST_Point(bounds[3], bounds[4]));
+
+  ---------DO---------
+  getmeta_parameters := '[ ';
+  FOREACH measurement IN ARRAY do_measurements LOOP 
+    getmeta_parameters := getmeta_parameters || '{"numer_id":"' || measurement || '","geom_id":"' || geography_level || '"},';
+  END LOOP;
+  getmeta_parameters := substring(getmeta_parameters from 1 for length(getmeta_parameters) - 1) || ' ]';
+
+  meta := cdb_observatory.obs_getmeta(geom, getmeta_parameters::json, 1::integer, 1::integer, 1::integer);
+
+  IF meta IS NULL THEN
+    RETURN;
+  END IF;
+
+  SELECT  string_agg(distinct 'observatory.'||numer_tablename, ',') numer_tablenames,
+          string_agg(distinct numer_tablename||'.'||numer_colname, ',') numer_colnames,
+          string_agg(distinct numer_colname||'/area_ratio '||numer_colname, ',') numer_colnames_normalized,
+          (array_agg(distinct 'observatory.'||geom_tablename))[1] geom_tablenames,
+          (array_agg(distinct geom_colname))[1] geom_colnames,
+          (array_agg(distinct geom_geomref_colname))[1] geom_geomref_colnames,
+          (array_agg(distinct geom_tablename||'.'||geom_geomref_colname))[1] geom_geomref_colnames_qualified,
+          string_agg(distinct numer_tablename||'.'||numer_geomref_colname||'='||geom_tablename||'.'||geom_geomref_colname, ' AND ') geom_relations
+    INTO numer_tablenames_do, numer_colnames_do, numer_colnames_do_normalized, geom_tablenames, geom_colnames,
+         geom_geomref_colnames, geom_geomref_colnames_qualified, geom_relations_do
+    FROM json_to_recordset(meta)
+      AS x(id TEXT, numer_id TEXT, numer_aggregate TEXT, numer_colname TEXT, numer_geomref_colname TEXT, numer_tablename TEXT,
+           numer_type TEXT, denom_id TEXT, denom_aggregate TEXT, denom_colname TEXT, denom_geomref_colname TEXT, denom_tablename TEXT,
+           denom_type TEXT, denom_reltype TEXT, geom_id TEXT, geom_colname TEXT, geom_geomref_colname TEXT, geom_tablename TEXT,
+           geom_type TEXT, numer_timespan TEXT, geom_timespan TEXT, normalization TEXT, api_method TEXT, api_args JSON);
+
+  IF numer_tablenames_do IS NULL OR numer_colnames_do IS NULL OR numer_colnames_do_normalized IS NULL OR geom_tablenames IS NULL OR
+     geom_colnames IS NULL OR geom_geomref_colnames IS NULL OR geom_geomref_colnames_qualified IS NULL OR geom_relations_do IS NULL THEN
+    RETURN;
+  END IF;
+
+  ---------MasterCard---------
+  SELECT tablename from pg_tables
+    INTO mastercard_table
+   WHERE schemaname = mastercard_schema
+     AND tablename LIKE '%'||(string_to_array(geography_level, '.'))[array_length(string_to_array(geography_level, '.'), 1)]||'%';
+
+  SELECT string_agg(column_name, ','), string_agg(distinct column_name||'/area_ratio '||column_name, ',')
+    INTO numer_colnames_mc, numer_colnames_mc_normalized
+    FROM information_schema.columns
+   WHERE table_schema = mastercard_schema
+     AND table_name = mastercard_table
+     AND column_name = ANY(mastercard_measurements);
+
+  numer_tablenames_mc := '"'||mastercard_schema||'".'||mastercard_table;
+  geom_relations_mc := mastercard_table||'.'||mastercard_geoid||'='||geom_geomref_colnames_qualified;
+
+  IF numer_colnames_mc IS NULL OR numer_colnames_mc_normalized IS NULL OR numer_tablenames_mc IS NULL OR geom_relations_mc IS NULL THEN
+    RETURN;
+  END IF;
+
+  ---------Query build and execution---------
+    /*
+      %1$s: geom_colnames
+      %2$s: numer_colnames_do (DO measurements)
+      %3$s: numer_colnames_mc (MasterCard measurements)
+      %4$s: numer_tablenames_mc (MasterCard measurements)
+      %5$s: geom_tablenames
+      %6$s: geom_relations_do (DO measurements)
+      %7$s: geom_relations_mc (MasterCard measurements)
+      %8$s: numer_tablenames_do (DO measurements)
+      %9$s: mastercard_table
+      %10$s: mastercard_category_column
+      %11$s: numer_colnames_do_normalized (Area normalized DO measurements)
+      %12$s: numer_colnames_mc_normalized (Area normalized MasterCard measurements)
+      %13$s: geom_geomref_colnames
+
+      $1: ext (BOX2D)
+      $2: extent
+      $3: buf
+      $4: clip_geom
+      $5: geom
+      $6: mastercard_category
+    */
+  RETURN QUERY EXECUTE format(
+    $query$
+    SELECT ST_AsMVT(q, 'data', $2) FROM (
+      SELECT ST_AsMVTGeom(the_geom, $1, $2, $3, $4) AS mvtgeom, %13$s as id, %11$s, %12$s FROM (
+        SELECT  %1$s the_geom,  %13$s, %2$s, %3$s,
+                CASE  WHEN ST_Within($5, %1$s)
+                        THEN ST_Area($5) / Nullif(ST_Area(%1$s), 0)
+                      WHEN ST_Within(%1$s, $5)
+                        THEN 1
+                      ELSE ST_Area(cdb_observatory.safe_intersection($5, %1$s)) / Nullif(ST_Area(%1$s), 0)
+                END area_ratio
+          FROM %4$s, %5$s, %6$s
+        WHERE st_intersects(%1$s, $5)
+          AND %7$s
+          AND %8$s
+          AND %9$s.%10$s=$6
+      ) p
+    ) q
+    $query$,
+    geom_colnames, numer_colnames_do, numer_colnames_mc, numer_tablenames_do, numer_tablenames_mc,
+    geom_tablenames, geom_relations_do, geom_relations_mc, mastercard_table, mastercard_category_column,
+    numer_colnames_do_normalized, numer_colnames_mc_normalized, geom_geomref_colnames)
+  USING ext, extent, buf, clip_geom, geom, mastercard_category
+  RETURN;
+END
+$$ LANGUAGE plpgsql PARALLEL RESTRICTED;

From 467883726bc5451dfc03716297ec575f7a40e65e Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Thu, 17 May 2018 14:13:06 +0200
Subject: [PATCH 18/45] Added area normalization option

---
 src/pg/sql/45_observatory_mvt.sql | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql
index b6bb121..8ab3015 100644
--- a/src/pg/sql/45_observatory_mvt.sql
+++ b/src/pg/sql/45_observatory_mvt.sql
@@ -222,6 +222,7 @@ CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetMCDOMVT(z INTEGER, x INTEGER,
 geography_level TEXT,
 do_measurements TEXT[],
 mastercard_measurements TEXT[],
+area_normalized BOOLEAN DEFAULT False,
 mastercard_category TEXT DEFAULT 'Total Retail',
 extent INTEGER DEFAULT 4096, buf INTEGER DEFAULT 256, clip_geom BOOLEAN DEFAULT True)
 RETURNS TABLE (mvt BYTEA)
@@ -252,7 +253,13 @@ DECLARE
   geom_geomref_colnames_qualified TEXT;
   geom_relations_do TEXT;
   geom_relations_mc TEXT;
+
+  area_normalization TEXT DEFAULT '';
 BEGIN
+  IF area_normalized THEN
+    area_normalization := '/area_ratio';
+  END IF;
+
   bounds := cdb_observatory.OBS_GetTileBounds(z, x, y);
   geom := ST_MakeEnvelope(bounds[1], bounds[2], bounds[3], bounds[4], 4326);
   ext := ST_MakeBox2D(ST_Point(bounds[1], bounds[2]), ST_Point(bounds[3], bounds[4]));
@@ -272,7 +279,7 @@ BEGIN
 
   SELECT  string_agg(distinct 'observatory.'||numer_tablename, ',') numer_tablenames,
           string_agg(distinct numer_tablename||'.'||numer_colname, ',') numer_colnames,
-          string_agg(distinct numer_colname||'/area_ratio '||numer_colname, ',') numer_colnames_normalized,
+          string_agg(distinct numer_colname||area_normalization||' '||numer_colname, ',') numer_colnames_normalized,
           (array_agg(distinct 'observatory.'||geom_tablename))[1] geom_tablenames,
           (array_agg(distinct geom_colname))[1] geom_colnames,
           (array_agg(distinct geom_geomref_colname))[1] geom_geomref_colnames,
@@ -297,7 +304,7 @@ BEGIN
    WHERE schemaname = mastercard_schema
      AND tablename LIKE '%'||(string_to_array(geography_level, '.'))[array_length(string_to_array(geography_level, '.'), 1)]||'%';
 
-  SELECT string_agg(column_name, ','), string_agg(distinct column_name||'/area_ratio '||column_name, ',')
+  SELECT string_agg(column_name, ','), string_agg(distinct column_name||area_normalization||' '||column_name, ',')
     INTO numer_colnames_mc, numer_colnames_mc_normalized
     FROM information_schema.columns
    WHERE table_schema = mastercard_schema
@@ -337,7 +344,7 @@ BEGIN
   RETURN QUERY EXECUTE format(
     $query$
     SELECT ST_AsMVT(q, 'data', $2) FROM (
-      SELECT ST_AsMVTGeom(the_geom, $1, $2, $3, $4) AS mvtgeom, %13$s as id, %11$s, %12$s FROM (
+      SELECT ST_AsMVTGeom(the_geom, $1, $2, $3, $4) AS mvtgeom, %13$s as id, %11$s, %12$s, area_ratio FROM (
         SELECT  %1$s the_geom,  %13$s, %2$s, %3$s,
                 CASE  WHEN ST_Within($5, %1$s)
                         THEN ST_Area($5) / Nullif(ST_Area(%1$s), 0)

From f050c05a553e57cfbe2251f70bffa9789ee86241 Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Thu, 17 May 2018 15:11:14 +0200
Subject: [PATCH 19/45] Added outer joins on geometries

---
 src/pg/sql/45_observatory_mvt.sql | 32 +++++++++++++++++++------------
 1 file changed, 20 insertions(+), 12 deletions(-)

diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql
index 8ab3015..2bb896c 100644
--- a/src/pg/sql/45_observatory_mvt.sql
+++ b/src/pg/sql/45_observatory_mvt.sql
@@ -241,7 +241,9 @@ DECLARE
   getmeta_parameters TEXT;
   meta JSON;
 
-  numer_tablenames_do TEXT;
+  numer_tablename_do TEXT;
+  numer_tablenames_do TEXT[];
+  numer_tablenames_do_outer TEXT DEFAULT '';
   numer_tablenames_mc TEXT;
   numer_colnames_do TEXT;
   numer_colnames_do_normalized TEXT;
@@ -251,10 +253,11 @@ DECLARE
   geom_colnames TEXT;
   geom_geomref_colnames TEXT;
   geom_geomref_colnames_qualified TEXT;
-  geom_relations_do TEXT;
+  geom_relations_do TEXT[];
   geom_relations_mc TEXT;
 
   area_normalization TEXT DEFAULT '';
+  i INTEGER DEFAULT 0;
 BEGIN
   IF area_normalized THEN
     area_normalization := '/area_ratio';
@@ -277,14 +280,14 @@ BEGIN
     RETURN;
   END IF;
 
-  SELECT  string_agg(distinct 'observatory.'||numer_tablename, ',') numer_tablenames,
+  SELECT  array_agg(distinct 'observatory.'||numer_tablename) numer_tablenames,
           string_agg(distinct numer_tablename||'.'||numer_colname, ',') numer_colnames,
           string_agg(distinct numer_colname||area_normalization||' '||numer_colname, ',') numer_colnames_normalized,
           (array_agg(distinct 'observatory.'||geom_tablename))[1] geom_tablenames,
           (array_agg(distinct geom_colname))[1] geom_colnames,
           (array_agg(distinct geom_geomref_colname))[1] geom_geomref_colnames,
           (array_agg(distinct geom_tablename||'.'||geom_geomref_colname))[1] geom_geomref_colnames_qualified,
-          string_agg(distinct numer_tablename||'.'||numer_geomref_colname||'='||geom_tablename||'.'||geom_geomref_colname, ' AND ') geom_relations
+          array_agg(distinct numer_tablename||'.'||numer_geomref_colname||'='||geom_tablename||'.'||geom_geomref_colname) geom_relations
     INTO numer_tablenames_do, numer_colnames_do, numer_colnames_do_normalized, geom_tablenames, geom_colnames,
          geom_geomref_colnames, geom_geomref_colnames_qualified, geom_relations_do
     FROM json_to_recordset(meta)
@@ -298,6 +301,12 @@ BEGIN
     RETURN;
   END IF;
 
+  i := 0;
+  FOREACH numer_tablename_do IN ARRAY numer_tablenames_do LOOP
+    i := i + 1;
+    numer_tablenames_do_outer := numer_tablenames_do_outer || 'LEFT OUTER JOIN ' || numer_tablename_do || ' ON ' || geom_relations_do[i] || ' ';
+  END LOOP;
+
   ---------MasterCard---------
   SELECT tablename from pg_tables
     INTO mastercard_table
@@ -344,24 +353,23 @@ BEGIN
   RETURN QUERY EXECUTE format(
     $query$
     SELECT ST_AsMVT(q, 'data', $2) FROM (
-      SELECT ST_AsMVTGeom(the_geom, $1, $2, $3, $4) AS mvtgeom, %13$s as id, %11$s, %12$s, area_ratio FROM (
-        SELECT  %1$s the_geom,  %13$s, %2$s, %3$s,
+      SELECT ST_AsMVTGeom(the_geom, $1, $2, $3, $4) AS mvtgeom, %12$s as id, %10$s, %11$s, area_ratio FROM (
+        SELECT  %1$s the_geom,  %12$s, %2$s, %3$s,
                 CASE  WHEN ST_Within($5, %1$s)
                         THEN ST_Area($5) / Nullif(ST_Area(%1$s), 0)
                       WHEN ST_Within(%1$s, $5)
                         THEN 1
                       ELSE ST_Area(cdb_observatory.safe_intersection($5, %1$s)) / Nullif(ST_Area(%1$s), 0)
                 END area_ratio
-          FROM %4$s, %5$s, %6$s
+          FROM %6$s
+               %4$s
+               LEFT OUTER JOIN %5$s ON %7$s AND %8$s.%9$s=$6
         WHERE st_intersects(%1$s, $5)
-          AND %7$s
-          AND %8$s
-          AND %9$s.%10$s=$6
       ) p
     ) q
     $query$,
-    geom_colnames, numer_colnames_do, numer_colnames_mc, numer_tablenames_do, numer_tablenames_mc,
-    geom_tablenames, geom_relations_do, geom_relations_mc, mastercard_table, mastercard_category_column,
+    geom_colnames, numer_colnames_do, numer_colnames_mc, numer_tablenames_do_outer, numer_tablenames_mc,
+    geom_tablenames, geom_relations_mc, mastercard_table, mastercard_category_column,
     numer_colnames_do_normalized, numer_colnames_mc_normalized, geom_geomref_colnames)
   USING ext, extent, buf, clip_geom, geom, mastercard_category
   RETURN;

From f1668777d568c571bc8fb69d998225331ebce1a5 Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Thu, 17 May 2018 16:57:04 +0200
Subject: [PATCH 20/45] Using shoreline clipped geoms

---
 src/pg/sql/45_observatory_mvt.sql | 28 +++++++++++++++++++++++++---
 1 file changed, 25 insertions(+), 3 deletions(-)

diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql
index 2bb896c..fcf36c3 100644
--- a/src/pg/sql/45_observatory_mvt.sql
+++ b/src/pg/sql/45_observatory_mvt.sql
@@ -228,6 +228,12 @@ extent INTEGER DEFAULT 4096, buf INTEGER DEFAULT 256, clip_geom BOOLEAN DEFAULT
 RETURNS TABLE (mvt BYTEA)
 AS $$
 DECLARE
+  state_geoname CONSTANT TEXT DEFAULT 'us.census.tiger.state';
+  county_geoname CONSTANT TEXT DEFAULT 'us.census.tiger.county';
+  tract_geoname CONSTANT TEXT DEFAULT 'us.census.tiger.census_tract';
+  blockgroup_geoname CONSTANT TEXT DEFAULT 'us.census.tiger.block_group';
+  block_geoname CONSTANT TEXT DEFAULT 'us.census.tiger.block';
+
   mastercard_schema CONSTANT TEXT DEFAULT 'us.mastercard';
   mastercard_geoid CONSTANT TEXT DEFAULT 'region_id';
   mastercard_category_column CONSTANT TEXT DEFAULT 'category';
@@ -256,6 +262,7 @@ DECLARE
   geom_relations_do TEXT[];
   geom_relations_mc TEXT;
 
+  simplification_tolerance NUMERIC DEFAULT 0.0001;
   area_normalization TEXT DEFAULT '';
   i INTEGER DEFAULT 0;
 BEGIN
@@ -263,6 +270,21 @@ BEGIN
     area_normalization := '/area_ratio';
   END IF;
 
+  CASE
+    WHEN geography_level = state_geoname THEN
+      simplification_tolerance := 0.1;
+    WHEN geography_level = county_geoname THEN
+      simplification_tolerance := 0.01;
+    WHEN geography_level = tract_geoname THEN
+      simplification_tolerance := 0.001;
+    WHEN geography_level = blockgroup_geoname THEN
+      simplification_tolerance := 0.0001;
+    WHEN geography_level = block_geoname THEN
+      simplification_tolerance := 0.0001;
+    ELSE
+      RETURN;
+  END CASE;
+
   bounds := cdb_observatory.OBS_GetTileBounds(z, x, y);
   geom := ST_MakeEnvelope(bounds[1], bounds[2], bounds[3], bounds[4], 4326);
   ext := ST_MakeBox2D(ST_Point(bounds[1], bounds[2]), ST_Point(bounds[3], bounds[4]));
@@ -270,7 +292,7 @@ BEGIN
   ---------DO---------
   getmeta_parameters := '[ ';
   FOREACH measurement IN ARRAY do_measurements LOOP 
-    getmeta_parameters := getmeta_parameters || '{"numer_id":"' || measurement || '","geom_id":"' || geography_level || '"},';
+    getmeta_parameters := getmeta_parameters || '{"numer_id":"' || measurement || '","geom_id":"' || geography_level || '_clipped"},';
   END LOOP;
   getmeta_parameters := substring(getmeta_parameters from 1 for length(getmeta_parameters) - 1) || ' ]';
 
@@ -359,7 +381,7 @@ BEGIN
                         THEN ST_Area($5) / Nullif(ST_Area(%1$s), 0)
                       WHEN ST_Within(%1$s, $5)
                         THEN 1
-                      ELSE ST_Area(cdb_observatory.safe_intersection($5, %1$s)) / Nullif(ST_Area(%1$s), 0)
+                      ELSE ST_Area(cdb_observatory.safe_intersection(st_simplifyvw(%1$s, $7), $5)) / Nullif(ST_Area(%1$s), 0)
                 END area_ratio
           FROM %6$s
                %4$s
@@ -371,7 +393,7 @@ BEGIN
     geom_colnames, numer_colnames_do, numer_colnames_mc, numer_tablenames_do_outer, numer_tablenames_mc,
     geom_tablenames, geom_relations_mc, mastercard_table, mastercard_category_column,
     numer_colnames_do_normalized, numer_colnames_mc_normalized, geom_geomref_colnames)
-  USING ext, extent, buf, clip_geom, geom, mastercard_category
+  USING ext, extent, buf, clip_geom, geom, mastercard_category, simplification_tolerance
   RETURN;
 END
 $$ LANGUAGE plpgsql PARALLEL RESTRICTED;

From 811009a8adceb3ebcd923125eec2575139df1d64 Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Thu, 17 May 2018 17:03:27 +0200
Subject: [PATCH 21/45] Using clipped geometries parametrized

---
 src/pg/sql/45_observatory_mvt.sql | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql
index fcf36c3..1ebc113 100644
--- a/src/pg/sql/45_observatory_mvt.sql
+++ b/src/pg/sql/45_observatory_mvt.sql
@@ -265,6 +265,7 @@ DECLARE
   simplification_tolerance NUMERIC DEFAULT 0.0001;
   area_normalization TEXT DEFAULT '';
   i INTEGER DEFAULT 0;
+  clipped TEXT default '';
 BEGIN
   IF area_normalized THEN
     area_normalization := '/area_ratio';
@@ -274,12 +275,16 @@ BEGIN
     WHEN geography_level = state_geoname THEN
       simplification_tolerance := 0.1;
     WHEN geography_level = county_geoname THEN
+      clipped := '_clipped';
       simplification_tolerance := 0.01;
     WHEN geography_level = tract_geoname THEN
+      clipped := '_clipped';
       simplification_tolerance := 0.001;
     WHEN geography_level = blockgroup_geoname THEN
+      clipped := '_clipped';
       simplification_tolerance := 0.0001;
     WHEN geography_level = block_geoname THEN
+      clipped := '_clipped';
       simplification_tolerance := 0.0001;
     ELSE
       RETURN;
@@ -292,7 +297,7 @@ BEGIN
   ---------DO---------
   getmeta_parameters := '[ ';
   FOREACH measurement IN ARRAY do_measurements LOOP 
-    getmeta_parameters := getmeta_parameters || '{"numer_id":"' || measurement || '","geom_id":"' || geography_level || '_clipped"},';
+    getmeta_parameters := getmeta_parameters || '{"numer_id":"' || measurement || '","geom_id":"' || geography_level || clipped ||'"},';
   END LOOP;
   getmeta_parameters := substring(getmeta_parameters from 1 for length(getmeta_parameters) - 1) || ' ]';
 

From 0ae15880e05912682951b5585d9aa160eaca3152 Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Fri, 18 May 2018 12:49:58 +0200
Subject: [PATCH 22/45] Function returning MVT ready geoms and MVT data in JSON

---
 src/pg/sql/45_observatory_mvt.sql | 71 +++++++++++++++----------------
 1 file changed, 34 insertions(+), 37 deletions(-)

diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql
index 1ebc113..5401b53 100644
--- a/src/pg/sql/45_observatory_mvt.sql
+++ b/src/pg/sql/45_observatory_mvt.sql
@@ -222,10 +222,16 @@ CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetMCDOMVT(z INTEGER, x INTEGER,
 geography_level TEXT,
 do_measurements TEXT[],
 mastercard_measurements TEXT[],
+shoreline_clipped BOOLEAN DEFAULT True,
+optimize_clipping BOOLEAN DEFAULT False,
+simplify_geometries BOOLEAN DEFAULT False,
 area_normalized BOOLEAN DEFAULT False,
 mastercard_category TEXT DEFAULT 'Total Retail',
 extent INTEGER DEFAULT 4096, buf INTEGER DEFAULT 256, clip_geom BOOLEAN DEFAULT True)
-RETURNS TABLE (mvt BYTEA)
+RETURNS TABLE (
+  mvtgeom GEOMETRY,
+  mvtdata JSON
+)
 AS $$
 DECLARE
   state_geoname CONSTANT TEXT DEFAULT 'us.census.tiger.state';
@@ -252,6 +258,7 @@ DECLARE
   numer_tablenames_do_outer TEXT DEFAULT '';
   numer_tablenames_mc TEXT;
   numer_colnames_do TEXT;
+  numer_colnames_do_qualified TEXT;
   numer_colnames_do_normalized TEXT;
   numer_colnames_mc TEXT;
   numer_colnames_mc_normalized TEXT;
@@ -262,7 +269,7 @@ DECLARE
   geom_relations_do TEXT[];
   geom_relations_mc TEXT;
 
-  simplification_tolerance NUMERIC DEFAULT 0.0001;
+  simplification_tolerance NUMERIC DEFAULT 0;
   area_normalization TEXT DEFAULT '';
   i INTEGER DEFAULT 0;
   clipped TEXT default '';
@@ -271,25 +278,32 @@ BEGIN
     area_normalization := '/area_ratio';
   END IF;
 
+  IF shoreline_clipped THEN
+    clipped := '_clipped';
+  END IF;
+
   CASE
     WHEN geography_level = state_geoname THEN
       simplification_tolerance := 0.1;
+      IF optimize_clipping THEN
+        clipped := '';
+      END IF;
     WHEN geography_level = county_geoname THEN
-      clipped := '_clipped';
       simplification_tolerance := 0.01;
     WHEN geography_level = tract_geoname THEN
-      clipped := '_clipped';
       simplification_tolerance := 0.001;
     WHEN geography_level = blockgroup_geoname THEN
-      clipped := '_clipped';
       simplification_tolerance := 0.0001;
     WHEN geography_level = block_geoname THEN
-      clipped := '_clipped';
       simplification_tolerance := 0.0001;
     ELSE
       RETURN;
   END CASE;
 
+  IF NOT simplify_geometries THEN
+    simplification_tolerance := 0;
+  END IF;
+
   bounds := cdb_observatory.OBS_GetTileBounds(z, x, y);
   geom := ST_MakeEnvelope(bounds[1], bounds[2], bounds[3], bounds[4], 4326);
   ext := ST_MakeBox2D(ST_Point(bounds[1], bounds[2]), ST_Point(bounds[3], bounds[4]));
@@ -308,14 +322,15 @@ BEGIN
   END IF;
 
   SELECT  array_agg(distinct 'observatory.'||numer_tablename) numer_tablenames,
-          string_agg(distinct numer_tablename||'.'||numer_colname, ',') numer_colnames,
+          string_agg(distinct numer_colname, ',') numer_colnames,
+          string_agg(distinct numer_tablename||'.'||numer_colname, ',') numer_colnames_qualified,
           string_agg(distinct numer_colname||area_normalization||' '||numer_colname, ',') numer_colnames_normalized,
           (array_agg(distinct 'observatory.'||geom_tablename))[1] geom_tablenames,
           (array_agg(distinct geom_colname))[1] geom_colnames,
           (array_agg(distinct geom_geomref_colname))[1] geom_geomref_colnames,
           (array_agg(distinct geom_tablename||'.'||geom_geomref_colname))[1] geom_geomref_colnames_qualified,
           array_agg(distinct numer_tablename||'.'||numer_geomref_colname||'='||geom_tablename||'.'||geom_geomref_colname) geom_relations
-    INTO numer_tablenames_do, numer_colnames_do, numer_colnames_do_normalized, geom_tablenames, geom_colnames,
+    INTO numer_tablenames_do, numer_colnames_do, numer_colnames_do_qualified, numer_colnames_do_normalized, geom_tablenames, geom_colnames,
          geom_geomref_colnames, geom_geomref_colnames_qualified, geom_relations_do
     FROM json_to_recordset(meta)
       AS x(id TEXT, numer_id TEXT, numer_aggregate TEXT, numer_colname TEXT, numer_geomref_colname TEXT, numer_tablename TEXT,
@@ -323,8 +338,9 @@ BEGIN
            denom_type TEXT, denom_reltype TEXT, geom_id TEXT, geom_colname TEXT, geom_geomref_colname TEXT, geom_tablename TEXT,
            geom_type TEXT, numer_timespan TEXT, geom_timespan TEXT, normalization TEXT, api_method TEXT, api_args JSON);
 
-  IF numer_tablenames_do IS NULL OR numer_colnames_do IS NULL OR numer_colnames_do_normalized IS NULL OR geom_tablenames IS NULL OR
-     geom_colnames IS NULL OR geom_geomref_colnames IS NULL OR geom_geomref_colnames_qualified IS NULL OR geom_relations_do IS NULL THEN
+  IF numer_tablenames_do IS NULL OR numer_colnames_do IS NULL OR numer_colnames_do_qualified IS NULL OR numer_colnames_do_normalized IS NULL
+     OR geom_tablenames IS NULL OR geom_colnames IS NULL OR geom_geomref_colnames IS NULL OR geom_geomref_colnames_qualified IS NULL
+     OR geom_relations_do IS NULL THEN
     RETURN;
   END IF;
 
@@ -350,36 +366,17 @@ BEGIN
   numer_tablenames_mc := '"'||mastercard_schema||'".'||mastercard_table;
   geom_relations_mc := mastercard_table||'.'||mastercard_geoid||'='||geom_geomref_colnames_qualified;
 
-  IF numer_colnames_mc IS NULL OR numer_colnames_mc_normalized IS NULL OR numer_tablenames_mc IS NULL OR geom_relations_mc IS NULL THEN
+  IF numer_colnames_mc IS NULL OR numer_colnames_mc_normalized IS NULL
+    OR numer_tablenames_mc IS NULL OR geom_relations_mc IS NULL THEN
     RETURN;
   END IF;
 
   ---------Query build and execution---------
-    /*
-      %1$s: geom_colnames
-      %2$s: numer_colnames_do (DO measurements)
-      %3$s: numer_colnames_mc (MasterCard measurements)
-      %4$s: numer_tablenames_mc (MasterCard measurements)
-      %5$s: geom_tablenames
-      %6$s: geom_relations_do (DO measurements)
-      %7$s: geom_relations_mc (MasterCard measurements)
-      %8$s: numer_tablenames_do (DO measurements)
-      %9$s: mastercard_table
-      %10$s: mastercard_category_column
-      %11$s: numer_colnames_do_normalized (Area normalized DO measurements)
-      %12$s: numer_colnames_mc_normalized (Area normalized MasterCard measurements)
-      %13$s: geom_geomref_colnames
-
-      $1: ext (BOX2D)
-      $2: extent
-      $3: buf
-      $4: clip_geom
-      $5: geom
-      $6: mastercard_category
-    */
   RETURN QUERY EXECUTE format(
     $query$
-    SELECT ST_AsMVT(q, 'data', $2) FROM (
+    SELECT  mvtgeom,
+            (select row_to_json(_) from (select id, %13$s, %3$s, area_ratio) as _) as mvtdata
+          FROM (
       SELECT ST_AsMVTGeom(the_geom, $1, $2, $3, $4) AS mvtgeom, %12$s as id, %10$s, %11$s, area_ratio FROM (
         SELECT  %1$s the_geom,  %12$s, %2$s, %3$s,
                 CASE  WHEN ST_Within($5, %1$s)
@@ -395,9 +392,9 @@ BEGIN
       ) p
     ) q
     $query$,
-    geom_colnames, numer_colnames_do, numer_colnames_mc, numer_tablenames_do_outer, numer_tablenames_mc,
-    geom_tablenames, geom_relations_mc, mastercard_table, mastercard_category_column,
-    numer_colnames_do_normalized, numer_colnames_mc_normalized, geom_geomref_colnames)
+    geom_colnames, numer_colnames_do_qualified, numer_colnames_mc, numer_tablenames_do_outer, numer_tablenames_mc,
+    geom_tablenames, geom_relations_mc, mastercard_table, mastercard_category_column, numer_colnames_do_normalized,
+    numer_colnames_mc_normalized, geom_geomref_colnames, numer_colnames_do)
   USING ext, extent, buf, clip_geom, geom, mastercard_category, simplification_tolerance
   RETURN;
 END

From 7c559fe7d53ad372e5dabb5751550bc35086b31b Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Tue, 22 May 2018 13:39:41 +0200
Subject: [PATCH 23/45] If getmeta returns null, retry using only the geography
 level

---
 src/pg/sql/45_observatory_mvt.sql | 96 ++++++++++++++++++-------------
 1 file changed, 56 insertions(+), 40 deletions(-)

diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql
index 5401b53..861fd78 100644
--- a/src/pg/sql/45_observatory_mvt.sql
+++ b/src/pg/sql/45_observatory_mvt.sql
@@ -253,20 +253,20 @@ DECLARE
   getmeta_parameters TEXT;
   meta JSON;
 
-  numer_tablename_do TEXT;
-  numer_tablenames_do TEXT[];
+  numer_tablename_do TEXT DEFAULT '';
+  numer_tablenames_do TEXT[] DEFAULT ARRAY['']::TEXT[];
   numer_tablenames_do_outer TEXT DEFAULT '';
   numer_tablenames_mc TEXT;
-  numer_colnames_do TEXT;
-  numer_colnames_do_qualified TEXT;
-  numer_colnames_do_normalized TEXT;
+  numer_colnames_do TEXT DEFAULT '';
+  numer_colnames_do_qualified TEXT DEFAULT '';
+  numer_colnames_do_normalized TEXT DEFAULT '';
   numer_colnames_mc TEXT;
   numer_colnames_mc_normalized TEXT;
   geom_tablenames TEXT;
   geom_colnames TEXT;
   geom_geomref_colnames TEXT;
   geom_geomref_colnames_qualified TEXT;
-  geom_relations_do TEXT[];
+  geom_relations_do TEXT[] DEFAULT ARRAY['']::TEXT[];
   geom_relations_mc TEXT;
 
   simplification_tolerance NUMERIC DEFAULT 0;
@@ -317,39 +317,55 @@ BEGIN
 
   meta := cdb_observatory.obs_getmeta(geom, getmeta_parameters::json, 1::integer, 1::integer, 1::integer);
 
-  IF meta IS NULL THEN
-    RETURN;
+  IF meta IS NOT NULL THEN
+    SELECT  array_agg(distinct 'observatory.'||numer_tablename) numer_tablenames,
+            string_agg(distinct numer_colname, ',')||',' numer_colnames,
+            string_agg(distinct numer_tablename||'.'||numer_colname, ',')||',' numer_colnames_qualified,
+            string_agg(distinct numer_colname||area_normalization||' '||numer_colname, ',')||',' numer_colnames_normalized,
+            (array_agg(distinct 'observatory.'||geom_tablename))[1] geom_tablenames,
+            (array_agg(distinct geom_colname))[1] geom_colnames,
+            (array_agg(distinct geom_geomref_colname))[1] geom_geomref_colnames,
+            (array_agg(distinct geom_tablename||'.'||geom_geomref_colname))[1] geom_geomref_colnames_qualified,
+            array_agg(distinct numer_tablename||'.'||numer_geomref_colname||'='||geom_tablename||'.'||geom_geomref_colname) geom_relations
+      INTO numer_tablenames_do, numer_colnames_do, numer_colnames_do_qualified, numer_colnames_do_normalized, geom_tablenames, geom_colnames,
+          geom_geomref_colnames, geom_geomref_colnames_qualified, geom_relations_do
+      FROM json_to_recordset(meta)
+        AS x(id TEXT, numer_id TEXT, numer_aggregate TEXT, numer_colname TEXT, numer_geomref_colname TEXT, numer_tablename TEXT,
+            numer_type TEXT, denom_id TEXT, denom_aggregate TEXT, denom_colname TEXT, denom_geomref_colname TEXT, denom_tablename TEXT,
+            denom_type TEXT, denom_reltype TEXT, geom_id TEXT, geom_colname TEXT, geom_geomref_colname TEXT, geom_tablename TEXT,
+            geom_type TEXT, numer_timespan TEXT, geom_timespan TEXT, normalization TEXT, api_method TEXT, api_args JSON);
+
+    IF numer_tablenames_do IS NULL OR numer_colnames_do IS NULL OR numer_colnames_do_qualified IS NULL OR numer_colnames_do_normalized IS NULL
+      OR geom_tablenames IS NULL OR geom_colnames IS NULL OR geom_geomref_colnames IS NULL OR geom_geomref_colnames_qualified IS NULL
+      OR geom_relations_do IS NULL THEN
+      RETURN;
+    END IF;
+
+    i := 0;
+    FOREACH numer_tablename_do IN ARRAY numer_tablenames_do LOOP
+      i := i + 1;
+      numer_tablenames_do_outer := numer_tablenames_do_outer || 'LEFT OUTER JOIN ' || numer_tablename_do || ' ON ' || geom_relations_do[i] || ' ';
+    END LOOP;
+  ELSE
+    getmeta_parameters := '[{"geom_id":"' || geography_level || clipped ||'"}]';
+    meta := cdb_observatory.obs_getmeta(geom, getmeta_parameters::json, 1::integer, 1::integer, 1::integer);
+
+    IF meta IS NULL THEN
+      RETURN;
+    END IF;
+
+    SELECT  (array_agg(distinct 'observatory.'||geom_tablename))[1] geom_tablenames,
+            (array_agg(distinct geom_colname))[1] geom_colnames,
+            (array_agg(distinct geom_geomref_colname))[1] geom_geomref_colnames,
+            (array_agg(distinct geom_tablename||'.'||geom_geomref_colname))[1] geom_geomref_colnames_qualified
+      FROM json_to_recordset(meta)
+      INTO geom_tablenames, geom_colnames, geom_geomref_colnames, geom_geomref_colnames_qualified
+        AS x(id TEXT, numer_id TEXT, numer_aggregate TEXT, numer_colname TEXT, numer_geomref_colname TEXT, numer_tablename TEXT,
+            numer_type TEXT, denom_id TEXT, denom_aggregate TEXT, denom_colname TEXT, denom_geomref_colname TEXT, denom_tablename TEXT,
+            denom_type TEXT, denom_reltype TEXT, geom_id TEXT, geom_colname TEXT, geom_geomref_colname TEXT, geom_tablename TEXT,
+            geom_type TEXT, numer_timespan TEXT, geom_timespan TEXT, normalization TEXT, api_method TEXT, api_args JSON);
   END IF;
 
-  SELECT  array_agg(distinct 'observatory.'||numer_tablename) numer_tablenames,
-          string_agg(distinct numer_colname, ',') numer_colnames,
-          string_agg(distinct numer_tablename||'.'||numer_colname, ',') numer_colnames_qualified,
-          string_agg(distinct numer_colname||area_normalization||' '||numer_colname, ',') numer_colnames_normalized,
-          (array_agg(distinct 'observatory.'||geom_tablename))[1] geom_tablenames,
-          (array_agg(distinct geom_colname))[1] geom_colnames,
-          (array_agg(distinct geom_geomref_colname))[1] geom_geomref_colnames,
-          (array_agg(distinct geom_tablename||'.'||geom_geomref_colname))[1] geom_geomref_colnames_qualified,
-          array_agg(distinct numer_tablename||'.'||numer_geomref_colname||'='||geom_tablename||'.'||geom_geomref_colname) geom_relations
-    INTO numer_tablenames_do, numer_colnames_do, numer_colnames_do_qualified, numer_colnames_do_normalized, geom_tablenames, geom_colnames,
-         geom_geomref_colnames, geom_geomref_colnames_qualified, geom_relations_do
-    FROM json_to_recordset(meta)
-      AS x(id TEXT, numer_id TEXT, numer_aggregate TEXT, numer_colname TEXT, numer_geomref_colname TEXT, numer_tablename TEXT,
-           numer_type TEXT, denom_id TEXT, denom_aggregate TEXT, denom_colname TEXT, denom_geomref_colname TEXT, denom_tablename TEXT,
-           denom_type TEXT, denom_reltype TEXT, geom_id TEXT, geom_colname TEXT, geom_geomref_colname TEXT, geom_tablename TEXT,
-           geom_type TEXT, numer_timespan TEXT, geom_timespan TEXT, normalization TEXT, api_method TEXT, api_args JSON);
-
-  IF numer_tablenames_do IS NULL OR numer_colnames_do IS NULL OR numer_colnames_do_qualified IS NULL OR numer_colnames_do_normalized IS NULL
-     OR geom_tablenames IS NULL OR geom_colnames IS NULL OR geom_geomref_colnames IS NULL OR geom_geomref_colnames_qualified IS NULL
-     OR geom_relations_do IS NULL THEN
-    RETURN;
-  END IF;
-
-  i := 0;
-  FOREACH numer_tablename_do IN ARRAY numer_tablenames_do LOOP
-    i := i + 1;
-    numer_tablenames_do_outer := numer_tablenames_do_outer || 'LEFT OUTER JOIN ' || numer_tablename_do || ' ON ' || geom_relations_do[i] || ' ';
-  END LOOP;
-
   ---------MasterCard---------
   SELECT tablename from pg_tables
     INTO mastercard_table
@@ -375,10 +391,10 @@ BEGIN
   RETURN QUERY EXECUTE format(
     $query$
     SELECT  mvtgeom,
-            (select row_to_json(_) from (select id, %13$s, %3$s, area_ratio) as _) as mvtdata
+            (select row_to_json(_) from (select id, %13$s %3$s, area_ratio) as _) as mvtdata
           FROM (
-      SELECT ST_AsMVTGeom(the_geom, $1, $2, $3, $4) AS mvtgeom, %12$s as id, %10$s, %11$s, area_ratio FROM (
-        SELECT  %1$s the_geom,  %12$s, %2$s, %3$s,
+      SELECT ST_AsMVTGeom(the_geom, $1, $2, $3, $4) AS mvtgeom, %12$s as id, %10$s %11$s, area_ratio FROM (
+        SELECT  %1$s the_geom, %12$s, %2$s %3$s,
                 CASE  WHEN ST_Within($5, %1$s)
                         THEN ST_Area($5) / Nullif(ST_Area(%1$s), 0)
                       WHEN ST_Within(%1$s, $5)

From d1c302a9dfdb4022bbe77780ee1f8bea3be02f6f Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Wed, 30 May 2018 09:11:27 +0200
Subject: [PATCH 24/45] Added transform to 3857

---
 src/pg/sql/45_observatory_mvt.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql
index 861fd78..b94c4f4 100644
--- a/src/pg/sql/45_observatory_mvt.sql
+++ b/src/pg/sql/45_observatory_mvt.sql
@@ -393,7 +393,7 @@ BEGIN
     SELECT  mvtgeom,
             (select row_to_json(_) from (select id, %13$s %3$s, area_ratio) as _) as mvtdata
           FROM (
-      SELECT ST_AsMVTGeom(the_geom, $1, $2, $3, $4) AS mvtgeom, %12$s as id, %10$s %11$s, area_ratio FROM (
+      SELECT ST_AsMVTGeom(ST_Transform(the_geom, 3857), $1, $2, $3, $4) AS mvtgeom, %12$s as id, %10$s %11$s, area_ratio FROM (
         SELECT  %1$s the_geom, %12$s, %2$s %3$s,
                 CASE  WHEN ST_Within($5, %1$s)
                         THEN ST_Area($5) / Nullif(ST_Area(%1$s), 0)

From e2017d41cf64b7a7b655256440ae747923733686 Mon Sep 17 00:00:00 2001
From: Mario de Frutos <ethervoid@gmail.com>
Date: Wed, 30 May 2018 16:38:26 +0200
Subject: [PATCH 25/45] Add exception for the mc geography levels in the new
 file

---
 src/pg/sql/45_observatory_mvt.sql | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql
index b94c4f4..ddcec17 100644
--- a/src/pg/sql/45_observatory_mvt.sql
+++ b/src/pg/sql/45_observatory_mvt.sql
@@ -252,6 +252,7 @@ DECLARE
   measurement TEXT;
   getmeta_parameters TEXT;
   meta JSON;
+  mc_geography_level TEXT;
 
   numer_tablename_do TEXT DEFAULT '';
   numer_tablenames_do TEXT[] DEFAULT ARRAY['']::TEXT[];
@@ -310,7 +311,7 @@ BEGIN
 
   ---------DO---------
   getmeta_parameters := '[ ';
-  FOREACH measurement IN ARRAY do_measurements LOOP 
+  FOREACH measurement IN ARRAY do_measurements LOOP
     getmeta_parameters := getmeta_parameters || '{"numer_id":"' || measurement || '","geom_id":"' || geography_level || clipped ||'"},';
   END LOOP;
   getmeta_parameters := substring(getmeta_parameters from 1 for length(getmeta_parameters) - 1) || ' ]';
@@ -367,10 +368,16 @@ BEGIN
   END IF;
 
   ---------MasterCard---------
+  IF geography_level = 'us.census.tiger.census_tract' THEN
+    mc_geography_level := 'tract';
+  ELSE
+    mc_geography_level := (string_to_array(geography_level, '.'))[array_length(string_to_array(geography_level, '.'), 1)];
+  END IF;
+
   SELECT tablename from pg_tables
     INTO mastercard_table
    WHERE schemaname = mastercard_schema
-     AND tablename LIKE '%'||(string_to_array(geography_level, '.'))[array_length(string_to_array(geography_level, '.'), 1)]||'%';
+     AND tablename LIKE '%'||mc_geography_level||'%';
 
   SELECT string_agg(column_name, ','), string_agg(distinct column_name||area_normalization||' '||column_name, ',')
     INTO numer_colnames_mc, numer_colnames_mc_normalized

From 23088efc916a800d82c0a626271858dab16e5853 Mon Sep 17 00:00:00 2001
From: Mario de Frutos <ethervoid@gmail.com>
Date: Wed, 30 May 2018 16:49:36 +0200
Subject: [PATCH 26/45] Function OBS_GetMCDOMVT must return jsonb

---
 src/pg/sql/45_observatory_mvt.sql | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql
index ddcec17..2a98aaf 100644
--- a/src/pg/sql/45_observatory_mvt.sql
+++ b/src/pg/sql/45_observatory_mvt.sql
@@ -230,7 +230,7 @@ mastercard_category TEXT DEFAULT 'Total Retail',
 extent INTEGER DEFAULT 4096, buf INTEGER DEFAULT 256, clip_geom BOOLEAN DEFAULT True)
 RETURNS TABLE (
   mvtgeom GEOMETRY,
-  mvtdata JSON
+  mvtdata JSONB
 )
 AS $$
 DECLARE
@@ -398,7 +398,7 @@ BEGIN
   RETURN QUERY EXECUTE format(
     $query$
     SELECT  mvtgeom,
-            (select row_to_json(_) from (select id, %13$s %3$s, area_ratio) as _) as mvtdata
+            (select row_to_json(_)::jsonb from (select id, %13$s %3$s, area_ratio) as _) as mvtdata
           FROM (
       SELECT ST_AsMVTGeom(ST_Transform(the_geom, 3857), $1, $2, $3, $4) AS mvtgeom, %12$s as id, %10$s %11$s, area_ratio FROM (
         SELECT  %1$s the_geom, %12$s, %2$s %3$s,

From 0d172353fb111a9953e10117a877228067efed70 Mon Sep 17 00:00:00 2001
From: Mario de Frutos <ethervoid@gmail.com>
Date: Wed, 30 May 2018 17:20:06 +0200
Subject: [PATCH 27/45] Revert "Added transform to 3857"

This reverts commit d1c302a9dfdb4022bbe77780ee1f8bea3be02f6f.
---
 src/pg/sql/45_observatory_mvt.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql
index 2a98aaf..90e5b90 100644
--- a/src/pg/sql/45_observatory_mvt.sql
+++ b/src/pg/sql/45_observatory_mvt.sql
@@ -400,7 +400,7 @@ BEGIN
     SELECT  mvtgeom,
             (select row_to_json(_)::jsonb from (select id, %13$s %3$s, area_ratio) as _) as mvtdata
           FROM (
-      SELECT ST_AsMVTGeom(ST_Transform(the_geom, 3857), $1, $2, $3, $4) AS mvtgeom, %12$s as id, %10$s %11$s, area_ratio FROM (
+      SELECT ST_AsMVTGeom(the_geom, $1, $2, $3, $4) AS mvtgeom, %12$s as id, %10$s %11$s, area_ratio FROM (
         SELECT  %1$s the_geom, %12$s, %2$s %3$s,
                 CASE  WHEN ST_Within($5, %1$s)
                         THEN ST_Area($5) / Nullif(ST_Area(%1$s), 0)

From c62d1cd4b868d761804b17c77aab18dd8c494d37 Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Mon, 11 Jun 2018 17:53:25 +0200
Subject: [PATCH 28/45] Fixed MVT projection

---
 src/pg/sql/45_observatory_mvt.sql | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql
index 90e5b90..aa24851 100644
--- a/src/pg/sql/45_observatory_mvt.sql
+++ b/src/pg/sql/45_observatory_mvt.sql
@@ -307,7 +307,8 @@ BEGIN
 
   bounds := cdb_observatory.OBS_GetTileBounds(z, x, y);
   geom := ST_MakeEnvelope(bounds[1], bounds[2], bounds[3], bounds[4], 4326);
-  ext := ST_MakeBox2D(ST_Point(bounds[1], bounds[2]), ST_Point(bounds[3], bounds[4]));
+  ext := ST_MakeBox2D(ST_Transform(ST_SetSRID(ST_Point(bounds[1], bounds[2]), 4326), 3857),
+                      ST_Transform(ST_SetSRID(ST_Point(bounds[3], bounds[4]), 4326), 3857));
 
   ---------DO---------
   getmeta_parameters := '[ ';
@@ -400,7 +401,7 @@ BEGIN
     SELECT  mvtgeom,
             (select row_to_json(_)::jsonb from (select id, %13$s %3$s, area_ratio) as _) as mvtdata
           FROM (
-      SELECT ST_AsMVTGeom(the_geom, $1, $2, $3, $4) AS mvtgeom, %12$s as id, %10$s %11$s, area_ratio FROM (
+      SELECT ST_AsMVTGeom(ST_Transform(the_geom, 3857), $1, $2, $3, $4) AS mvtgeom, %12$s as id, %10$s %11$s, area_ratio FROM (
         SELECT  %1$s the_geom, %12$s, %2$s %3$s,
                 CASE  WHEN ST_Within($5, %1$s)
                         THEN ST_Area($5) / Nullif(ST_Area(%1$s), 0)

From 6924cc4512e579f682254b6842002f823d767f0e Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Wed, 13 Jun 2018 11:00:21 +0200
Subject: [PATCH 29/45] Added cache for OBS_GetMeta function results

---
 src/pg/sql/45_observatory_mvt.sql | 75 ++++++++++++++++++++++++++-----
 1 file changed, 64 insertions(+), 11 deletions(-)

diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql
index aa24851..8a6185e 100644
--- a/src/pg/sql/45_observatory_mvt.sql
+++ b/src/pg/sql/45_observatory_mvt.sql
@@ -218,16 +218,65 @@ BEGIN
 END
 $$ LANGUAGE plpgsql;
 
-CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetMCDOMVT(z INTEGER, x INTEGER, y INTEGER,
-geography_level TEXT,
-do_measurements TEXT[],
-mastercard_measurements TEXT[],
-shoreline_clipped BOOLEAN DEFAULT True,
-optimize_clipping BOOLEAN DEFAULT False,
-simplify_geometries BOOLEAN DEFAULT False,
-area_normalized BOOLEAN DEFAULT False,
-mastercard_category TEXT DEFAULT 'Total Retail',
-extent INTEGER DEFAULT 4096, buf INTEGER DEFAULT 256, clip_geom BOOLEAN DEFAULT True)
+DROP TABLE IF EXISTS cdb_observatory.OBS_CachedMeta;
+CREATE TABLE cdb_observatory.OBS_CachedMeta(
+  z INTEGER,
+  parameters TEXT,
+  num_timespans INTEGER,
+  num_scores INTEGER,
+  num_target_geoms INTEGER,
+  result JSON,
+  PRIMARY KEY (z, parameters, num_timespans, num_scores, num_target_geoms)
+);
+
+CREATE OR REPLACE FUNCTION cdb_observatory.OBS_RetrieveMeta(
+  zoom INTEGER,
+  geom geometry(Geometry, 4326),
+  getmeta_parameters JSON,
+  num_timespan_options INTEGER DEFAULT NULL,
+  num_score_options INTEGER DEFAULT NULL,
+  target_geoms INTEGER DEFAULT NULL)
+RETURNS JSON
+AS $$
+DECLARE
+  result JSON;
+BEGIN
+  SELECT c.result
+    INTO result
+    FROM cdb_observatory.OBS_CachedMeta c
+   WHERE c.z = zoom
+     AND c.parameters = getmeta_parameters::TEXT
+     AND c.num_timespans = num_timespan_options
+     AND c.num_scores = num_score_options
+     AND c.num_target_geoms = target_geoms;
+
+  IF result IS NULL THEN
+    result := cdb_observatory.obs_getmeta(geom, getmeta_parameters, num_timespan_options, num_score_options, target_geoms);
+
+    INSERT INTO cdb_observatory.OBS_CachedMeta(z, parameters, num_timespans, num_scores, num_target_geoms, result)
+    SELECT zoom, getmeta_parameters::TEXT, num_timespan_options, num_score_options, target_geoms, result
+    ON CONFLICT (z, parameters, num_timespans, num_scores, num_target_geoms) 
+    DO UPDATE SET result = EXCLUDED.result;
+  END IF;
+
+  return result;
+END
+$$ LANGUAGE plpgsql PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetMCDOMVT(
+  z INTEGER, x INTEGER, y INTEGER,
+  geography_level TEXT,
+  do_measurements TEXT[],
+  mastercard_measurements TEXT[],
+  use_meta_cache BOOLEAN DEFAULT True,
+  shoreline_clipped BOOLEAN DEFAULT True,
+  optimize_clipping BOOLEAN DEFAULT False,
+  simplify_geometries BOOLEAN DEFAULT False,
+  area_normalized BOOLEAN DEFAULT False,
+  mastercard_category TEXT DEFAULT 'Total Retail',
+  extent INTEGER DEFAULT 4096,
+  buf INTEGER DEFAULT 256,
+  clip_geom BOOLEAN DEFAULT True)
 RETURNS TABLE (
   mvtgeom GEOMETRY,
   mvtdata JSONB
@@ -317,7 +366,11 @@ BEGIN
   END LOOP;
   getmeta_parameters := substring(getmeta_parameters from 1 for length(getmeta_parameters) - 1) || ' ]';
 
-  meta := cdb_observatory.obs_getmeta(geom, getmeta_parameters::json, 1::integer, 1::integer, 1::integer);
+  IF use_meta_cache THEN
+    meta := cdb_observatory.OBS_RetrieveMeta(z, geom, getmeta_parameters::json, 1::integer, 1::integer, 1::integer);
+  ELSE
+    meta := cdb_observatory.obs_getmeta(geom, getmeta_parameters::json, 1::integer, 1::integer, 1::integer);
+  END IF;
 
   IF meta IS NOT NULL THEN
     SELECT  array_agg(distinct 'observatory.'||numer_tablename) numer_tablenames,

From 865c1a38c338acd733783a6b6c9e55331d4da854 Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Wed, 13 Jun 2018 12:53:44 +0200
Subject: [PATCH 30/45] Added plpythonu

---
 .travis.yml                    | 1 +
 scripts/ci/install_postgres.sh | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index a7a3cbf..d477a07 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -14,6 +14,7 @@ matrix:
 
 before_install:
   - sudo bash $TRAVIS_BUILD_DIR/scripts/ci/install_postgres.sh
+  - sudo make clean-all
 
 install:
   - sudo make install
diff --git a/scripts/ci/install_postgres.sh b/scripts/ci/install_postgres.sh
index 9ea6676..561e050 100644
--- a/scripts/ci/install_postgres.sh
+++ b/scripts/ci/install_postgres.sh
@@ -32,6 +32,8 @@ apt-get -y install postgresql-${PGSQL_VERSION} postgresql-${PGSQL_VERSION}-postg
 # Configure it to accept local connections from postgres
 echo -e "# TYPE  DATABASE        USER            ADDRESS                 METHOD \nlocal   all             postgres                                trust\nlocal   all             all                                     trust\nhost    all             all             127.0.0.1/32            trust" > /etc/postgresql/${PGSQL_VERSION}/main/pg_hba.conf
 
+psql -c  "CREATE LANGUAGE plpythonu;"
+
 # Restart PostgreSQL
 /etc/init.d/postgresql restart ${PGSQL_VERSION}
 

From f148057e459de2e0dae63669257caad4f83e0df7 Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Wed, 13 Jun 2018 13:08:50 +0200
Subject: [PATCH 31/45] Added plpythonu

---
 scripts/ci/install_postgres.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/ci/install_postgres.sh b/scripts/ci/install_postgres.sh
index 561e050..b80f25b 100644
--- a/scripts/ci/install_postgres.sh
+++ b/scripts/ci/install_postgres.sh
@@ -32,7 +32,7 @@ apt-get -y install postgresql-${PGSQL_VERSION} postgresql-${PGSQL_VERSION}-postg
 # Configure it to accept local connections from postgres
 echo -e "# TYPE  DATABASE        USER            ADDRESS                 METHOD \nlocal   all             postgres                                trust\nlocal   all             all                                     trust\nhost    all             all             127.0.0.1/32            trust" > /etc/postgresql/${PGSQL_VERSION}/main/pg_hba.conf
 
-psql -c  "CREATE LANGUAGE plpythonu;"
+psql -U postgres -c  "CREATE LANGUAGE plpythonu;"
 
 # Restart PostgreSQL
 /etc/init.d/postgresql restart ${PGSQL_VERSION}

From 0f9c53ddc2fe0502a63aee8eae17460138a02983 Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Wed, 13 Jun 2018 13:43:23 +0200
Subject: [PATCH 32/45] Added plpythonu

---
 scripts/ci/install_postgres.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/ci/install_postgres.sh b/scripts/ci/install_postgres.sh
index b80f25b..391971d 100644
--- a/scripts/ci/install_postgres.sh
+++ b/scripts/ci/install_postgres.sh
@@ -27,7 +27,7 @@ done
 apt-get -y autoremove
 
 # Install PostgreSQL
-apt-get -y install postgresql-${PGSQL_VERSION} postgresql-${PGSQL_VERSION}-postgis-${POSTGIS_VERSION} postgresql-server-dev-${PGSQL_VERSION}
+apt-get -y install postgresql-${PGSQL_VERSION} postgresql-${PGSQL_VERSION}-postgis-${POSTGIS_VERSION} postgresql-server-dev-${PGSQL_VERSION} postgresql-plpython-${PGSQL_VERSION}
 
 # Configure it to accept local connections from postgres
 echo -e "# TYPE  DATABASE        USER            ADDRESS                 METHOD \nlocal   all             postgres                                trust\nlocal   all             all                                     trust\nhost    all             all             127.0.0.1/32            trust" > /etc/postgresql/${PGSQL_VERSION}/main/pg_hba.conf

From e72b4ea7b651adb2702df2a49d861a213bd5a59f Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Wed, 13 Jun 2018 14:03:42 +0200
Subject: [PATCH 33/45] Added plpythonu

---
 scripts/ci/install_postgres.sh           | 2 --
 src/pg/test/expected/01_install_test.out | 1 +
 src/pg/test/sql/01_install_test.sql      | 1 +
 3 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/ci/install_postgres.sh b/scripts/ci/install_postgres.sh
index 391971d..1b3929f 100644
--- a/scripts/ci/install_postgres.sh
+++ b/scripts/ci/install_postgres.sh
@@ -32,8 +32,6 @@ apt-get -y install postgresql-${PGSQL_VERSION} postgresql-${PGSQL_VERSION}-postg
 # Configure it to accept local connections from postgres
 echo -e "# TYPE  DATABASE        USER            ADDRESS                 METHOD \nlocal   all             postgres                                trust\nlocal   all             all                                     trust\nhost    all             all             127.0.0.1/32            trust" > /etc/postgresql/${PGSQL_VERSION}/main/pg_hba.conf
 
-psql -U postgres -c  "CREATE LANGUAGE plpythonu;"
-
 # Restart PostgreSQL
 /etc/init.d/postgresql restart ${PGSQL_VERSION}
 
diff --git a/src/pg/test/expected/01_install_test.out b/src/pg/test/expected/01_install_test.out
index d6e253f..213e9c2 100644
--- a/src/pg/test/expected/01_install_test.out
+++ b/src/pg/test/expected/01_install_test.out
@@ -1,5 +1,6 @@
 -- Install dependencies
 CREATE EXTENSION postgis;
+CREATE LANGUAGE plpythonu;
 -- Install the extension
 CREATE EXTENSION observatory VERSION 'dev';
 \i test/fixtures/load_fixtures.sql
diff --git a/src/pg/test/sql/01_install_test.sql b/src/pg/test/sql/01_install_test.sql
index 5c2dbf3..fe536ca 100644
--- a/src/pg/test/sql/01_install_test.sql
+++ b/src/pg/test/sql/01_install_test.sql
@@ -1,5 +1,6 @@
 -- Install dependencies
 CREATE EXTENSION postgis;
+CREATE LANGUAGE plpythonu;
 
 -- Install the extension
 CREATE EXTENSION observatory VERSION 'dev';

From f5d6716acc119d5d13cfbf5deeea6c05688f5b20 Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Wed, 13 Jun 2018 14:24:04 +0200
Subject: [PATCH 34/45] Added plpythonu

---
 .travis.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index d477a07..c28e5ea 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,7 +8,6 @@ env:
 
 matrix:
   include:
-    - env: PGSQL_VERSION=9.5 POSTGIS_VERSION=2.2
     - env: PGSQL_VERSION=9.6 POSTGIS_VERSION=2.3
     - env: PGSQL_VERSION=10 POSTGIS_VERSION=2.4
 

From 56a83f3eb51fc3231bfc30b45db4ebc94a7e0de3 Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Thu, 14 Jun 2018 14:51:13 +0200
Subject: [PATCH 35/45] Added multiple categories option

---
 src/pg/sql/45_observatory_mvt.sql | 91 ++++++++++++++++++++-----------
 1 file changed, 60 insertions(+), 31 deletions(-)

diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql
index 8a6185e..2d881d4 100644
--- a/src/pg/sql/45_observatory_mvt.sql
+++ b/src/pg/sql/45_observatory_mvt.sql
@@ -263,17 +263,30 @@ BEGIN
 END
 $$ LANGUAGE plpgsql PARALLEL RESTRICTED;
 
+CREATE OR REPLACE FUNCTION cdb_observatory.OBS_DecodeCategory(category TEXT)
+RETURNS TEXT
+AS $$
+categories = {
+    'NEP': 'non eating places',
+	  'EP': 'eating places',
+	  'APP': 'apparel',
+	  'SB': 'small business',
+	  'TR': 'total retail',
+}
+return categories.get(category)
+$$ LANGUAGE plpythonu;
+
 CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetMCDOMVT(
   z INTEGER, x INTEGER, y INTEGER,
   geography_level TEXT,
   do_measurements TEXT[],
-  mastercard_measurements TEXT[],
+  mc_measurements TEXT[],
+  mc_categories TEXT[] DEFAULT ARRAY['TR']::TEXT[],
   use_meta_cache BOOLEAN DEFAULT True,
   shoreline_clipped BOOLEAN DEFAULT True,
   optimize_clipping BOOLEAN DEFAULT False,
   simplify_geometries BOOLEAN DEFAULT False,
   area_normalized BOOLEAN DEFAULT False,
-  mastercard_category TEXT DEFAULT 'Total Retail',
   extent INTEGER DEFAULT 4096,
   buf INTEGER DEFAULT 256,
   clip_geom BOOLEAN DEFAULT True)
@@ -289,10 +302,12 @@ DECLARE
   blockgroup_geoname CONSTANT TEXT DEFAULT 'us.census.tiger.block_group';
   block_geoname CONSTANT TEXT DEFAULT 'us.census.tiger.block';
 
-  mastercard_schema CONSTANT TEXT DEFAULT 'us.mastercard';
-  mastercard_geoid CONSTANT TEXT DEFAULT 'region_id';
-  mastercard_category_column CONSTANT TEXT DEFAULT 'category';
-  mastercard_table TEXT;
+  mc_schema CONSTANT TEXT DEFAULT 'us.mastercard';
+  mc_geoid CONSTANT TEXT DEFAULT 'region_id';
+  mc_category_column CONSTANT TEXT DEFAULT 'category';
+  mc_table TEXT;
+  mc_category TEXT;
+  mc_table_categories TEXT;
 
   bounds NUMERIC[];
   geom GEOMETRY;
@@ -311,13 +326,18 @@ DECLARE
   numer_colnames_do_qualified TEXT DEFAULT '';
   numer_colnames_do_normalized TEXT DEFAULT '';
   numer_colnames_mc TEXT;
+  numer_colnames_mc_current TEXT;
+  numer_colnames_mc_qualified TEXT;
+  numer_colnames_mc_qualified_current TEXT;
   numer_colnames_mc_normalized TEXT;
+  numer_colnames_mc_normalized_current TEXT;
   geom_tablenames TEXT;
   geom_colnames TEXT;
   geom_geomref_colnames TEXT;
   geom_geomref_colnames_qualified TEXT;
   geom_relations_do TEXT[] DEFAULT ARRAY['']::TEXT[];
   geom_relations_mc TEXT;
+  geom_mc_outerjoins TEXT;
 
   simplification_tolerance NUMERIC DEFAULT 0;
   area_normalization TEXT DEFAULT '';
@@ -421,7 +441,7 @@ BEGIN
             geom_type TEXT, numer_timespan TEXT, geom_timespan TEXT, normalization TEXT, api_method TEXT, api_args JSON);
   END IF;
 
-  ---------MasterCard---------
+  ---------MC---------
   IF geography_level = 'us.census.tiger.census_tract' THEN
     mc_geography_level := 'tract';
   ELSE
@@ -429,50 +449,59 @@ BEGIN
   END IF;
 
   SELECT tablename from pg_tables
-    INTO mastercard_table
-   WHERE schemaname = mastercard_schema
+    INTO mc_table
+   WHERE schemaname = mc_schema
      AND tablename LIKE '%'||mc_geography_level||'%';
 
-  SELECT string_agg(column_name, ','), string_agg(distinct column_name||area_normalization||' '||column_name, ',')
-    INTO numer_colnames_mc, numer_colnames_mc_normalized
-    FROM information_schema.columns
-   WHERE table_schema = mastercard_schema
-     AND table_name = mastercard_table
-     AND column_name = ANY(mastercard_measurements);
+  FOREACH mc_category IN ARRAY mc_categories LOOP
+    SELECT string_agg(column_name||'_'||mc_category, ','),
+           string_agg(mc_table||'_'||mc_category||'.'||column_name||' '||column_name||'_'||mc_category, ','),
+           string_agg(distinct column_name||'_'||mc_category||area_normalization||' '||column_name||'_'||mc_category, ',')
+      INTO numer_colnames_mc_current, numer_colnames_mc_qualified_current, numer_colnames_mc_normalized_current
+      FROM information_schema.columns
+    WHERE table_schema = mc_schema
+      AND table_name = mc_table
+      AND column_name = ANY(mc_measurements);
 
-  numer_tablenames_mc := '"'||mastercard_schema||'".'||mastercard_table;
-  geom_relations_mc := mastercard_table||'.'||mastercard_geoid||'='||geom_geomref_colnames_qualified;
+    IF numer_colnames_mc_current IS NULL OR numer_colnames_mc_qualified_current IS NULL OR numer_colnames_mc_normalized_current IS NULL THEN
+      RETURN;
+    END IF;
 
-  IF numer_colnames_mc IS NULL OR numer_colnames_mc_normalized IS NULL
-    OR numer_tablenames_mc IS NULL OR geom_relations_mc IS NULL THEN
-    RETURN;
-  END IF;
+    numer_colnames_mc := coalesce(numer_colnames_mc, '')||numer_colnames_mc_current||',';
+    numer_colnames_mc_qualified := coalesce(numer_colnames_mc_qualified, '')||numer_colnames_mc_qualified_current||',';
+    numer_colnames_mc_normalized := coalesce(numer_colnames_mc_normalized, '')||numer_colnames_mc_normalized_current||',';
+
+    numer_tablenames_mc := '"'||mc_schema||'".'||mc_table||' '||mc_table||'_'||mc_category;
+    geom_relations_mc := mc_table||'_'||mc_category||'.'||mc_geoid||'='||geom_geomref_colnames_qualified;
+    mc_table_categories := mc_table||'_'||mc_category||'.'||mc_category_column||'='''||cdb_observatory.OBS_DecodeCategory(mc_category)||'''';
+
+    geom_mc_outerjoins := coalesce(geom_mc_outerjoins, '')||' LEFT OUTER JOIN '||numer_tablenames_mc||' ON '||geom_relations_mc||' AND '||mc_table_categories;
+  END LOOP;
 
   ---------Query build and execution---------
   RETURN QUERY EXECUTE format(
     $query$
     SELECT  mvtgeom,
-            (select row_to_json(_)::jsonb from (select id, %13$s %3$s, area_ratio) as _) as mvtdata
+            (select row_to_json(_)::jsonb from (select id, %9$s %3$s area_ratio) as _) as mvtdata
           FROM (
-      SELECT ST_AsMVTGeom(ST_Transform(the_geom, 3857), $1, $2, $3, $4) AS mvtgeom, %12$s as id, %10$s %11$s, area_ratio FROM (
-        SELECT  %1$s the_geom, %12$s, %2$s %3$s,
+      SELECT ST_AsMVTGeom(ST_Transform(the_geom, 3857), $1, $2, $3, $4) AS mvtgeom, %8$s as id, %6$s %7$s area_ratio FROM (
+        SELECT  %1$s the_geom, %8$s, %2$s %10$s
                 CASE  WHEN ST_Within($5, %1$s)
                         THEN ST_Area($5) / Nullif(ST_Area(%1$s), 0)
                       WHEN ST_Within(%1$s, $5)
                         THEN 1
-                      ELSE ST_Area(cdb_observatory.safe_intersection(st_simplifyvw(%1$s, $7), $5)) / Nullif(ST_Area(%1$s), 0)
+                      ELSE ST_Area(cdb_observatory.safe_intersection(st_simplifyvw(%1$s, $6), $5)) / Nullif(ST_Area(%1$s), 0)
                 END area_ratio
-          FROM %6$s
+          FROM %5$s
                %4$s
-               LEFT OUTER JOIN %5$s ON %7$s AND %8$s.%9$s=$6
+               %11$s
         WHERE st_intersects(%1$s, $5)
       ) p
     ) q
     $query$,
-    geom_colnames, numer_colnames_do_qualified, numer_colnames_mc, numer_tablenames_do_outer, numer_tablenames_mc,
-    geom_tablenames, geom_relations_mc, mastercard_table, mastercard_category_column, numer_colnames_do_normalized,
-    numer_colnames_mc_normalized, geom_geomref_colnames, numer_colnames_do)
-  USING ext, extent, buf, clip_geom, geom, mastercard_category, simplification_tolerance
+    geom_colnames, numer_colnames_do_qualified, numer_colnames_mc, numer_tablenames_do_outer, geom_tablenames, numer_colnames_do_normalized,
+    numer_colnames_mc_normalized, geom_geomref_colnames, numer_colnames_do, numer_colnames_mc_qualified, geom_mc_outerjoins)
+  USING ext, extent, buf, clip_geom, geom, simplification_tolerance
   RETURN;
 END
 $$ LANGUAGE plpgsql PARALLEL RESTRICTED;

From 3172a11d9d64a6c5401ced37e5e0abceb61232f0 Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Thu, 14 Jun 2018 16:05:20 +0200
Subject: [PATCH 36/45] Fixed categories

---
 src/pg/sql/45_observatory_mvt.sql | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql
index 2d881d4..887531c 100644
--- a/src/pg/sql/45_observatory_mvt.sql
+++ b/src/pg/sql/45_observatory_mvt.sql
@@ -267,11 +267,11 @@ CREATE OR REPLACE FUNCTION cdb_observatory.OBS_DecodeCategory(category TEXT)
 RETURNS TEXT
 AS $$
 categories = {
-    'NEP': 'non eating places',
-	  'EP': 'eating places',
+    'NEP': 'non_eating_places',
+	  'EP': 'eating_places',
 	  'APP': 'apparel',
-	  'SB': 'small business',
-	  'TR': 'total retail',
+	  'SB': 'small_business',
+	  'TR': 'total_retail',
 }
 return categories.get(category)
 $$ LANGUAGE plpythonu;

From 4e57558da64badd6e4669f1c62ad5cecdbffd2ac Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Thu, 14 Jun 2018 16:29:36 +0200
Subject: [PATCH 37/45] Fixed categories

---
 src/pg/sql/45_observatory_mvt.sql | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql
index 887531c..2d881d4 100644
--- a/src/pg/sql/45_observatory_mvt.sql
+++ b/src/pg/sql/45_observatory_mvt.sql
@@ -267,11 +267,11 @@ CREATE OR REPLACE FUNCTION cdb_observatory.OBS_DecodeCategory(category TEXT)
 RETURNS TEXT
 AS $$
 categories = {
-    'NEP': 'non_eating_places',
-	  'EP': 'eating_places',
+    'NEP': 'non eating places',
+	  'EP': 'eating places',
 	  'APP': 'apparel',
-	  'SB': 'small_business',
-	  'TR': 'total_retail',
+	  'SB': 'small business',
+	  'TR': 'total retail',
 }
 return categories.get(category)
 $$ LANGUAGE plpythonu;

From c6ef9b7f3a70d4bd45e6fc10b47ec9544c6cf192 Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Fri, 15 Jun 2018 11:19:07 +0200
Subject: [PATCH 38/45] Added filter by months

---
 src/pg/sql/45_observatory_mvt.sql | 45 ++++++++++++++++++-------------
 1 file changed, 27 insertions(+), 18 deletions(-)

diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql
index 2d881d4..52a8db4 100644
--- a/src/pg/sql/45_observatory_mvt.sql
+++ b/src/pg/sql/45_observatory_mvt.sql
@@ -282,6 +282,7 @@ CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetMCDOMVT(
   do_measurements TEXT[],
   mc_measurements TEXT[],
   mc_categories TEXT[] DEFAULT ARRAY['TR']::TEXT[],
+  mc_months TEXT[] DEFAULT ARRAY['2018-02-01']::TEXT[],
   use_meta_cache BOOLEAN DEFAULT True,
   shoreline_clipped BOOLEAN DEFAULT True,
   optimize_clipping BOOLEAN DEFAULT False,
@@ -305,9 +306,12 @@ DECLARE
   mc_schema CONSTANT TEXT DEFAULT 'us.mastercard';
   mc_geoid CONSTANT TEXT DEFAULT 'region_id';
   mc_category_column CONSTANT TEXT DEFAULT 'category';
+  mc_month_column CONSTANT TEXT DEFAULT 'month';
   mc_table TEXT;
   mc_category TEXT;
   mc_table_categories TEXT;
+  mc_month TEXT;
+  mc_month_slug TEXT;
 
   bounds NUMERIC[];
   geom GEOMETRY;
@@ -454,28 +458,33 @@ BEGIN
      AND tablename LIKE '%'||mc_geography_level||'%';
 
   FOREACH mc_category IN ARRAY mc_categories LOOP
-    SELECT string_agg(column_name||'_'||mc_category, ','),
-           string_agg(mc_table||'_'||mc_category||'.'||column_name||' '||column_name||'_'||mc_category, ','),
-           string_agg(distinct column_name||'_'||mc_category||area_normalization||' '||column_name||'_'||mc_category, ',')
-      INTO numer_colnames_mc_current, numer_colnames_mc_qualified_current, numer_colnames_mc_normalized_current
-      FROM information_schema.columns
-    WHERE table_schema = mc_schema
-      AND table_name = mc_table
-      AND column_name = ANY(mc_measurements);
+    FOREACH mc_month IN ARRAY mc_months LOOP
+      mc_month_slug := replace(mc_month, '-', '');
 
-    IF numer_colnames_mc_current IS NULL OR numer_colnames_mc_qualified_current IS NULL OR numer_colnames_mc_normalized_current IS NULL THEN
-      RETURN;
-    END IF;
+      SELECT string_agg(column_name||'_'||mc_category||'_'||mc_month_slug, ','),
+            string_agg(mc_table||'_'||mc_category||'_'||mc_month_slug||'.'||column_name||' '||column_name||'_'||mc_category||'_'||mc_month_slug, ','),
+            string_agg(distinct column_name||'_'||mc_category||'_'||mc_month_slug||area_normalization||' '||column_name||'_'||mc_category||'_'||mc_month_slug, ',')
+        INTO numer_colnames_mc_current, numer_colnames_mc_qualified_current, numer_colnames_mc_normalized_current
+        FROM information_schema.columns
+      WHERE table_schema = mc_schema
+        AND table_name = mc_table
+        AND column_name = ANY(mc_measurements);
 
-    numer_colnames_mc := coalesce(numer_colnames_mc, '')||numer_colnames_mc_current||',';
-    numer_colnames_mc_qualified := coalesce(numer_colnames_mc_qualified, '')||numer_colnames_mc_qualified_current||',';
-    numer_colnames_mc_normalized := coalesce(numer_colnames_mc_normalized, '')||numer_colnames_mc_normalized_current||',';
+      IF numer_colnames_mc_current IS NULL OR numer_colnames_mc_qualified_current IS NULL OR numer_colnames_mc_normalized_current IS NULL THEN
+        RETURN;
+      END IF;
 
-    numer_tablenames_mc := '"'||mc_schema||'".'||mc_table||' '||mc_table||'_'||mc_category;
-    geom_relations_mc := mc_table||'_'||mc_category||'.'||mc_geoid||'='||geom_geomref_colnames_qualified;
-    mc_table_categories := mc_table||'_'||mc_category||'.'||mc_category_column||'='''||cdb_observatory.OBS_DecodeCategory(mc_category)||'''';
+      numer_colnames_mc := coalesce(numer_colnames_mc, '')||numer_colnames_mc_current||',';
+      numer_colnames_mc_qualified := coalesce(numer_colnames_mc_qualified, '')||numer_colnames_mc_qualified_current||',';
+      numer_colnames_mc_normalized := coalesce(numer_colnames_mc_normalized, '')||numer_colnames_mc_normalized_current||',';
 
-    geom_mc_outerjoins := coalesce(geom_mc_outerjoins, '')||' LEFT OUTER JOIN '||numer_tablenames_mc||' ON '||geom_relations_mc||' AND '||mc_table_categories;
+      numer_tablenames_mc := '"'||mc_schema||'".'||mc_table||' '||mc_table||'_'||mc_category||'_'||mc_month_slug;
+      geom_relations_mc := mc_table||'_'||mc_category||'_'||mc_month_slug||'.'||mc_geoid||'='||geom_geomref_colnames_qualified;
+      mc_table_categories := mc_table||'_'||mc_category||'_'||mc_month_slug||'.'||mc_category_column||'='''||cdb_observatory.OBS_DecodeCategory(mc_category)||''''||
+                             ' AND '||mc_table||'_'||mc_category||'_'||mc_month_slug||'.'||mc_month_column||'='''||mc_month||'''';
+
+      geom_mc_outerjoins := coalesce(geom_mc_outerjoins, '')||' LEFT OUTER JOIN '||numer_tablenames_mc||' ON '||geom_relations_mc||' AND '||mc_table_categories;
+    END LOOP;
   END LOOP;
 
   ---------Query build and execution---------

From b9791671569b292fbd0a72c53b07959ae0a8a929 Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Tue, 26 Jun 2018 14:39:04 +0200
Subject: [PATCH 39/45] Added function to get dates from a given month

---
 src/pg/sql/45_observatory_mvt.sql | 57 +++++++++++++++++++++++++++----
 1 file changed, 51 insertions(+), 6 deletions(-)

diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql
index 52a8db4..5f02417 100644
--- a/src/pg/sql/45_observatory_mvt.sql
+++ b/src/pg/sql/45_observatory_mvt.sql
@@ -263,7 +263,7 @@ BEGIN
 END
 $$ LANGUAGE plpgsql PARALLEL RESTRICTED;
 
-CREATE OR REPLACE FUNCTION cdb_observatory.OBS_DecodeCategory(category TEXT)
+CREATE OR REPLACE FUNCTION cdb_observatory.OBS_DecodeMCCategory(category TEXT)
 RETURNS TEXT
 AS $$
 categories = {
@@ -276,6 +276,54 @@ categories = {
 return categories.get(category)
 $$ LANGUAGE plpythonu;
 
+CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetMCDates(
+  mc_schema TEXT,
+  geo_level TEXT,
+  month_no TEXT DEFAULT NULL)
+RETURNS TEXT[]
+AS $$
+DECLARE
+  mc_table TEXT;
+  where_clause TEXT DEFAULT '';
+  dates TEXT[];
+BEGIN
+  mc_table := cdb_observatory.OBS_GetMCTable(mc_schema, geo_level);
+
+  IF month_no IS NOT NULL THEN
+    where_clause := format(
+      $query$
+      WHERE month LIKE '____-%1$s-__'
+      $query$, LPAD(month_no, 2, '0'));
+  END IF;
+
+  EXECUTE
+  format(
+    $query$
+    SELECT ARRAY_AGG(DISTINCT month) dates
+      FROM "%1$s".%2$s
+      %3$s
+    $query$, mc_schema, mc_table, where_clause)
+  INTO dates;
+
+  RETURN dates;
+END
+$$ LANGUAGE plpgsql PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetMCTable(mc_schema TEXT, geo_level TEXT)
+RETURNS TEXT
+AS $$
+DECLARE
+  mc_table TEXT;
+BEGIN
+  SELECT tablename from pg_tables
+    INTO mc_table
+   WHERE schemaname = mc_schema
+     AND tablename LIKE '%'||geo_level||'%';
+
+  RETURN mc_table;
+END
+$$ LANGUAGE plpgsql PARALLEL RESTRICTED;
+
 CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetMCDOMVT(
   z INTEGER, x INTEGER, y INTEGER,
   geography_level TEXT,
@@ -452,10 +500,7 @@ BEGIN
     mc_geography_level := (string_to_array(geography_level, '.'))[array_length(string_to_array(geography_level, '.'), 1)];
   END IF;
 
-  SELECT tablename from pg_tables
-    INTO mc_table
-   WHERE schemaname = mc_schema
-     AND tablename LIKE '%'||mc_geography_level||'%';
+  mc_table := cdb_observatory.OBS_GetMCTable(mc_schema, mc_geography_level);
 
   FOREACH mc_category IN ARRAY mc_categories LOOP
     FOREACH mc_month IN ARRAY mc_months LOOP
@@ -480,7 +525,7 @@ BEGIN
 
       numer_tablenames_mc := '"'||mc_schema||'".'||mc_table||' '||mc_table||'_'||mc_category||'_'||mc_month_slug;
       geom_relations_mc := mc_table||'_'||mc_category||'_'||mc_month_slug||'.'||mc_geoid||'='||geom_geomref_colnames_qualified;
-      mc_table_categories := mc_table||'_'||mc_category||'_'||mc_month_slug||'.'||mc_category_column||'='''||cdb_observatory.OBS_DecodeCategory(mc_category)||''''||
+      mc_table_categories := mc_table||'_'||mc_category||'_'||mc_month_slug||'.'||mc_category_column||'='''||cdb_observatory.OBS_DecodeMCCategory(mc_category)||''''||
                              ' AND '||mc_table||'_'||mc_category||'_'||mc_month_slug||'.'||mc_month_column||'='''||mc_month||'''';
 
       geom_mc_outerjoins := coalesce(geom_mc_outerjoins, '')||' LEFT OUTER JOIN '||numer_tablenames_mc||' ON '||geom_relations_mc||' AND '||mc_table_categories;

From d932be1a9fa2340ac87f6f8615e630b8ac222074 Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Wed, 27 Jun 2018 15:52:25 +0200
Subject: [PATCH 40/45] MC measurements are not required + areas

---
 src/pg/sql/45_observatory_mvt.sql | 55 +++++++++++++++++--------------
 1 file changed, 30 insertions(+), 25 deletions(-)

diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql
index 5f02417..ab2f30e 100644
--- a/src/pg/sql/45_observatory_mvt.sql
+++ b/src/pg/sql/45_observatory_mvt.sql
@@ -357,7 +357,7 @@ DECLARE
   mc_month_column CONSTANT TEXT DEFAULT 'month';
   mc_table TEXT;
   mc_category TEXT;
-  mc_table_categories TEXT;
+  mc_table_categories TEXT DEFAULT '';
   mc_month TEXT;
   mc_month_slug TEXT;
 
@@ -373,23 +373,23 @@ DECLARE
   numer_tablename_do TEXT DEFAULT '';
   numer_tablenames_do TEXT[] DEFAULT ARRAY['']::TEXT[];
   numer_tablenames_do_outer TEXT DEFAULT '';
-  numer_tablenames_mc TEXT;
+  numer_tablenames_mc TEXT DEFAULT '';
   numer_colnames_do TEXT DEFAULT '';
   numer_colnames_do_qualified TEXT DEFAULT '';
   numer_colnames_do_normalized TEXT DEFAULT '';
-  numer_colnames_mc TEXT;
-  numer_colnames_mc_current TEXT;
-  numer_colnames_mc_qualified TEXT;
-  numer_colnames_mc_qualified_current TEXT;
-  numer_colnames_mc_normalized TEXT;
-  numer_colnames_mc_normalized_current TEXT;
+  numer_colnames_mc TEXT DEFAULT '';
+  numer_colnames_mc_current TEXT DEFAULT '';
+  numer_colnames_mc_qualified TEXT DEFAULT '';
+  numer_colnames_mc_qualified_current TEXT DEFAULT '';
+  numer_colnames_mc_normalized TEXT DEFAULT '';
+  numer_colnames_mc_normalized_current TEXT DEFAULT '';
   geom_tablenames TEXT;
   geom_colnames TEXT;
   geom_geomref_colnames TEXT;
   geom_geomref_colnames_qualified TEXT;
   geom_relations_do TEXT[] DEFAULT ARRAY['']::TEXT[];
-  geom_relations_mc TEXT;
-  geom_mc_outerjoins TEXT;
+  geom_relations_mc TEXT DEFAULT '';
+  geom_mc_outerjoins TEXT DEFAULT '';
 
   simplification_tolerance NUMERIC DEFAULT 0;
   area_normalization TEXT DEFAULT '';
@@ -419,7 +419,7 @@ BEGIN
     WHEN geography_level = block_geoname THEN
       simplification_tolerance := 0.0001;
     ELSE
-      RETURN;
+      simplification_tolerance := 0;
   END CASE;
 
   IF NOT simplify_geometries THEN
@@ -515,20 +515,24 @@ BEGIN
         AND table_name = mc_table
         AND column_name = ANY(mc_measurements);
 
-      IF numer_colnames_mc_current IS NULL OR numer_colnames_mc_qualified_current IS NULL OR numer_colnames_mc_normalized_current IS NULL THEN
-        RETURN;
+      IF numer_colnames_mc_current IS NOT NULL THEN
+        numer_colnames_mc := coalesce(numer_colnames_mc, '')||numer_colnames_mc_current||',';
+      END IF;
+      IF numer_colnames_mc_qualified_current IS NOT NULL THEN
+        numer_colnames_mc_qualified := coalesce(numer_colnames_mc_qualified, '')||numer_colnames_mc_qualified_current||',';
+      END IF;
+      IF numer_colnames_mc_normalized_current IS NOT NULL THEN
+        numer_colnames_mc_normalized := coalesce(numer_colnames_mc_normalized, '')||numer_colnames_mc_normalized_current||',';
       END IF;
 
-      numer_colnames_mc := coalesce(numer_colnames_mc, '')||numer_colnames_mc_current||',';
-      numer_colnames_mc_qualified := coalesce(numer_colnames_mc_qualified, '')||numer_colnames_mc_qualified_current||',';
-      numer_colnames_mc_normalized := coalesce(numer_colnames_mc_normalized, '')||numer_colnames_mc_normalized_current||',';
+      IF mc_table IS NOT NULL THEN
+        numer_tablenames_mc := '"'||mc_schema||'".'||mc_table||' '||mc_table||'_'||mc_category||'_'||mc_month_slug;
+        geom_relations_mc := mc_table||'_'||mc_category||'_'||mc_month_slug||'.'||mc_geoid||'='||geom_geomref_colnames_qualified;
+        mc_table_categories := mc_table||'_'||mc_category||'_'||mc_month_slug||'.'||mc_category_column||'='''||cdb_observatory.OBS_DecodeMCCategory(mc_category)||''''||
+                              ' AND '||mc_table||'_'||mc_category||'_'||mc_month_slug||'.'||mc_month_column||'='''||mc_month||'''';
 
-      numer_tablenames_mc := '"'||mc_schema||'".'||mc_table||' '||mc_table||'_'||mc_category||'_'||mc_month_slug;
-      geom_relations_mc := mc_table||'_'||mc_category||'_'||mc_month_slug||'.'||mc_geoid||'='||geom_geomref_colnames_qualified;
-      mc_table_categories := mc_table||'_'||mc_category||'_'||mc_month_slug||'.'||mc_category_column||'='''||cdb_observatory.OBS_DecodeMCCategory(mc_category)||''''||
-                             ' AND '||mc_table||'_'||mc_category||'_'||mc_month_slug||'.'||mc_month_column||'='''||mc_month||'''';
-
-      geom_mc_outerjoins := coalesce(geom_mc_outerjoins, '')||' LEFT OUTER JOIN '||numer_tablenames_mc||' ON '||geom_relations_mc||' AND '||mc_table_categories;
+        geom_mc_outerjoins := coalesce(geom_mc_outerjoins, '')||' LEFT OUTER JOIN '||numer_tablenames_mc||' ON '||geom_relations_mc||' AND '||mc_table_categories;
+      END IF;
     END LOOP;
   END LOOP;
 
@@ -536,16 +540,17 @@ BEGIN
   RETURN QUERY EXECUTE format(
     $query$
     SELECT  mvtgeom,
-            (select row_to_json(_)::jsonb from (select id, %9$s %3$s area_ratio) as _) as mvtdata
+            (select row_to_json(_)::jsonb from (select id, %9$s %3$s area_ratio, area) as _) as mvtdata
           FROM (
-      SELECT ST_AsMVTGeom(ST_Transform(the_geom, 3857), $1, $2, $3, $4) AS mvtgeom, %8$s as id, %6$s %7$s area_ratio FROM (
+      SELECT ST_AsMVTGeom(ST_Transform(the_geom, 3857), $1, $2, $3, $4) AS mvtgeom, %8$s as id, %6$s %7$s area_ratio, area FROM (
         SELECT  %1$s the_geom, %8$s, %2$s %10$s
                 CASE  WHEN ST_Within($5, %1$s)
                         THEN ST_Area($5) / Nullif(ST_Area(%1$s), 0)
                       WHEN ST_Within(%1$s, $5)
                         THEN 1
                       ELSE ST_Area(cdb_observatory.safe_intersection(st_simplifyvw(%1$s, $6), $5)) / Nullif(ST_Area(%1$s), 0)
-                END area_ratio
+                END area_ratio,
+                ROUND(ST_Area(ST_Transform(the_geom,3857))::NUMERIC, 2) area
           FROM %5$s
                %4$s
                %11$s

From 0a029404c8a7f8f7c92989f7207446f5cb33c7c6 Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Wed, 27 Jun 2018 17:10:45 +0200
Subject: [PATCH 41/45] Added CR suggestion

---
 src/pg/sql/45_observatory_mvt.sql | 24 ++++++++++--------------
 1 file changed, 10 insertions(+), 14 deletions(-)

diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql
index ab2f30e..bdae0f8 100644
--- a/src/pg/sql/45_observatory_mvt.sql
+++ b/src/pg/sql/45_observatory_mvt.sql
@@ -263,19 +263,6 @@ BEGIN
 END
 $$ LANGUAGE plpgsql PARALLEL RESTRICTED;
 
-CREATE OR REPLACE FUNCTION cdb_observatory.OBS_DecodeMCCategory(category TEXT)
-RETURNS TEXT
-AS $$
-categories = {
-    'NEP': 'non eating places',
-	  'EP': 'eating places',
-	  'APP': 'apparel',
-	  'SB': 'small business',
-	  'TR': 'total retail',
-}
-return categories.get(category)
-$$ LANGUAGE plpythonu;
-
 CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetMCDates(
   mc_schema TEXT,
   geo_level TEXT,
@@ -357,6 +344,7 @@ DECLARE
   mc_month_column CONSTANT TEXT DEFAULT 'month';
   mc_table TEXT;
   mc_category TEXT;
+  mc_category_name TEXT;
   mc_table_categories TEXT DEFAULT '';
   mc_month TEXT;
   mc_month_slug TEXT;
@@ -506,6 +494,14 @@ BEGIN
     FOREACH mc_month IN ARRAY mc_months LOOP
       mc_month_slug := replace(mc_month, '-', '');
 
+      SELECT CASE
+        WHEN mc_category = 'NEP' THEN 'non eating places'
+	      WHEN mc_category = 'EP'  THEN 'eating places'
+	      WHEN mc_category = 'APP' THEN 'apparel'
+	      WHEN mc_category = 'SB'  THEN 'small business'
+	      WHEN mc_category = 'TR'  THEN 'total retail'
+        END INTO mc_category_name;
+
       SELECT string_agg(column_name||'_'||mc_category||'_'||mc_month_slug, ','),
             string_agg(mc_table||'_'||mc_category||'_'||mc_month_slug||'.'||column_name||' '||column_name||'_'||mc_category||'_'||mc_month_slug, ','),
             string_agg(distinct column_name||'_'||mc_category||'_'||mc_month_slug||area_normalization||' '||column_name||'_'||mc_category||'_'||mc_month_slug, ',')
@@ -528,7 +524,7 @@ BEGIN
       IF mc_table IS NOT NULL THEN
         numer_tablenames_mc := '"'||mc_schema||'".'||mc_table||' '||mc_table||'_'||mc_category||'_'||mc_month_slug;
         geom_relations_mc := mc_table||'_'||mc_category||'_'||mc_month_slug||'.'||mc_geoid||'='||geom_geomref_colnames_qualified;
-        mc_table_categories := mc_table||'_'||mc_category||'_'||mc_month_slug||'.'||mc_category_column||'='''||cdb_observatory.OBS_DecodeMCCategory(mc_category)||''''||
+        mc_table_categories := mc_table||'_'||mc_category||'_'||mc_month_slug||'.'||mc_category_column||'='''||mc_category_name||''''||
                               ' AND '||mc_table||'_'||mc_category||'_'||mc_month_slug||'.'||mc_month_column||'='''||mc_month||'''';
 
         geom_mc_outerjoins := coalesce(geom_mc_outerjoins, '')||' LEFT OUTER JOIN '||numer_tablenames_mc||' ON '||geom_relations_mc||' AND '||mc_table_categories;

From 96b5d84394ed396cdf5a7a567e9d3bcb01e998d4 Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Tue, 3 Jul 2018 15:01:28 +0200
Subject: [PATCH 42/45] CR suggestion

---
 src/pg/sql/45_observatory_mvt.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql
index bdae0f8..aa5c51a 100644
--- a/src/pg/sql/45_observatory_mvt.sql
+++ b/src/pg/sql/45_observatory_mvt.sql
@@ -544,7 +544,7 @@ BEGIN
                         THEN ST_Area($5) / Nullif(ST_Area(%1$s), 0)
                       WHEN ST_Within(%1$s, $5)
                         THEN 1
-                      ELSE ST_Area(cdb_observatory.safe_intersection(st_simplifyvw(%1$s, $6), $5)) / Nullif(ST_Area(%1$s), 0)
+                      ELSE ST_Area(ST_Intersection(st_simplifyvw(%1$s, $6), $5)) / Nullif(ST_Area(%1$s), 0)
                 END area_ratio,
                 ROUND(ST_Area(ST_Transform(the_geom,3857))::NUMERIC, 2) area
           FROM %5$s

From c354d48f28cb98de90f95ac137da9041274716d0 Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Tue, 3 Jul 2018 15:53:19 +0200
Subject: [PATCH 43/45] Updated NEWS.md

---
 NEWS.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/NEWS.md b/NEWS.md
index faa25cc..e9fd785 100644
--- a/NEWS.md
+++ b/NEWS.md
@@ -1,3 +1,11 @@
+1.10.0 (2018-07-??)
+-------------------
+
+__Improvements__
+
+* Added `OBS_GetMVT` function to get DO data as MVT
+* Added `OBS_GetMCDOMVT` function
+
 1.9.0 (2018-04-20)
 ------------------
 

From f4b8c398703f7594874d246c765bfad6ea9887c2 Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Thu, 12 Jul 2018 15:19:21 +0200
Subject: [PATCH 44/45] New function to return the whole table from a zoom
 level/geography

---
 src/pg/sql/45_observatory_mvt.sql | 280 ++++++++++++++++++++++++++++++
 1 file changed, 280 insertions(+)

diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql
index aa5c51a..850611b 100644
--- a/src/pg/sql/45_observatory_mvt.sql
+++ b/src/pg/sql/45_observatory_mvt.sql
@@ -560,3 +560,283 @@ BEGIN
   RETURN;
 END
 $$ LANGUAGE plpgsql PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetMCDOMVT(
+  z INTEGER,
+  geography_level TEXT,
+  do_measurements TEXT[],
+  mc_measurements TEXT[],
+  mc_categories TEXT[] DEFAULT ARRAY['TR']::TEXT[],
+  mc_months TEXT[] DEFAULT ARRAY['2018-02-01']::TEXT[],
+  use_meta_cache BOOLEAN DEFAULT True,
+  shoreline_clipped BOOLEAN DEFAULT True,
+  optimize_clipping BOOLEAN DEFAULT False,
+  simplify_geometries BOOLEAN DEFAULT False,
+  area_normalized BOOLEAN DEFAULT False,
+  extent INTEGER DEFAULT 4096,
+  buf INTEGER DEFAULT 256,
+  clip_geom BOOLEAN DEFAULT True)
+RETURNS TABLE (
+  x INTEGER,
+  y INTEGER,
+  zoom INTEGER,
+  mvtgeom GEOMETRY,
+  mvtdata JSONB
+)
+AS $$
+DECLARE
+  state_geoname CONSTANT TEXT DEFAULT 'us.census.tiger.state';
+  county_geoname CONSTANT TEXT DEFAULT 'us.census.tiger.county';
+  tract_geoname CONSTANT TEXT DEFAULT 'us.census.tiger.census_tract';
+  blockgroup_geoname CONSTANT TEXT DEFAULT 'us.census.tiger.block_group';
+  block_geoname CONSTANT TEXT DEFAULT 'us.census.tiger.block';
+
+  tiler_table_prefix CONSTANT TEXT DEFAULT 'tiler.xyz_us_mc_tiles_temp_';
+  avg_x INTEGER;
+  avg_y INTEGER;
+
+  mc_schema CONSTANT TEXT DEFAULT 'us.mastercard';
+  mc_geoid CONSTANT TEXT DEFAULT 'region_id';
+  mc_category_column CONSTANT TEXT DEFAULT 'category';
+  mc_month_column CONSTANT TEXT DEFAULT 'month';
+  mc_table TEXT;
+  mc_category TEXT;
+  mc_category_name TEXT;
+  mc_table_categories TEXT DEFAULT '';
+  mc_month TEXT;
+  mc_month_slug TEXT;
+
+  measurement TEXT;
+  getmeta_parameters TEXT;
+  meta JSON;
+  mc_geography_level TEXT;
+
+  simplification_tolerance NUMERIC DEFAULT 0;
+  area_normalization TEXT DEFAULT '';
+  clipped TEXT default '';
+  i INTEGER DEFAULT 0;
+
+  bounds NUMERIC[];
+  geom GEOMETRY;
+  ext BOX2D;
+
+  numer_tablename_do TEXT DEFAULT '';
+  numer_tablenames_do TEXT[] DEFAULT ARRAY['']::TEXT[];
+  numer_tablenames_do_outer TEXT DEFAULT '';
+  numer_tablenames_mc TEXT DEFAULT '';
+  numer_colnames_do TEXT DEFAULT '';
+  numer_colnames_do_qualified TEXT DEFAULT '';
+  numer_colnames_do_normalized TEXT DEFAULT '';
+  numer_colnames_mc TEXT DEFAULT '';
+  numer_colnames_mc_current TEXT DEFAULT '';
+  numer_colnames_mc_qualified TEXT DEFAULT '';
+  numer_colnames_mc_qualified_current TEXT DEFAULT '';
+  numer_colnames_mc_normalized TEXT DEFAULT '';
+  numer_colnames_mc_normalized_current TEXT DEFAULT '';
+  geom_tablenames TEXT;
+  geom_colnames TEXT;
+  geom_geomref_colnames TEXT;
+  geom_geomref_colnames_qualified TEXT;
+  geom_relations_do TEXT[] DEFAULT ARRAY['']::TEXT[];
+  geom_relations_mc TEXT DEFAULT '';
+  geom_mc_outerjoins TEXT DEFAULT '';
+BEGIN
+  IF geography_level = 'us.census.tiger.census_tract' THEN
+    mc_geography_level := 'tract';
+  ELSE
+    mc_geography_level := (string_to_array(geography_level, '.'))[array_length(string_to_array(geography_level, '.'), 1)];
+  END IF;
+
+  -- Get the average x and y (in the middle of the BBox)
+  EXECUTE
+  format(
+    $query$
+    SELECT ROUND(AVG(x)) AS x, ROUND(AVG(y)) as y
+    FROM tiler.xyz_us_mc_tiles_temp_%1$s_%2$s
+    $query$, mc_geography_level, z)
+  INTO avg_x, avg_y;
+
+  IF area_normalized THEN
+    area_normalization := '/area_ratio';
+  END IF;
+
+  IF shoreline_clipped THEN
+    clipped := '_clipped';
+  END IF;
+
+  CASE
+    WHEN geography_level = state_geoname THEN
+      simplification_tolerance := 0.1;
+      IF optimize_clipping THEN
+        clipped := '';
+      END IF;
+    WHEN geography_level = county_geoname THEN
+      simplification_tolerance := 0.01;
+    WHEN geography_level = tract_geoname THEN
+      simplification_tolerance := 0.001;
+    WHEN geography_level = blockgroup_geoname THEN
+      simplification_tolerance := 0.0001;
+    WHEN geography_level = block_geoname THEN
+      simplification_tolerance := 0.0001;
+    ELSE
+      simplification_tolerance := 0;
+  END CASE;
+
+  IF NOT simplify_geometries THEN
+    simplification_tolerance := 0;
+  END IF;
+
+  bounds := cdb_observatory.OBS_GetTileBounds(z, avg_x, avg_y);
+  geom := ST_MakeEnvelope(bounds[1], bounds[2], bounds[3], bounds[4], 4326);
+  ext := ST_MakeBox2D(ST_Transform(ST_SetSRID(ST_Point(bounds[1], bounds[2]), 4326), 3857),
+                      ST_Transform(ST_SetSRID(ST_Point(bounds[3], bounds[4]), 4326), 3857));
+
+  ---------DO---------
+  getmeta_parameters := '[ ';
+  FOREACH measurement IN ARRAY do_measurements LOOP
+    getmeta_parameters := getmeta_parameters || '{"numer_id":"' || measurement || '","geom_id":"' || geography_level || clipped ||'"},';
+  END LOOP;
+  getmeta_parameters := substring(getmeta_parameters from 1 for length(getmeta_parameters) - 1) || ' ]';
+
+  IF use_meta_cache THEN
+    meta := cdb_observatory.OBS_RetrieveMeta(z, geom, getmeta_parameters::json, 1::integer, 1::integer, 1::integer);
+  ELSE
+    meta := cdb_observatory.obs_getmeta(geom, getmeta_parameters::json, 1::integer, 1::integer, 1::integer);
+  END IF;
+
+  IF meta IS NOT NULL THEN
+    SELECT  array_agg(distinct 'observatory.'||numer_tablename) numer_tablenames,
+            string_agg(distinct numer_colname, ',')||',' numer_colnames,
+            string_agg(distinct numer_tablename||'.'||numer_colname, ',')||',' numer_colnames_qualified,
+            string_agg(distinct numer_colname||area_normalization||' '||numer_colname, ',')||',' numer_colnames_normalized,
+            (array_agg(distinct 'observatory.'||geom_tablename))[1] geom_tablenames,
+            (array_agg(distinct geom_colname))[1] geom_colnames,
+            (array_agg(distinct geom_geomref_colname))[1] geom_geomref_colnames,
+            (array_agg(distinct geom_tablename||'.'||geom_geomref_colname))[1] geom_geomref_colnames_qualified,
+            array_agg(distinct numer_tablename||'.'||numer_geomref_colname||'='||geom_tablename||'.'||geom_geomref_colname) geom_relations
+      INTO numer_tablenames_do, numer_colnames_do, numer_colnames_do_qualified, numer_colnames_do_normalized, geom_tablenames, geom_colnames,
+          geom_geomref_colnames, geom_geomref_colnames_qualified, geom_relations_do
+      FROM json_to_recordset(meta)
+        AS x(id TEXT, numer_id TEXT, numer_aggregate TEXT, numer_colname TEXT, numer_geomref_colname TEXT, numer_tablename TEXT,
+            numer_type TEXT, denom_id TEXT, denom_aggregate TEXT, denom_colname TEXT, denom_geomref_colname TEXT, denom_tablename TEXT,
+            denom_type TEXT, denom_reltype TEXT, geom_id TEXT, geom_colname TEXT, geom_geomref_colname TEXT, geom_tablename TEXT,
+            geom_type TEXT, numer_timespan TEXT, geom_timespan TEXT, normalization TEXT, api_method TEXT, api_args JSON);
+
+    IF numer_tablenames_do IS NULL OR numer_colnames_do IS NULL OR numer_colnames_do_qualified IS NULL OR numer_colnames_do_normalized IS NULL
+      OR geom_tablenames IS NULL OR geom_colnames IS NULL OR geom_geomref_colnames IS NULL OR geom_geomref_colnames_qualified IS NULL
+      OR geom_relations_do IS NULL THEN
+      RETURN;
+    END IF;
+
+    i := 0;
+    FOREACH numer_tablename_do IN ARRAY numer_tablenames_do LOOP
+      i := i + 1;
+      numer_tablenames_do_outer := numer_tablenames_do_outer || 'LEFT OUTER JOIN ' || numer_tablename_do || ' ON ' || geom_relations_do[i] || ' ';
+    END LOOP;
+  ELSE
+    getmeta_parameters := '[{"geom_id":"' || geography_level || clipped ||'"}]';
+    meta := cdb_observatory.obs_getmeta(geom, getmeta_parameters::json, 1::integer, 1::integer, 1::integer);
+
+    IF meta IS NULL THEN
+      RETURN;
+    END IF;
+
+    SELECT  (array_agg(distinct 'observatory.'||geom_tablename))[1] geom_tablenames,
+            (array_agg(distinct geom_colname))[1] geom_colnames,
+            (array_agg(distinct geom_geomref_colname))[1] geom_geomref_colnames,
+            (array_agg(distinct geom_tablename||'.'||geom_geomref_colname))[1] geom_geomref_colnames_qualified
+      FROM json_to_recordset(meta)
+      INTO geom_tablenames, geom_colnames, geom_geomref_colnames, geom_geomref_colnames_qualified
+        AS x(id TEXT, numer_id TEXT, numer_aggregate TEXT, numer_colname TEXT, numer_geomref_colname TEXT, numer_tablename TEXT,
+            numer_type TEXT, denom_id TEXT, denom_aggregate TEXT, denom_colname TEXT, denom_geomref_colname TEXT, denom_tablename TEXT,
+            denom_type TEXT, denom_reltype TEXT, geom_id TEXT, geom_colname TEXT, geom_geomref_colname TEXT, geom_tablename TEXT,
+            geom_type TEXT, numer_timespan TEXT, geom_timespan TEXT, normalization TEXT, api_method TEXT, api_args JSON);
+  END IF;
+
+  ---------MC---------
+  IF geography_level = 'us.census.tiger.census_tract' THEN
+    mc_geography_level := 'tract';
+  ELSE
+    mc_geography_level := (string_to_array(geography_level, '.'))[array_length(string_to_array(geography_level, '.'), 1)];
+  END IF;
+
+  mc_table := cdb_observatory.OBS_GetMCTable(mc_schema, mc_geography_level);
+
+  FOREACH mc_category IN ARRAY mc_categories LOOP
+    FOREACH mc_month IN ARRAY mc_months LOOP
+      mc_month_slug := replace(mc_month, '/', '');
+
+      SELECT CASE
+        WHEN mc_category = 'NEP' THEN 'non eating places'
+	      WHEN mc_category = 'EP'  THEN 'eating places'
+	      WHEN mc_category = 'APP' THEN 'apparel'
+	      WHEN mc_category = 'SB'  THEN 'small business'
+	      WHEN mc_category = 'TR'  THEN 'total retail'
+        END INTO mc_category_name;
+
+      SELECT string_agg(column_name||'_'||mc_category||'_'||mc_month_slug, ','),
+            string_agg(mc_table||'_'||mc_category||'_'||mc_month_slug||'.'||column_name||' '||column_name||'_'||mc_category||'_'||mc_month_slug, ','),
+            string_agg(distinct column_name||'_'||mc_category||'_'||mc_month_slug||area_normalization||' '||column_name||'_'||mc_category||'_'||mc_month_slug, ',')
+        INTO numer_colnames_mc_current, numer_colnames_mc_qualified_current, numer_colnames_mc_normalized_current
+        FROM information_schema.columns
+      WHERE table_schema = mc_schema
+        AND table_name = mc_table
+        AND column_name = ANY(mc_measurements);
+
+      IF numer_colnames_mc_current IS NOT NULL THEN
+        numer_colnames_mc := coalesce(numer_colnames_mc, '')||numer_colnames_mc_current||',';
+      END IF;
+      IF numer_colnames_mc_qualified_current IS NOT NULL THEN
+        numer_colnames_mc_qualified := coalesce(numer_colnames_mc_qualified, '')||numer_colnames_mc_qualified_current||',';
+      END IF;
+      IF numer_colnames_mc_normalized_current IS NOT NULL THEN
+        numer_colnames_mc_normalized := coalesce(numer_colnames_mc_normalized, '')||numer_colnames_mc_normalized_current||',';
+      END IF;
+
+      IF mc_table IS NOT NULL THEN
+        numer_tablenames_mc := '"'||mc_schema||'".'||mc_table||' '||mc_table||'_'||mc_category||'_'||mc_month_slug;
+        geom_relations_mc := mc_table||'_'||mc_category||'_'||mc_month_slug||'.'||mc_geoid||'='||geom_geomref_colnames_qualified;
+        mc_table_categories := mc_table||'_'||mc_category||'_'||mc_month_slug||'.'||mc_category_column||'='''||mc_category_name||''''||
+                              ' AND '||mc_table||'_'||mc_category||'_'||mc_month_slug||'.'||mc_month_column||'='''||mc_month||'''';
+
+        geom_mc_outerjoins := coalesce(geom_mc_outerjoins, '')||' LEFT OUTER JOIN '||numer_tablenames_mc||' ON '||geom_relations_mc||' AND '||mc_table_categories;
+      END IF;
+    END LOOP;
+  END LOOP;
+
+  ---------Query build and execution---------
+  RETURN QUERY EXECUTE format(
+    $query$
+    SELECT  x, y, z,
+            mvtgeom,
+            (select row_to_json(_)::jsonb from (select id, %9$s %3$s area_ratio, area) as _) as mvtdata
+          FROM (
+      SELECT x, y, z,
+             ST_AsMVTGeom(ST_Transform(the_geom, 3857),
+                          bbox2d, $1, $2, $3) AS mvtgeom, %8$s as id, %6$s %7$s area_ratio, area FROM (
+        SELECT  tx.x, tx.y, tx.z,
+                %1$s the_geom, %8$s, %2$s %10$s
+                CASE  WHEN ST_Within(tx.envelope, %1$s)
+                        THEN ST_Area(tx.envelope) / Nullif(ST_Area(%1$s), 0)
+                      WHEN ST_Within(%1$s, tx.envelope)
+                        THEN 1
+                      ELSE ST_Area(ST_Intersection(st_simplifyvw(%1$s, $4), tx.envelope)) / Nullif(ST_Area(%1$s), 0)
+                END area_ratio,
+                ROUND(ST_Area(ST_Transform(the_geom,3857))::NUMERIC, 2) area,
+                ST_MakeBox2D(ST_Transform(ST_SetSRID(ST_Point(tx.bounds[1], tx.bounds[2]), 4326), 3857),
+                             ST_Transform(ST_SetSRID(ST_Point(tx.bounds[3], tx.bounds[4]), 4326), 3857)) bbox2d
+          FROM tiler.xyz_us_mc_tiles_temp_%12$s_%13$s tx,
+               %5$s
+               %4$s
+               %11$s
+        WHERE st_intersects(%1$s, tx.envelope)
+      ) p
+    ) q
+    $query$,
+    geom_colnames, numer_colnames_do_qualified, numer_colnames_mc, numer_tablenames_do_outer, geom_tablenames, numer_colnames_do_normalized,
+    numer_colnames_mc_normalized, geom_geomref_colnames, numer_colnames_do, numer_colnames_mc_qualified, geom_mc_outerjoins,
+    mc_geography_level, z)
+  USING extent, buf, clip_geom, simplification_tolerance
+  RETURN;
+END
+$$ LANGUAGE plpgsql PARALLEL RESTRICTED;

From 44727ac44ce44e8c4543d387877930d56b6a6a66 Mon Sep 17 00:00:00 2001
From: antoniocarlon <antoniocarlon@gmail.com>
Date: Mon, 16 Jul 2018 16:50:41 +0200
Subject: [PATCH 45/45] Improved query performance removing outer joins

---
 src/pg/sql/45_observatory_mvt.sql | 105 +++++++++++++++---------------
 1 file changed, 51 insertions(+), 54 deletions(-)

diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql
index 850611b..59a6816 100644
--- a/src/pg/sql/45_observatory_mvt.sql
+++ b/src/pg/sql/45_observatory_mvt.sql
@@ -302,11 +302,11 @@ AS $$
 DECLARE
   mc_table TEXT;
 BEGIN
-  SELECT tablename from pg_tables
-    INTO mc_table
-   WHERE schemaname = mc_schema
-     AND tablename LIKE '%'||geo_level||'%';
-
+--  SELECT tablename from pg_tables
+--    INTO mc_table
+--   WHERE schemaname = mc_schema
+--     AND tablename LIKE '%'||geo_level||'%';
+  mc_table := 'mc_' || geo_level;
   RETURN mc_table;
 END
 $$ LANGUAGE plpgsql PARALLEL RESTRICTED;
@@ -338,16 +338,17 @@ DECLARE
   blockgroup_geoname CONSTANT TEXT DEFAULT 'us.census.tiger.block_group';
   block_geoname CONSTANT TEXT DEFAULT 'us.census.tiger.block';
 
-  mc_schema CONSTANT TEXT DEFAULT 'us.mastercard';
+  mc_schema CONSTANT TEXT DEFAULT 'tiler';
   mc_geoid CONSTANT TEXT DEFAULT 'region_id';
   mc_category_column CONSTANT TEXT DEFAULT 'category';
   mc_month_column CONSTANT TEXT DEFAULT 'month';
   mc_table TEXT;
   mc_category TEXT;
-  mc_category_name TEXT;
   mc_table_categories TEXT DEFAULT '';
   mc_month TEXT;
   mc_month_slug TEXT;
+  mc_measurements_categories TEXT[];
+  mc_measurement TEXT;
 
   bounds NUMERIC[];
   geom GEOMETRY;
@@ -490,26 +491,24 @@ BEGIN
 
   mc_table := cdb_observatory.OBS_GetMCTable(mc_schema, mc_geography_level);
 
-  FOREACH mc_category IN ARRAY mc_categories LOOP
-    FOREACH mc_month IN ARRAY mc_months LOOP
-      mc_month_slug := replace(mc_month, '-', '');
+  FOREACH mc_month IN ARRAY mc_months LOOP
+    mc_month_slug := replace(mc_month, '/', '');
+    FOREACH mc_category IN ARRAY mc_categories LOOP
+      mc_category := lower(mc_category);
+      mc_measurements_categories := ARRAY['']::TEXT[];
 
-      SELECT CASE
-        WHEN mc_category = 'NEP' THEN 'non eating places'
-	      WHEN mc_category = 'EP'  THEN 'eating places'
-	      WHEN mc_category = 'APP' THEN 'apparel'
-	      WHEN mc_category = 'SB'  THEN 'small business'
-	      WHEN mc_category = 'TR'  THEN 'total retail'
-        END INTO mc_category_name;
+      FOREACH mc_measurement IN ARRAY mc_measurements LOOP
+        mc_measurements_categories := array_append(mc_measurements_categories, mc_measurement||'_'||mc_category);
+      END LOOP;
 
-      SELECT string_agg(column_name||'_'||mc_category||'_'||mc_month_slug, ','),
-            string_agg(mc_table||'_'||mc_category||'_'||mc_month_slug||'.'||column_name||' '||column_name||'_'||mc_category||'_'||mc_month_slug, ','),
-            string_agg(distinct column_name||'_'||mc_category||'_'||mc_month_slug||area_normalization||' '||column_name||'_'||mc_category||'_'||mc_month_slug, ',')
+      SELECT string_agg(column_name||'_'||mc_month_slug, ','),
+            string_agg(mc_table||'_'||mc_month_slug||'.'||column_name||' '||column_name||'_'||mc_month_slug, ','),
+            string_agg(distinct column_name||'_'||mc_month_slug||area_normalization||' '||column_name||'_'||mc_month_slug, ',')
         INTO numer_colnames_mc_current, numer_colnames_mc_qualified_current, numer_colnames_mc_normalized_current
         FROM information_schema.columns
       WHERE table_schema = mc_schema
         AND table_name = mc_table
-        AND column_name = ANY(mc_measurements);
+        AND column_name = ANY(mc_measurements_categories);
 
       IF numer_colnames_mc_current IS NOT NULL THEN
         numer_colnames_mc := coalesce(numer_colnames_mc, '')||numer_colnames_mc_current||',';
@@ -520,16 +519,15 @@ BEGIN
       IF numer_colnames_mc_normalized_current IS NOT NULL THEN
         numer_colnames_mc_normalized := coalesce(numer_colnames_mc_normalized, '')||numer_colnames_mc_normalized_current||',';
       END IF;
-
-      IF mc_table IS NOT NULL THEN
-        numer_tablenames_mc := '"'||mc_schema||'".'||mc_table||' '||mc_table||'_'||mc_category||'_'||mc_month_slug;
-        geom_relations_mc := mc_table||'_'||mc_category||'_'||mc_month_slug||'.'||mc_geoid||'='||geom_geomref_colnames_qualified;
-        mc_table_categories := mc_table||'_'||mc_category||'_'||mc_month_slug||'.'||mc_category_column||'='''||mc_category_name||''''||
-                              ' AND '||mc_table||'_'||mc_category||'_'||mc_month_slug||'.'||mc_month_column||'='''||mc_month||'''';
-
-        geom_mc_outerjoins := coalesce(geom_mc_outerjoins, '')||' LEFT OUTER JOIN '||numer_tablenames_mc||' ON '||geom_relations_mc||' AND '||mc_table_categories;
-      END IF;
     END LOOP;
+
+    IF mc_table IS NOT NULL THEN
+      numer_tablenames_mc := '"'||mc_schema||'".'||mc_table||' '||mc_table||'_'||mc_month_slug;
+      geom_relations_mc := mc_table||'_'||mc_month_slug||'.'||mc_geoid||'='||geom_geomref_colnames_qualified;
+      mc_table_categories := mc_table||'_'||mc_month_slug||'.'||mc_month_column||'='''||mc_month||'''';
+
+      geom_mc_outerjoins := coalesce(geom_mc_outerjoins, '')||' LEFT OUTER JOIN '||numer_tablenames_mc||' ON '||geom_relations_mc||' AND '||mc_table_categories;
+    END IF;
   END LOOP;
 
   ---------Query build and execution---------
@@ -595,7 +593,7 @@ DECLARE
   avg_x INTEGER;
   avg_y INTEGER;
 
-  mc_schema CONSTANT TEXT DEFAULT 'us.mastercard';
+  mc_schema CONSTANT TEXT DEFAULT 'tiler';
   mc_geoid CONSTANT TEXT DEFAULT 'region_id';
   mc_category_column CONSTANT TEXT DEFAULT 'category';
   mc_month_column CONSTANT TEXT DEFAULT 'month';
@@ -605,6 +603,8 @@ DECLARE
   mc_table_categories TEXT DEFAULT '';
   mc_month TEXT;
   mc_month_slug TEXT;
+  mc_measurements_categories TEXT[];
+  mc_measurement TEXT;
 
   measurement TEXT;
   getmeta_parameters TEXT;
@@ -762,26 +762,24 @@ BEGIN
 
   mc_table := cdb_observatory.OBS_GetMCTable(mc_schema, mc_geography_level);
 
-  FOREACH mc_category IN ARRAY mc_categories LOOP
-    FOREACH mc_month IN ARRAY mc_months LOOP
-      mc_month_slug := replace(mc_month, '/', '');
+  FOREACH mc_month IN ARRAY mc_months LOOP
+    mc_month_slug := replace(mc_month, '/', '');
+    FOREACH mc_category IN ARRAY mc_categories LOOP
+      mc_category := lower(mc_category);
+      mc_measurements_categories := ARRAY['']::TEXT[];
 
-      SELECT CASE
-        WHEN mc_category = 'NEP' THEN 'non eating places'
-	      WHEN mc_category = 'EP'  THEN 'eating places'
-	      WHEN mc_category = 'APP' THEN 'apparel'
-	      WHEN mc_category = 'SB'  THEN 'small business'
-	      WHEN mc_category = 'TR'  THEN 'total retail'
-        END INTO mc_category_name;
+      FOREACH mc_measurement IN ARRAY mc_measurements LOOP
+        mc_measurements_categories := array_append(mc_measurements_categories, mc_measurement||'_'||mc_category);
+      END LOOP;
 
-      SELECT string_agg(column_name||'_'||mc_category||'_'||mc_month_slug, ','),
-            string_agg(mc_table||'_'||mc_category||'_'||mc_month_slug||'.'||column_name||' '||column_name||'_'||mc_category||'_'||mc_month_slug, ','),
-            string_agg(distinct column_name||'_'||mc_category||'_'||mc_month_slug||area_normalization||' '||column_name||'_'||mc_category||'_'||mc_month_slug, ',')
+      SELECT string_agg(column_name||'_'||mc_month_slug, ','),
+            string_agg(mc_table||'_'||mc_month_slug||'.'||column_name||' '||column_name||'_'||mc_month_slug, ','),
+            string_agg(distinct column_name||'_'||mc_month_slug||area_normalization||' '||column_name||'_'||mc_month_slug, ',')
         INTO numer_colnames_mc_current, numer_colnames_mc_qualified_current, numer_colnames_mc_normalized_current
         FROM information_schema.columns
       WHERE table_schema = mc_schema
         AND table_name = mc_table
-        AND column_name = ANY(mc_measurements);
+        AND column_name = ANY(mc_measurements_categories);
 
       IF numer_colnames_mc_current IS NOT NULL THEN
         numer_colnames_mc := coalesce(numer_colnames_mc, '')||numer_colnames_mc_current||',';
@@ -792,16 +790,15 @@ BEGIN
       IF numer_colnames_mc_normalized_current IS NOT NULL THEN
         numer_colnames_mc_normalized := coalesce(numer_colnames_mc_normalized, '')||numer_colnames_mc_normalized_current||',';
       END IF;
-
-      IF mc_table IS NOT NULL THEN
-        numer_tablenames_mc := '"'||mc_schema||'".'||mc_table||' '||mc_table||'_'||mc_category||'_'||mc_month_slug;
-        geom_relations_mc := mc_table||'_'||mc_category||'_'||mc_month_slug||'.'||mc_geoid||'='||geom_geomref_colnames_qualified;
-        mc_table_categories := mc_table||'_'||mc_category||'_'||mc_month_slug||'.'||mc_category_column||'='''||mc_category_name||''''||
-                              ' AND '||mc_table||'_'||mc_category||'_'||mc_month_slug||'.'||mc_month_column||'='''||mc_month||'''';
-
-        geom_mc_outerjoins := coalesce(geom_mc_outerjoins, '')||' LEFT OUTER JOIN '||numer_tablenames_mc||' ON '||geom_relations_mc||' AND '||mc_table_categories;
-      END IF;
     END LOOP;
+
+    IF mc_table IS NOT NULL THEN
+      numer_tablenames_mc := '"'||mc_schema||'".'||mc_table||' '||mc_table||'_'||mc_month_slug;
+      geom_relations_mc := mc_table||'_'||mc_month_slug||'.'||mc_geoid||'='||geom_geomref_colnames_qualified;
+      mc_table_categories := mc_table||'_'||mc_month_slug||'.'||mc_month_column||'='''||mc_month||'''';
+
+      geom_mc_outerjoins := coalesce(geom_mc_outerjoins, '')||' LEFT OUTER JOIN '||numer_tablenames_mc||' ON '||geom_relations_mc||' AND '||mc_table_categories;
+    END IF;
   END LOOP;
 
   ---------Query build and execution---------