Merge pull request #219 from CartoDB/develop

Release 1.1.4

NEWS.md

@@ -1,3 +1,14 @@
+1.1.4 (2016-11-21)
+
+__Bugfixes__
+
+* Fix duplicate subsections with only a partial set of measures appearing from
+  `OBS_GetLegacyMetadata` ([#216](https://github.com/CartoDB/observatory-extension/issues/216)).
+
+1.1.3 (2016-11-15)
+
+* Temporarily ignore EU data for the sake of testing
+
 1.1.2 (2016-11-09)
 
 __Improvements__

release/observatory.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB Observatory backend extension'
-default_version = '1.1.2'
+default_version = '1.1.4'
 requires = 'postgis, postgres_fdw'
 superuser = true
 schema = cdb_observatory

src/pg/observatory.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB Observatory backend extension'
-default_version = '1.1.2'
+default_version = '1.1.4'
 requires = 'postgis, postgres_fdw'
 superuser = true
 schema = cdb_observatory

src/pg/sql/42_observatory_exploration.sql

@@ -374,41 +374,39 @@ BEGIN
   END IF;
   RETURN QUERY
   EXECUTE format($string$
-  WITH expanded_subsections AS (
-    SELECT numer_id,
-           numer_name,
-           numer_tags,
-           jsonb_each_text(numer_tags) as subsection_tag_id_name
-      FROM cdb_observatory.OBS_GetAvailableNumerators()
-      WHERE numer_weight > 0 %s
-  ), expanded_sections AS (
-    SELECT JSONB_Agg(JSONB_Build_Object(
-           'f1', JSONB_Build_Object('id', numer_id, 'name', numer_name))) columns,
-         SUBSTR((subsection_tag_id_name).key, 12) subsection_id,
-         (subsection_tag_id_name).value subsection_name,
-         jsonb_each_text(numer_tags) as section_tag_id_name
-    FROM expanded_subsections
-    WHERE (subsection_tag_id_name).key LIKE 'subsection/%%'
-    GROUP BY (subsection_tag_id_name).key, (subsection_tag_id_name).value,
-             numer_tags
-  ), full_expansion AS (
-    SELECT columns, subsection_id, subsection_name,
-           SUBSTR((section_tag_id_name).key, 9) section_id,
-           (section_tag_id_name).value section_name
-    FROM expanded_sections
-    WHERE (section_tag_id_name).key LIKE 'section/%%'
-  )
-  SELECT section_name AS name, JSONB_Agg(
-    JSONB_Build_Object(
-      'f1', JSONB_Build_Object(
-        'name', subsection_name,
-        'id', subsection_id,
-        'columns', columns
+    WITH expanded AS (
+        SELECT JSONB_Build_Object('id', numer_id, 'name', numer_name) "column",
+               SUBSTR((sections).key, 9) section_id, (sections).value section_name,
+               SUBSTR((subsections).key, 12) subsection_id, (subsections).value subsection_name
+        FROM (
+          SELECT numer_id, numer_name,
+                 jsonb_each_text(numer_tags) as sections,
+                 jsonb_each_text as subsections
+          FROM (SELECT numer_id, numer_name, numer_tags,
+                       jsonb_each_text(numer_tags)
+                FROM cdb_observatory.obs_getavailablenumerators()
+                WHERE numer_weight > 0 %s
+               ) foo
+        ) bar
+        WHERE (sections).key LIKE 'section/%%'
+          AND (subsections).key LIKE 'subsection/%%'
+      ), grouped_by_subsections AS (
+        SELECT JSONB_Agg(JSONB_Build_Object('f1', "column")) AS columns,
+               section_id, section_name, subsection_id, subsection_name
+        FROM expanded
+        GROUP BY section_id, section_name, subsection_id, subsection_name
       )
-    )
-  ) as subsection
-  FROM full_expansion
-  GROUP BY section_name
+    SELECT section_name as name, JSONB_Agg(
+      JSONB_Build_Object(
+        'f1', JSONB_Build_Object(
+          'name', subsection_name,
+          'id', subsection_id,
+          'columns', columns
+        )
+      )
+    ) as subsection
+    FROM grouped_by_subsections
+    GROUP BY section_name
   $string$, aggregate_condition);
   RETURN;
 END

src/pg/test/expected/42_observatory_exploration_test.out

@@ -198,3 +198,6 @@ t
 _median_income_not_in_legacy_builder_metadata_sums
 t
 (1 row)
+_no_dupe_subsections_in_legacy_builder_metadata
+t
+(1 row)

src/pg/test/sql/42_observatory_exploration_test.sql

@@ -494,3 +494,12 @@ SELECT 'us.census.acs.B19013001' NOT IN (SELECT
   (jsonb_array_elements(((jsonb_array_elements(subsection))->'f1')->'columns')->'f1')->>'id' AS id
   FROM cdb_observatory.OBS_LegacyBuilderMetadata('sum')
 ) AS _median_income_not_in_legacy_builder_metadata_sums;
+
+SELECT COUNT(*) = 0 _no_dupe_subsections_in_legacy_builder_metadata FROM (
+  SELECT name, subsection, count(*) FROM
+    (SELECT name, ((JSONB_Array_Elements(subsection))->'f1')->>'id' subsection
+    FROM cdb_observatory.obs_legacybuildermetadata()) foo
+  GROUP BY name, subsection
+  HAVING count(*) > 1
+) bar;
+

src/python/test/autotest.py

@@ -145,7 +145,8 @@ def default_lonlat(column_id):
     elif column_id.startswith('us.epa.'):
         return (40.7, -73.9)
     elif column_id.startswith('eu.'):
-        return (52.52207036136366, 13.40606689453125)
+        raise SkipTest('No tests for Eurostat!')
+        #return (52.52207036136366, 13.40606689453125)
     else:
         raise Exception('No catalog point set for {}'.format(column_id))