Merge pull request #301 from CartoDB/develop

Release 1.6.0
2017-07-20 12:49:30 +02:00
parent 415a4ccc05 028c93170c
commit 3ed18ca1f0
10 changed files with 2729 additions and 14 deletions
--- a/NEWS.md
+++ b/NEWS.md
@@ -1,3 +1,21 @@
+1.6.0 (2017-07-20)
+
+__Improvements__
+
+* The current OBS_GetAvailableNumerators is not designed with our
+  UI in mind so it's causing a lot of troubles and we're doing so
+  many hacks to fit our UI needs and the interface of the function so this
+  function it's a better fit for our purposes. ([#300](https://github.com/CartoDB/observatory-extension/pull/300))
+* Now use the new meta table `obs_meta_geom_numer_timespan` to filter
+  the geometries by geometries timespan and/or numerator timespan (which
+  is what we get when we use the obs_getavailabletimespans) ([#302](https://github.com/CartoDB/observatory-extension/pull/302))
+
+__Bugfixes__
+
+* Right now we're doing INNER JOINS when we JOIN the `_procgeoms` and
+  the data so we end up with NULL value instead of id, NULL value. ([#298](https://github.com/CartoDB/observatory-extension/pull/298))
+
+
 1.5.1 (2017-05-16)

 __Improvements__
--- a/release/observatory--1.6.0.sql
+++ b/release/observatory--1.6.0.sql
--- a/release/observatory.control
+++ b/release/observatory.control
@@ -1,5 +1,5 @@
 comment = 'CartoDB Observatory backend extension'
-default_version = '1.5.1'
+default_version = '1.6.0'
 requires = 'postgis'
 superuser = true
 schema = cdb_observatory
--- a/src/pg/observatory.control
+++ b/src/pg/observatory.control
@@ -1,5 +1,5 @@
 comment = 'CartoDB Observatory backend extension'
-default_version = '1.5.1'
+default_version = '1.6.0'
 requires = 'postgis'
 superuser = true
 schema = cdb_observatory
--- a/src/pg/sql/41_observatory_augmentation.sql
+++ b/src/pg/sql/41_observatory_augmentation.sql
@@ -760,16 +760,13 @@ BEGIN
              || ') val_' || colid, ', ')
            || '
            FROM _procgeoms_' || Coalesce(geom_tablename || '_' || geom_geomref_colname, api_method) || ' _procgeoms ' ||
-              Coalesce(', ' || String_Agg(DISTINCT
-                  Coalesce('observatory.' || numer_tablename,
-                    'LATERAL (SELECT * FROM cdb_observatory.' || api_method || '(_procgeoms.geom' || Coalesce(', ' ||
+              Coalesce(String_Agg(DISTINCT
+                  Coalesce('LEFT JOIN observatory.' || numer_tablename || ' ON _procgeoms.geomref = observatory.' || numer_tablename || '.' || numer_geomref_colname,
+                    ', LATERAL (SELECT * FROM cdb_observatory.' || api_method || '(_procgeoms.geom' || Coalesce(', ' ||
                        (SELECT STRING_AGG(REPLACE(val::text, '"', ''''), ', ')
                          FROM (SELECT JSON_Array_Elements(api_args) as val) as vals),
                        '') || ')) AS ' || api_method)
-              , ', '), '') ||
-            Coalesce(' WHERE ' || String_Agg(DISTINCT
-              '_procgeoms.geomref = ' || numer_tablename || '.' || numer_geomref_colname, ' AND '
-            ), '') ||
+              , ' '), '') ||
            CASE $3 WHEN True THEN E'\n GROUP BY _procgeoms.id ORDER BY _procgeoms.id '
                    ELSE           E'\n GROUP BY _procgeoms.id, _procgeoms.geomref
                                        ORDER BY _procgeoms.id, _procgeoms.geomref' END
--- a/src/pg/sql/42_observatory_exploration.sql
+++ b/src/pg/sql/42_observatory_exploration.sql
@@ -181,6 +181,86 @@ BEGIN
 END
 $$ LANGUAGE plpgsql;

+CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetNumerators(
+  bounds GEOMETRY DEFAULT NULL,
+  section_tags TEXT[] DEFAULT ARRAY[]::TEXT[],
+  subsection_tags TEXT[] DEFAULT ARRAY[]::TEXT[],
+  other_tags TEXT[] DEFAULT ARRAY[]::TEXT[],
+  ids TEXT[] DEFAULT ARRAY[]::TEXT[],
+  name TEXT DEFAULT NULL,
+  denom_id TEXT DEFAULT '',
+  geom_id TEXT DEFAULT '',
+  timespan TEXT DEFAULT ''
+) RETURNS TABLE (
+  numer_id TEXT,
+  numer_name TEXT,
+  numer_description TEXT,
+  numer_weight NUMERIC,
+  numer_license TEXT,
+  numer_source TEXT,
+  numer_type TEXT,
+  numer_aggregate TEXT,
+  numer_extra JSONB,
+  numer_tags JSONB,
+  valid_denom BOOLEAN,
+  valid_geom BOOLEAN,
+  valid_timespan BOOLEAN
+) AS $$
+DECLARE
+  where_clause_elements TEXT[];
+  geom_clause TEXT;
+  where_clause TEXT;
+BEGIN
+  where_clause_elements := (ARRAY[])::TEXT[];
+  where_clause := '';
+
+  IF bounds IS NOT NULL THEN
+    where_clause_elements := array_append(where_clause_elements, format($data$ST_Intersects(the_geom, '%s'::geometry)$data$, bounds));
+  END IF;
+  IF cardinality(section_tags) > 0 THEN
+    where_clause_elements := array_append(where_clause_elements, format($data$numer_tags ?| '%s'$data$, section_tags));
+  END IF;
+  IF cardinality(subsection_tags) > 0 THEN
+    where_clause_elements := array_append(where_clause_elements, format($data$numer_tags ?| '%s'$data$, subsection_tags));
+  END IF;
+  IF cardinality(other_tags) > 0 THEN
+    where_clause_elements := array_append(where_clause_elements, format($data$numer_tags ?| '%s'$data$, other_tags));
+  END IF;
+  IF cardinality(ids) > 0 THEN
+    where_clause_elements := array_append(where_clause_elements, format($data$numer_id IN (array_to_string('%s'::text[], ','))$data$, ids));
+  END IF;
+  IF name IS NOT NULL AND name != '' THEN
+    where_clause_elements := array_append(where_clause_elements, format($data$numer_name ilike '%%%s%%'$data$, name));
+  END IF;
+  IF cardinality(where_clause_elements) > 0 THEN
+    where_clause := format($clause$WHERE %s$clause$, array_to_string(where_clause_elements, ' AND '));
+  END IF;
+  RAISE DEBUG '%', array_to_string(where_clause_elements, ' AND ');
+
+  RETURN QUERY
+  EXECUTE
+  format($string$
+    SELECT numer_id::TEXT,
+           numer_name::TEXT,
+           numer_description::TEXT,
+           numer_weight::NUMERIC,
+           NULL::TEXT license,
+           NULL::TEXT source,
+           numer_type numer_type,
+           numer_aggregate numer_aggregate,
+           numer_extra::JSONB numer_extra,
+           numer_tags numer_tags,
+    $1 = ANY(denoms) valid_denom,
+    $2 = ANY(geoms) valid_geom,
+    $3 = ANY(timespans) valid_timespan
+    FROM observatory.obs_meta_numer
+    %s
+  $string$, where_clause)
+  USING denom_id, geom_id, timespan;
+  RETURN;
+END
+$$ LANGUAGE plpgsql;
+
 CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetAvailableDenominators(
  bounds GEOMETRY DEFAULT NULL,
  filter_tags TEXT[] DEFAULT NULL,
@@ -292,16 +372,28 @@ BEGIN
             geom_type::TEXT,
             geom_extra::JSONB,
             geom_tags::JSONB,
-      $1 = ANY(numers) valid_numer,
-      $2 = ANY(denoms) valid_denom,
-      $3 = ANY(timespans) valid_timespan
-      FROM observatory.obs_meta_geom
+             $1 = ANY(numers) valid_numer,
+             $2 = ANY(denoms) valid_denom,
+             CASE WHEN $3 IS NOT NULL AND $3 != '' THEN
+                -- Here we are looking for geometries with: a) geometry timespan or b) numerators linked to that geometries that fit in the
+                -- timespan passed. For example it look for geometries with timespan '2015 - 2015' or numerators linked to that geometry that has
+                -- '2015 - 2015' as one of the valid timespans.
+                -- If we pass a numerator_id, we filter by that numerator
+                CASE WHEN $1 IS NOT NULL AND $1 != '' THEN
+                  EXISTS (SELECT 1 FROM observatory.obs_meta_geom_numer_timespan onu WHERE o.geom_id = onu.geom_id AND onu.numer_id = $1 AND ($3 = ANY(onu.timespans) OR $3 IN (select(unnest(o.timespans)))))
+                ELSE
+                  EXISTS (SELECT 1 FROM observatory.obs_meta_geom_numer_timespan onu WHERE o.geom_id = onu.geom_id AND ($3 = ANY(onu.timespans) OR $3 IN (select(unnest(o.timespans)))))
+                END
+             ELSE
+              false
+             END as valid_timespan
+      FROM observatory.obs_meta_geom o
      WHERE %s (geom_tags ?& $4 OR CARDINALITY($4) = 0)
    ), scores AS (
      SELECT * FROM cdb_observatory._OBS_GetGeometryScores($5,
        (SELECT ARRAY_AGG(geom_id) FROM available_geoms)
      )
-    ) SELECT available_geoms.*, score, numtiles, notnull_percent, numgeoms,
+    ) SELECT DISTINCT ON (geom_id) available_geoms.*, score, numtiles, notnull_percent, numgeoms,
             percentfill, estnumgeoms, meanmediansize
      FROM available_geoms, scores
      WHERE available_geoms.geom_id = scores.column_id
--- a/src/pg/test/expected/41_observatory_augmentation_test.out
+++ b/src/pg/test/expected/41_observatory_augmentation_test.out
@@ -210,6 +210,9 @@ t|t|t
 id|data_polygon_measure_one_null|data_polygon_measure_two_null
 t|t|t
 (1 row)
+id|data_polygon_measure_one_null|data_polygon_measure_two_null
+t|t|t
+(1 row)
 id|data_polygon_measure_one_predenom|data_polygon_measure_two_predenom
 t|t|t
 (1 row)
--- a/src/pg/test/expected/42_observatory_exploration_test.out
+++ b/src/pg/test/expected/42_observatory_exploration_test.out
@@ -48,6 +48,63 @@ t
 _obs_getavailablenumerators_no_total_pop_1996
 t
 (1 row)
+_obs_getnumerators_usa_pop_in_all
+t
+(1 row)
+_obs_getnumerators_usa_pop_in_nyc_point
+t
+(1 row)
+_obs_getnumerators_usa_pop_in_usa_extents
+t
+(1 row)
+_obs_getnumerators_no_usa_pop_not_in_zero_point
+t
+(1 row)
+_obs_getnumerators_usa_pop_in_age_gender_subsection
+t
+(1 row)
+_obs_getnumerators_no_pop_in_income_subsection
+t
+(1 row)
+_obs_getnumerators_male_pop_denom_by_total_pop
+t
+(1 row)
+_obs_getnumerators_no_income_denom_by_total_pop
+t
+(1 row)
+_obs_getnumerators_zillow_at_zcta5
+t
+(1 row)
+_obs_getnumerators_no_zillow_at_block_group
+t
+(1 row)
+_obs_getnumerators_total_pop_2010_2014
+t
+(1 row)
+_obs_getnumerators_no_total_pop_1996
+t
+(1 row)
+_obs_getnumerators_total_pop_by_name
+t
+(1 row)
+_obs_getnumerators_total_pop_by_section
+t
+(1 row)
+_obs_getnumerators_total_pop_not_in_canada
+t
+(1 row)
+_obs_getnumerators_total_pop_by_subsection
+t
+(1 row)
+_obs_getnumerators_total_pop_not_in_employment_subsection
+t
+(1 row)
+_obs_getnumerators_total_pop_by_id
+t
+(1 row)
+_obs_getnumerators_total_pop_not_with_other_id
+t
+(1 row)
 _obs_getavailabledenominators_usa_pop_in_all
 t
 (1 row)
--- a/src/pg/test/sql/41_observatory_augmentation_test.sql
+++ b/src/pg/test/sql/41_observatory_augmentation_test.sql
@@ -596,6 +596,18 @@ SELECT id = 1 id,
       abs((data->1->>'value')::Numeric - 0.4902) / 0.4902 < 0.001 data_polygon_measure_two_null
 FROM data;

+-- OBS_GetData/OBS_GetMeta by geom with two measures and one return null
+WITH
+meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
+  '[{"numer_id": "us.census.acs.B19013001_quantile"}, {"numer_id": "us.census.acs.B01001002"}]') meta),
+data AS (SELECT * FROM cdb_observatory.OBS_GetData(
+    ARRAY[(cdb_observatory._TestArea(), 1)::geomval],
+  (SELECT meta FROM meta)))
+SELECT id = 1 id,
+       (data->0->>'value') is NULL data_polygon_measure_one_null,
+       abs((data->1->>'value')::Numeric - 0.4902) / 0.4902 < 0.001 data_polygon_measure_two_null
+FROM data;
+
 -- OBS_GetData/OBS_GetMeta by geom with two standard measures predenom normalization
 WITH
 meta AS (SELECT cdb_observatory.OBS_GetMeta(cdb_observatory._TestArea(),
--- a/src/pg/test/sql/42_observatory_exploration_test.sql
+++ b/src/pg/test/sql/42_observatory_exploration_test.sql
@@ -119,6 +119,142 @@ FROM cdb_observatory.OBS_GetAvailableNumerators(
 ) WHERE valid_timespan = True)
 AS _obs_getavailablenumerators_no_total_pop_1996;

+--
+-- _OBS_GetNumerators tests
+--
+
+SELECT 'us.census.acs.B01003001' IN (SELECT numer_id
+FROM cdb_observatory._OBS_GetNumerators())
+AS _obs_getnumerators_usa_pop_in_all;
+
+SELECT 'us.census.acs.B01003001' IN (SELECT numer_id
+FROM cdb_observatory._OBS_GetNumerators(
+  ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326),
+  NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL
+)) AS _obs_getnumerators_usa_pop_in_nyc_point;
+
+SELECT 'us.census.acs.B01003001' IN (SELECT numer_id
+FROM cdb_observatory._OBS_GetNumerators(
+  ST_SetSRID(ST_MakeEnvelope(
+      -169.8046875, 21.289374355860424,
+      -47.4609375, 72.0739114882038
+  ), 4326),
+  NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL
+)) AS _obs_getnumerators_usa_pop_in_usa_extents;
+
+SELECT 'us.census.acs.B01003001' NOT IN (SELECT numer_id
+FROM cdb_observatory._OBS_GetNumerators(
+  ST_SetSRID(ST_MakePoint(0, 0), 4326),
+  NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL
+)) AS _obs_getnumerators_no_usa_pop_not_in_zero_point;
+
+SELECT 'us.census.acs.B01003001' IN (SELECT numer_id
+FROM cdb_observatory._OBS_GetNumerators(
+  ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326),
+  subsection_tags => ARRAY['subsection/tags.age_gender']
+))
+AS _obs_getnumerators_usa_pop_in_age_gender_subsection;
+
+SELECT 'us.census.acs.B01003001' NOT IN (SELECT numer_id
+FROM cdb_observatory._OBS_GetNumerators(
+  ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326),
+  subsection_tags => ARRAY['subsection/tags.income']
+))
+AS _obs_getnumerators_no_pop_in_income_subsection;
+
+SELECT 'us.census.acs.B01001002' IN (SELECT numer_id
+FROM cdb_observatory._OBS_GetNumerators(
+  ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326),
+  denom_id => 'us.census.acs.B01003001'
+) WHERE valid_denom = True)
+AS _obs_getnumerators_male_pop_denom_by_total_pop;
+
+SELECT 'us.census.acs.B19013001' NOT IN (SELECT numer_id
+FROM cdb_observatory._OBS_GetNumerators(
+  ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326),
+  denom_id => 'us.census.acs.B01003001'
+) WHERE valid_denom = True)
+AS _obs_getnumerators_no_income_denom_by_total_pop;
+
+SELECT 'us.zillow.AllHomes_Zhvi' IN (SELECT numer_id
+FROM cdb_observatory._OBS_GetNumerators(
+  ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326),
+  geom_id => 'us.census.tiger.zcta5'
+) WHERE valid_geom = True)
+AS _obs_getnumerators_zillow_at_zcta5;
+
+SELECT 'us.zillow.AllHomes_Zhvi' NOT IN (SELECT numer_id
+FROM cdb_observatory._OBS_GetNumerators(
+  ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326),
+  geom_id => 'us.census.tiger.block_group'
+) WHERE valid_geom = True)
+AS _obs_getnumerators_no_zillow_at_block_group;
+
+SELECT 'us.census.acs.B01003001' IN (SELECT numer_id
+FROM cdb_observatory._OBS_GetNumerators(
+  ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326),
+  timespan => '2010 - 2014'
+) WHERE valid_timespan = True)
+AS _obs_getnumerators_total_pop_2010_2014;
+
+SELECT 'us.census.acs.B01003001' NOT IN (SELECT numer_id
+FROM cdb_observatory._OBS_GetNumerators(
+  ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326),
+  timespan => '1996'
+) WHERE valid_timespan = True)
+AS _obs_getnumerators_no_total_pop_1996;
+
+SELECT 'us.census.acs.B01003001' IN (SELECT numer_id
+FROM cdb_observatory._OBS_GetNumerators(
+  ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326),
+  name => 'tot'
+))
+AS _obs_getnumerators_total_pop_by_name;
+
+SELECT 'us.census.acs.B01003001' IN (SELECT numer_id
+FROM cdb_observatory._OBS_GetNumerators(
+  ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326),
+  section_tags => '{section/tags.united_states}'
+))
+AS _obs_getnumerators_total_pop_by_section;
+
+SELECT 'us.census.acs.B01003001' NOT IN (SELECT numer_id
+FROM cdb_observatory._OBS_GetNumerators(
+  ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326),
+  section_tags => '{section/tags.ca}'
+))
+AS _obs_getnumerators_total_pop_not_in_canada;
+
+SELECT 'us.census.acs.B01003001' IN (SELECT numer_id
+FROM cdb_observatory._OBS_GetNumerators(
+  ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326),
+  section_tags => '{section/tags.united_states}',
+  subsection_tags => '{subsection/tags.age_gender}'
+))
+AS _obs_getnumerators_total_pop_by_subsection;
+
+SELECT 'us.census.acs.B01003001' NOT IN (SELECT numer_id
+FROM cdb_observatory._OBS_GetNumerators(
+  ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326),
+  section_tags => '{section/tags.united_states}',
+  subsection_tags => '{subsection/tags.employment}'
+))
+AS _obs_getnumerators_total_pop_not_in_employment_subsection;
+
+SELECT 'us.census.acs.B01003001' IN (SELECT numer_id
+FROM cdb_observatory._OBS_GetNumerators(
+  ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326),
+  ids => '{us.census.acs.B01003001}'
+))
+AS _obs_getnumerators_total_pop_by_id;
+
+SELECT 'us.census.acs.B01003001' NOT IN (SELECT numer_id
+FROM cdb_observatory._OBS_GetNumerators(
+  ST_SetSRID(ST_MakePoint(-73.9, 40.7), 4326),
+  ids => '{us.census.acs.B01003002}'
+))
+AS _obs_getnumerators_total_pop_not_with_other_id;
+
 --
 -- OBS_GetAvailableDenominators tests
 --