From 1ba7299fe4f049b8000317cff1a1d851f3678f23 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 24 May 2016 11:02:35 -0400 Subject: [PATCH 01/10] adds obs_getmeasurebyid --- src/pg/sql/41_observatory_augmentation.sql | 44 ++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index a8c42b2..75121b3 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -380,6 +380,50 @@ END; $$ LANGUAGE plpgsql; +CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetMeasureById( + geom_ref TEXT, + measure_id TEXT, + boundary_id TEXT, + time_span TEXT DEFAULT NULL +) +RETURNS NUMERIC +AS $$ +DECLARE + target_table TEXT; + colname TEXT; + measure_val NUMERIC; + data_geoid_colname TEXT; +BEGIN + -- TODO look at how timespans are decided in other functions + SELECT x ->> 'colname', x ->> 'tablename' INTO colname, target_table + FROM cdb_observatory._OBS_GetColumnData(boundary_id, Array[measure_id], time_span) As x; + + EXECUTE + format('SELECT ct.colname + FROM observatory.obs_column_to_column c2c, + observatory.obs_column_table ct, + observatory.obs_table t + WHERE c2c.reltype = ''geom_ref'' + AND ct.column_id = c2c.source_id + AND ct.table_id = t.id + AND t.tablename = %L' + , target_table) + INTO data_geoid_colname; + + EXECUTE format( + 'SELECT %I + FROM observatory.%I + WHERE %I.%I = %L' + colname, + target_table, + target_table, data_geoid_colname, measure_id) + INTO measure_val; + + RETURN measure_val; + +END; +$$ LANGUAGE plpgsql; + CREATE OR REPLACE FUNCTION cdb_observatory.OBS_GetCategory( geom geometry(Geometry, 4326), category_id TEXT, From 3f77a384c75800096cb16a47c112c4e4bc6f5f90 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 24 May 2016 11:06:53 -0400 Subject: [PATCH 02/10] adds missing comma --- src/pg/sql/41_observatory_augmentation.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index 75121b3..b80f807 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -413,7 +413,7 @@ BEGIN EXECUTE format( 'SELECT %I FROM observatory.%I - WHERE %I.%I = %L' + WHERE %I.%I = %L', colname, target_table, target_table, data_geoid_colname, measure_id) From 2e312464aa8cbaa7e236bc567e95647756a3e613 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 24 May 2016 11:22:39 -0400 Subject: [PATCH 03/10] fixes geom_ref in where condition --- src/pg/sql/41_observatory_augmentation.sql | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index b80f807..349ca69 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -393,6 +393,7 @@ DECLARE colname TEXT; measure_val NUMERIC; data_geoid_colname TEXT; + test_query TEXT; BEGIN -- TODO look at how timespans are decided in other functions SELECT x ->> 'colname', x ->> 'tablename' INTO colname, target_table @@ -410,13 +411,18 @@ BEGIN , target_table) INTO data_geoid_colname; - EXECUTE format( + RAISE NOTICE 'target_table %, colname %', target_table, colname; + + test_query := format( 'SELECT %I FROM observatory.%I WHERE %I.%I = %L', colname, target_table, - target_table, data_geoid_colname, measure_id) + target_table, data_geoid_colname, geom_ref); + RAISE NOTICE 'test_query: %', test_query; + + EXECUTE test_query INTO measure_val; RETURN measure_val; From 94756a2377174f878b9a80dc7640fb8c2c926485 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 24 May 2016 11:27:41 -0400 Subject: [PATCH 04/10] minor formatting changes --- src/pg/sql/41_observatory_augmentation.sql | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index 349ca69..ec721f5 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -411,18 +411,15 @@ BEGIN , target_table) INTO data_geoid_colname; - RAISE NOTICE 'target_table %, colname %', target_table, colname; + RAISE DEBUG 'target_table %, colname %', target_table, colname; - test_query := format( + EXECUTE format( 'SELECT %I FROM observatory.%I WHERE %I.%I = %L', colname, target_table, - target_table, data_geoid_colname, geom_ref); - RAISE NOTICE 'test_query: %', test_query; - - EXECUTE test_query + target_table, data_geoid_colname, geom_ref) INTO measure_val; RETURN measure_val; From afb548c75d70432897681a78c53ccf2fcecca99d Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 24 May 2016 12:02:32 -0400 Subject: [PATCH 05/10] adds tests --- .../41_observatory_augmentation_test.out | 9 ++++++++ .../sql/41_observatory_augmentation_test.sql | 23 +++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/src/pg/test/expected/41_observatory_augmentation_test.out b/src/pg/test/expected/41_observatory_augmentation_test.out index 0094351..959549c 100644 --- a/src/pg/test/expected/41_observatory_augmentation_test.out +++ b/src/pg/test/expected/41_observatory_augmentation_test.out @@ -76,3 +76,12 @@ t obs_getuscensuscategory_polygon t (1 row) +obs_getmeasurebyid_cartodb_census_tract +t +(1 row) +obs_getmeasurebyid_cartodb_block_group +t +(1 row) +obs_getmeasurebyid_nulls +t +(1 row) diff --git a/src/pg/test/sql/41_observatory_augmentation_test.sql b/src/pg/test/sql/41_observatory_augmentation_test.sql index 9a14047..3370e78 100644 --- a/src/pg/test/sql/41_observatory_augmentation_test.sql +++ b/src/pg/test/sql/41_observatory_augmentation_test.sql @@ -204,4 +204,27 @@ SELECT cdb_observatory.OBS_GetUSCensusCategory( cdb_observatory._testarea(), 'Spielman-Singleton Segments: 10 Clusters') = 'Low income, mix of minorities' As OBS_GetUSCensusCategory_polygon; +-- OBS_GetMeasureById tests +SELECT (cdb_observatory.OBS_GetMeasureById( + '36047048500', + 'us.census.acs.B01003001', + 'us.census.tiger.census_tract', + '2010 - 2014' +) - 3241) / 3241 < 0.0001 As OBS_GetMeasureById_cartodb_census_tract; + +SELECT (cdb_observatory.OBS_GetMeasureById( + '360470485002', + 'us.census.acs.B01003001', + 'us.census.tiger.block_group', + '2010 - 2014' +) - 1900) / 1900 < 0.0001 As OBS_GetMeasureById_cartodb_block_group; + +-- geom ref / boundary mismatch +SELECT cdb_observatory.OBS_GetMeasureById( + '36047048500', + 'us.census.acs.B01003001', + 'us.census.tiger.block_group', + '2010 - 2014' +) IS NULL As OBS_GetMeasureById_nulls; + \i test/fixtures/drop_fixtures.sql From b5267c14bf5d9b3af6e3bdac0351ef131f70c9dc Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 24 May 2016 13:15:19 -0400 Subject: [PATCH 06/10] removes unneeded todo --- src/pg/sql/41_observatory_augmentation.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index ec721f5..e355950 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -395,7 +395,7 @@ DECLARE data_geoid_colname TEXT; test_query TEXT; BEGIN - -- TODO look at how timespans are decided in other functions + SELECT x ->> 'colname', x ->> 'tablename' INTO colname, target_table FROM cdb_observatory._OBS_GetColumnData(boundary_id, Array[measure_id], time_span) As x; From f9ecf1595c575f34bd1fda3a5dfe54dccb1c8090 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 24 May 2016 13:28:21 -0400 Subject: [PATCH 07/10] adds docs for obs_getmeasurebyid --- doc/measures_functions.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/doc/measures_functions.md b/doc/measures_functions.md index a07ad93..2403e1b 100644 --- a/doc/measures_functions.md +++ b/doc/measures_functions.md @@ -134,6 +134,40 @@ SET household_count = OBS_GetMeasure(the_geom, 'us.census.acs.B11001001') * If an unrecognized normalization type is input, raise an error: `'Only valid inputs for "normalize" are "area" (default) and "denominator".` +## OBS_GetMeasureById(geom_ref text, measure_id text, boundary_id text) + +The ```OBS_GetMeasureById(geom_ref, measure_id, boundary_id)``` function returns any Data Observatory measure that corresponds to the boundary in ```boundary_id``` that has a geometry reference of ```geom_ref```. + +#### Arguments + +Name |Description +--- | --- +geom_ref | a geometry reference (e.g., a US Census geoid) +measure_id | a measure identifier from the Data Observatory ([see available measures](https://cartodb.github.io/bigmetadata/observatory.pdf)) +boundary_id | source of geometries to pull measure from (e.g., 'us.census.tiger.census_tract') +time_span (optional) | time span of interest (e.g., 2010 - 2014) + +#### Returns + +A NUMERIC value + +Key | Description +--- | --- +value | the raw measure associated with `geom_ref` + +#### Example + +Add a measure to an empty column based on county geoids in your table + +```SQL +UPDATE tablename +SET household_count = OBS_GetMeasureById(geoid_column, 'us.census.acs.B11001001', 'us.census.tiger.county') +``` + +#### Errors + +* Returns `NULL` if there is a mismatch between the geometry reference and the boundary id such as using the geoid of a county with the boundary of block groups + ## OBS_GetCategory(point geometry, category_id text) The ```OBS_GetCategory(point, category_id)``` function returns any Data Observatory Category value at a point location. The Categories available are currently limited to Segmentation categories. See the Segmentation section of the [Catalog](https://cartodb.github.io/bigmetadata/observatory.pdf) for more detail. From 350228283592189a2f3a7f60c05ed482498594a2 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 24 May 2016 13:30:22 -0400 Subject: [PATCH 08/10] adds more description for timespan --- doc/measures_functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/measures_functions.md b/doc/measures_functions.md index 2403e1b..be7e9f8 100644 --- a/doc/measures_functions.md +++ b/doc/measures_functions.md @@ -145,7 +145,7 @@ Name |Description geom_ref | a geometry reference (e.g., a US Census geoid) measure_id | a measure identifier from the Data Observatory ([see available measures](https://cartodb.github.io/bigmetadata/observatory.pdf)) boundary_id | source of geometries to pull measure from (e.g., 'us.census.tiger.census_tract') -time_span (optional) | time span of interest (e.g., 2010 - 2014) +time_span (optional) | time span of interest (e.g., 2010 - 2014). If `NULL` is passed, the measure from the most recent data will be used. #### Returns From 49d584822c07d61d9a073dbb24f594612cb275bc Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 26 May 2016 13:28:56 -0400 Subject: [PATCH 09/10] adds another test on different types of inputs --- src/pg/test/expected/41_observatory_augmentation_test.out | 3 +++ src/pg/test/sql/41_observatory_augmentation_test.sql | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/src/pg/test/expected/41_observatory_augmentation_test.out b/src/pg/test/expected/41_observatory_augmentation_test.out index 959549c..8311660 100644 --- a/src/pg/test/expected/41_observatory_augmentation_test.out +++ b/src/pg/test/expected/41_observatory_augmentation_test.out @@ -79,6 +79,9 @@ t obs_getmeasurebyid_cartodb_census_tract t (1 row) +obs_getmeasurebyid_null_boundary_null_timespan +t +(1 row) obs_getmeasurebyid_cartodb_block_group t (1 row) diff --git a/src/pg/test/sql/41_observatory_augmentation_test.sql b/src/pg/test/sql/41_observatory_augmentation_test.sql index 3370e78..49513c6 100644 --- a/src/pg/test/sql/41_observatory_augmentation_test.sql +++ b/src/pg/test/sql/41_observatory_augmentation_test.sql @@ -212,6 +212,14 @@ SELECT (cdb_observatory.OBS_GetMeasureById( '2010 - 2014' ) - 3241) / 3241 < 0.0001 As OBS_GetMeasureById_cartodb_census_tract; +-- OBS_GetMeasureById tests +SELECT cdb_observatory.OBS_GetMeasureById( + '36047048500', + 'us.census.acs.B01003001', + NULL, + NULL +) IS NULL As OBS_GetMeasureById_null_boundary_null_timespan; + SELECT (cdb_observatory.OBS_GetMeasureById( '360470485002', 'us.census.acs.B01003001', From 076e285ed7d9be84d455fe4f48addadd8cbdb633 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Thu, 26 May 2016 13:37:02 -0400 Subject: [PATCH 10/10] adds notes on test calls --- src/pg/test/sql/41_observatory_augmentation_test.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pg/test/sql/41_observatory_augmentation_test.sql b/src/pg/test/sql/41_observatory_augmentation_test.sql index 92eb224..8a3f023 100644 --- a/src/pg/test/sql/41_observatory_augmentation_test.sql +++ b/src/pg/test/sql/41_observatory_augmentation_test.sql @@ -205,6 +205,7 @@ SELECT cdb_observatory.OBS_GetUSCensusCategory( -- OBS_GetMeasureById tests +-- typical query SELECT (cdb_observatory.OBS_GetMeasureById( '36047048500', 'us.census.acs.B01003001', @@ -212,7 +213,7 @@ SELECT (cdb_observatory.OBS_GetMeasureById( '2010 - 2014' ) - 3241) / 3241 < 0.0001 As OBS_GetMeasureById_cartodb_census_tract; --- OBS_GetMeasureById tests +-- no boundary_id should give null SELECT cdb_observatory.OBS_GetMeasureById( '36047048500', 'us.census.acs.B01003001', @@ -220,6 +221,7 @@ SELECT cdb_observatory.OBS_GetMeasureById( NULL ) IS NULL As OBS_GetMeasureById_null_boundary_null_timespan; +-- query at block_group level SELECT (cdb_observatory.OBS_GetMeasureById( '360470485002', 'us.census.acs.B01003001',