From 111d07c80a785a907e44e43db93dbf47390a77cf Mon Sep 17 00:00:00 2001 From: John Krauss Date: Thu, 5 May 2016 14:31:16 -0400 Subject: [PATCH 1/7] update test --- src/pg/test/expected/40_observatory_utility_test.out | 2 +- src/pg/test/sql/40_observatory_utility_test.sql | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/pg/test/expected/40_observatory_utility_test.out b/src/pg/test/expected/40_observatory_utility_test.out index 2895652..bb7b556 100644 --- a/src/pg/test/expected/40_observatory_utility_test.out +++ b/src/pg/test/expected/40_observatory_utility_test.out @@ -1,4 +1,4 @@ -\i test/sql/load_fixtures.sql +\i test/fixtures/load_fixtures.sql SET client_min_messages TO WARNING; \set ECHO none Loading obs_table.sql fixture file... diff --git a/src/pg/test/sql/40_observatory_utility_test.sql b/src/pg/test/sql/40_observatory_utility_test.sql index 2fe41bd..2fbc183 100644 --- a/src/pg/test/sql/40_observatory_utility_test.sql +++ b/src/pg/test/sql/40_observatory_utility_test.sql @@ -1,6 +1,6 @@ -\i test/sql/load_fixtures.sql - +\pset format unaligned \set ECHO all +\i test/fixtures/load_fixtures.sql -- OBS_GeomTable -- get table with known geometry_id @@ -80,4 +80,4 @@ SELECT cdb_observatory._OBS_GetRelatedColumn( -- should give back a standardized measure name SELECT cdb_observatory._OBS_StandardizeMeasureName('test 343 %% 2 qqq }}{{}}'); -\i test/sql/drop_fixtures.sql +\i test/fixtures/drop_fixtures.sql From 7aabc1be76469186f70b6dd78314cd57fa76ee86 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Thu, 5 May 2016 14:36:07 -0400 Subject: [PATCH 2/7] update expectations for 40 --- .../expected/40_observatory_utility_test.out | 169 ++---------------- 1 file changed, 15 insertions(+), 154 deletions(-) diff --git a/src/pg/test/expected/40_observatory_utility_test.out b/src/pg/test/expected/40_observatory_utility_test.out index bb7b556..06d4958 100644 --- a/src/pg/test/expected/40_observatory_utility_test.out +++ b/src/pg/test/expected/40_observatory_utility_test.out @@ -1,166 +1,27 @@ \i test/fixtures/load_fixtures.sql SET client_min_messages TO WARNING; \set ECHO none -Loading obs_table.sql fixture file... -Done. -Loading obs_column.sql fixture file... -Done. -Loading obs_column_table.sql fixture file... -Done. -Loading obs_column_to_column.sql fixture file... -Done. -Loading obs_column_tag.sql fixture file... -Done. -Loading obs_tag.sql fixture file... -Done. -Loading obs_85328201013baa14e8e8a4a57a01e6f6fbc5f9b1.sql fixture file... -Done. -Loading obs_3e7cc9cfd403b912c57b42d5f9195af9ce2f3cdb.sql fixture file... -Done. -Loading obs_ab038198aaab3f3cb055758638ee4de28ad70146.sql fixture file... -Done. -Loading obs_a92e1111ad3177676471d66bb8036e6d057f271b.sql fixture file... -Done. -Loading obs_11ee8b82c877c073438bc935a91d3dfccef875d1.sql fixture file... -Done. -Loading obs_d34555209878e8c4b37cf0b2b3d072ff129ec470.sql fixture file... -Done. -Loading obs_b0ef6dd68d5faddbf231fd7f02916b3d00ec43c4.sql fixture file... -Done. --- OBS_GeomTable --- get table with known geometry_id --- should give back a table like obs_{hex hash} -SELECT - cdb_observatory._OBS_GeomTable( - ST_SetSRID(ST_Point(-74.0059, 40.7128), 4326), - 'us.census.tiger.census_tract' - ); - _obs_geomtable ----------------------------------------------- - obs_a92e1111ad3177676471d66bb8036e6d057f271b +_obs_geomtable +obs_a92e1111ad3177676471d66bb8036e6d057f271b (1 row) +_obs_geomtable --- get null for unknown geometry_id --- should give back null -SELECT - cdb_observatory._OBS_GeomTable( - ST_SetSRID(ST_Point(-74.0059, 40.7128), 4326), - 'us.census.tiger.nonexistant_id' -- not in catalog - ); - _obs_geomtable ----------------- - (1 row) - --- future test: give back nulls when geometry doesn't intersect --- SELECT --- cdb_observatory._OBS_GeomTable( --- ST_SetSRID(ST_Point(0,0)), -- should give back null since it's in the ocean? --- 'us.census.tiger.census_tract' --- ); --- OBS_GetColumnData --- should give back: --- colname | tablename | aggregate --- -----------|-----------------|----------- --- geoid | obs_{hex table} | null --- total_pop | obs_{hex table} | sum -WITH result as ( -SELECT - array_agg(a) expected from cdb_observatory._OBS_GetColumnData( - 'us.census.tiger.census_tract', - Array['us.census.tiger.census_tract_geoid', 'us.census.acs.B01001001'], - '2009 - 2013') a -) -select (expected)[1]::text = '{"colname":"geoid","tablename":"obs_d34555209878e8c4b37cf0b2b3d072ff129ec470","aggregate":null,"name":"US Census Tract Geoids","type":"Text","description":""}' as test_get_obs_column_with_geoid_and_census_1, - (expected)[2]::text = '{"colname":"geoid","tablename":"obs_ab038198aaab3f3cb055758638ee4de28ad70146","aggregate":null,"name":"US Census Tract Geoids","type":"Text","description":""}' as test_get_obs_column_with_geoid_and_census_2 -from result; - test_get_obs_column_with_geoid_and_census_1 | test_get_obs_column_with_geoid_and_census_2 ----------------------------------------------+--------------------------------------------- - t | t +test_get_obs_column_with_geoid_and_census_1|test_get_obs_column_with_geoid_and_census_2 +t|t (1 row) - --- should be null-valued -WITH result as ( -SELECT - array_agg(a) expected from cdb_observatory._OBS_GetColumnData( - 'us.census.tiger.census_tract', - Array['us.census.tiger.baloney'], - '2009 - 2013') a -) -select expected is null as OBS_GetColumnData_missing_measure -from result; - obs_getcolumndata_missing_measure ------------------------------------ - t +obs_getcolumndata_missing_measure +t (1 row) - --- OBS_BuildSnapshotQuery --- Should give back: SELECT vals[1] As total_pop, vals[2] As male_pop, vals[3] As female_pop, vals[4] As median_age -SELECT - cdb_observatory._OBS_BuildSnapshotQuery( - Array['total_pop','male_pop','female_pop','median_age'] - ); - _obs_buildsnapshotquery -------------------------------------------------------------------------------------------------- - SELECT vals[1] As total_pop, vals[2] As male_pop, vals[3] As female_pop, vals[4] As median_age +_obs_buildsnapshotquery +SELECT vals[1] As total_pop, vals[2] As male_pop, vals[3] As female_pop, vals[4] As median_age (1 row) - --- should give back: SELECT vals[1] As mandarin_orange -SELECT - cdb_observatory._OBS_BuildSnapshotQuery( - Array['mandarin_orange'] - ); - _obs_buildsnapshotquery ------------------------------------- - SELECT vals[1] As mandarin_orange +_obs_buildsnapshotquery +SELECT vals[1] As mandarin_orange (1 row) - -SELECT cdb_observatory._OBS_GetRelatedColumn( - Array[ - 'es.ine.pop_0_4', - 'us.census.acs.B01001001', - 'us.census.acs.B01001002' - ], - 'denominator' - ); - _obs_getrelatedcolumn -------------------------------------------------- - {es.ine.total_pop,NULL,us.census.acs.B01001001} +_obs_getrelatedcolumn +{es.ine.total_pop,NULL,us.census.acs.B01001001} (1 row) - --- should give back a standardized measure name -SELECT cdb_observatory._OBS_StandardizeMeasureName('test 343 %% 2 qqq }}{{}}'); - _obs_standardizemeasurename ------------------------------ - test_343_2_qqq +_obs_standardizemeasurename +test_343_2_qqq (1 row) - -\i test/sql/drop_fixtures.sql -SET client_min_messages TO NOTICE; -\set ECHO none -Dropping obs_table.sql fixture table... -Done. -Dropping obs_column.sql fixture table... -Done. -Dropping obs_column_table.sql fixture table... -Done. -Dropping obs_column_to_column.sql fixture table... -Done. -Dropping obs_column_tag.sql fixture table... -Done. -Dropping obs_tag.sql fixture table... -Done. -Dropping obs_85328201013baa14e8e8a4a57a01e6f6fbc5f9b1 fixture table... -Done. -Dropping obs_3e7cc9cfd403b912c57b42d5f9195af9ce2f3cdb fixture table... -Done. -Dropping obs_ab038198aaab3f3cb055758638ee4de28ad70146 fixture table... -Done. -Dropping obs_a92e1111ad3177676471d66bb8036e6d057f271b fixture table... -Done. -Dropping obs_11ee8b82c877c073438bc935a91d3dfccef875d1 fixture table... -Done. -Dropping obs_d34555209878e8c4b37cf0b2b3d072ff129ec470 fixture table... -Done. -Dropping obs_b0ef6dd68d5faddbf231fd7f02916b3d00ec43c4 fixture table... -Done. From 73f8ea1b4ef40c07ddab2c5e17e2b80e2389981c Mon Sep 17 00:00:00 2001 From: John Krauss Date: Thu, 5 May 2016 14:45:56 -0400 Subject: [PATCH 3/7] allow _obs_geomtable to take a timespan, and default to most recent --- src/pg/sql/40_observatory_utility.sql | 9 ++++++--- .../test/sql/40_observatory_utility_test.sql | 18 +++++++++++++++--- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/pg/sql/40_observatory_utility.sql b/src/pg/sql/40_observatory_utility.sql index cb5611d..87c5c8b 100644 --- a/src/pg/sql/40_observatory_utility.sql +++ b/src/pg/sql/40_observatory_utility.sql @@ -5,7 +5,8 @@ -- geometries. CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GeomTable( geom geometry, - geometry_id text + geometry_id text, + time_span text DEFAULT NULL ) RETURNS TEXT AS $$ @@ -22,10 +23,12 @@ BEGIN WHERE type ILIKE ''geometry'' AND coltable.column_id = col.id AND coltable.table_id = tab.id - AND col.id = $1 + AND col.id = $1, + AND CASE WHEN $2 IS NOT NULL THEN timespan = $2 ELSE TRUE END + ORDER BY timespan DESC LIMIT 1 ) ' - USING geometry_id, geom + USING geometry_id, geom, time_span INTO result; return result; diff --git a/src/pg/test/sql/40_observatory_utility_test.sql b/src/pg/test/sql/40_observatory_utility_test.sql index 2fbc183..39e3d09 100644 --- a/src/pg/test/sql/40_observatory_utility_test.sql +++ b/src/pg/test/sql/40_observatory_utility_test.sql @@ -8,7 +8,8 @@ SELECT cdb_observatory._OBS_GeomTable( ST_SetSRID(ST_Point(-74.0059, 40.7128), 4326), - 'us.census.tiger.census_tract' + 'us.census.tiger.census_tract', + '2013' ); -- get null for unknown geometry_id @@ -32,6 +33,17 @@ SELECT -- -----------|-----------------|----------- -- geoid | obs_{hex table} | null -- total_pop | obs_{hex table} | sum +--WITH result as ( +--SELECT +-- array_agg(a) expected from cdb_observatory._OBS_GetColumnData( +-- 'us.census.tiger.census_tract', +-- Array['us.census.tiger.census_tract_geoid', 'us.census.acs.B01001001'], +-- '2009 - 2013') a +--) +--select (expected)[1]::text = '{"colname":"geoid","tablename":"obs_d34555209878e8c4b37cf0b2b3d072ff129ec470","aggregate":null,"name":"US Census Tract Geoids","type":"Text","description":""}' as test_get_obs_column_with_geoid_and_census_1, +-- (expected)[2]::text = '{"colname":"geoid","tablename":"obs_ab038198aaab3f3cb055758638ee4de28ad70146","aggregate":null,"name":"US Census Tract Geoids","type":"Text","description":""}' as test_get_obs_column_with_geoid_and_census_2 +--from result; + WITH result as ( SELECT array_agg(a) expected from cdb_observatory._OBS_GetColumnData( @@ -39,8 +51,8 @@ SELECT Array['us.census.tiger.census_tract_geoid', 'us.census.acs.B01001001'], '2009 - 2013') a ) -select (expected)[1]::text = '{"colname":"geoid","tablename":"obs_d34555209878e8c4b37cf0b2b3d072ff129ec470","aggregate":null,"name":"US Census Tract Geoids","type":"Text","description":""}' as test_get_obs_column_with_geoid_and_census_1, - (expected)[2]::text = '{"colname":"geoid","tablename":"obs_ab038198aaab3f3cb055758638ee4de28ad70146","aggregate":null,"name":"US Census Tract Geoids","type":"Text","description":""}' as test_get_obs_column_with_geoid_and_census_2 +select (expected)[1]::text as test_get_obs_column_with_geoid_and_census_1, + (expected)[2]::text as test_get_obs_column_with_geoid_and_census_2 from result; From 5dcec6e126b91902928bba6866bea9b54a05b73a Mon Sep 17 00:00:00 2001 From: John Krauss Date: Thu, 5 May 2016 14:48:34 -0400 Subject: [PATCH 4/7] fix comma bug --- src/pg/sql/40_observatory_utility.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg/sql/40_observatory_utility.sql b/src/pg/sql/40_observatory_utility.sql index 87c5c8b..f0eb438 100644 --- a/src/pg/sql/40_observatory_utility.sql +++ b/src/pg/sql/40_observatory_utility.sql @@ -23,7 +23,7 @@ BEGIN WHERE type ILIKE ''geometry'' AND coltable.column_id = col.id AND coltable.table_id = tab.id - AND col.id = $1, + AND col.id = $1 AND CASE WHEN $2 IS NOT NULL THEN timespan = $2 ELSE TRUE END ORDER BY timespan DESC LIMIT 1 ) From 499eb6da62943d96ac68618d11accceb08b3b14c Mon Sep 17 00:00:00 2001 From: John Krauss Date: Thu, 5 May 2016 14:49:43 -0400 Subject: [PATCH 5/7] fix casting issue --- src/pg/sql/40_observatory_utility.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pg/sql/40_observatory_utility.sql b/src/pg/sql/40_observatory_utility.sql index f0eb438..28288d3 100644 --- a/src/pg/sql/40_observatory_utility.sql +++ b/src/pg/sql/40_observatory_utility.sql @@ -24,7 +24,7 @@ BEGIN AND coltable.column_id = col.id AND coltable.table_id = tab.id AND col.id = $1 - AND CASE WHEN $2 IS NOT NULL THEN timespan = $2 ELSE TRUE END + AND CASE WHEN $2::TEXT IS NOT NULL THEN timespan ILIKE $2::TEXT ELSE TRUE END ORDER BY timespan DESC LIMIT 1 ) ' From e306408a6ad62cbc1c844769a3b0e2ea2a0d5505 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Thu, 5 May 2016 14:58:06 -0400 Subject: [PATCH 6/7] fix fixture, test and function bugs --- src/pg/sql/40_observatory_utility.sql | 2 +- .../expected/40_observatory_utility_test.out | 2 +- .../test/sql/40_observatory_utility_test.sql | 18 +++--------------- 3 files changed, 5 insertions(+), 17 deletions(-) diff --git a/src/pg/sql/40_observatory_utility.sql b/src/pg/sql/40_observatory_utility.sql index 28288d3..03164e5 100644 --- a/src/pg/sql/40_observatory_utility.sql +++ b/src/pg/sql/40_observatory_utility.sql @@ -24,7 +24,7 @@ BEGIN AND coltable.column_id = col.id AND coltable.table_id = tab.id AND col.id = $1 - AND CASE WHEN $2::TEXT IS NOT NULL THEN timespan ILIKE $2::TEXT ELSE TRUE END + AND CASE WHEN $3::TEXT IS NOT NULL THEN timespan ILIKE $3::TEXT ELSE TRUE END ORDER BY timespan DESC LIMIT 1 ) ' diff --git a/src/pg/test/expected/40_observatory_utility_test.out b/src/pg/test/expected/40_observatory_utility_test.out index 06d4958..364f836 100644 --- a/src/pg/test/expected/40_observatory_utility_test.out +++ b/src/pg/test/expected/40_observatory_utility_test.out @@ -2,7 +2,7 @@ SET client_min_messages TO WARNING; \set ECHO none _obs_geomtable -obs_a92e1111ad3177676471d66bb8036e6d057f271b +obs_fc050f0b8673cfe3c6aa1040f749eb40975691b7 (1 row) _obs_geomtable diff --git a/src/pg/test/sql/40_observatory_utility_test.sql b/src/pg/test/sql/40_observatory_utility_test.sql index 39e3d09..d0fbf93 100644 --- a/src/pg/test/sql/40_observatory_utility_test.sql +++ b/src/pg/test/sql/40_observatory_utility_test.sql @@ -9,7 +9,7 @@ SELECT cdb_observatory._OBS_GeomTable( ST_SetSRID(ST_Point(-74.0059, 40.7128), 4326), 'us.census.tiger.census_tract', - '2013' + '2014' ); -- get null for unknown geometry_id @@ -33,17 +33,6 @@ SELECT -- -----------|-----------------|----------- -- geoid | obs_{hex table} | null -- total_pop | obs_{hex table} | sum ---WITH result as ( ---SELECT --- array_agg(a) expected from cdb_observatory._OBS_GetColumnData( --- 'us.census.tiger.census_tract', --- Array['us.census.tiger.census_tract_geoid', 'us.census.acs.B01001001'], --- '2009 - 2013') a ---) ---select (expected)[1]::text = '{"colname":"geoid","tablename":"obs_d34555209878e8c4b37cf0b2b3d072ff129ec470","aggregate":null,"name":"US Census Tract Geoids","type":"Text","description":""}' as test_get_obs_column_with_geoid_and_census_1, --- (expected)[2]::text = '{"colname":"geoid","tablename":"obs_ab038198aaab3f3cb055758638ee4de28ad70146","aggregate":null,"name":"US Census Tract Geoids","type":"Text","description":""}' as test_get_obs_column_with_geoid_and_census_2 ---from result; - WITH result as ( SELECT array_agg(a) expected from cdb_observatory._OBS_GetColumnData( @@ -51,11 +40,10 @@ SELECT Array['us.census.tiger.census_tract_geoid', 'us.census.acs.B01001001'], '2009 - 2013') a ) -select (expected)[1]::text as test_get_obs_column_with_geoid_and_census_1, - (expected)[2]::text as test_get_obs_column_with_geoid_and_census_2 +select (expected)[1]::text = '{"colname":"geoid","tablename":"obs_d34555209878e8c4b37cf0b2b3d072ff129ec470","aggregate":null,"name":"US Census Tracts Geoids","type":"Text","description":null}' as test_get_obs_column_with_geoid_and_census_1, + (expected)[2]::text = '{"colname":"geoid","tablename":"obs_ab038198aaab3f3cb055758638ee4de28ad70146","aggregate":null,"name":"US Census Tracts Geoids","type":"Text","description":null}' as test_get_obs_column_with_geoid_and_census_2 from result; - -- should be null-valued WITH result as ( SELECT From 290ca3cb20e8f3376102b34b888a2fd7d5914822 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Thu, 5 May 2016 15:00:03 -0400 Subject: [PATCH 7/7] a few more lines --- src/pg/test/expected/40_observatory_utility_test.out | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pg/test/expected/40_observatory_utility_test.out b/src/pg/test/expected/40_observatory_utility_test.out index 364f836..4a8e6c8 100644 --- a/src/pg/test/expected/40_observatory_utility_test.out +++ b/src/pg/test/expected/40_observatory_utility_test.out @@ -1,3 +1,5 @@ +\pset format unaligned +\set ECHO all \i test/fixtures/load_fixtures.sql SET client_min_messages TO WARNING; \set ECHO none