From 14ab3bef6a5eb324f525a5b470def5e9e17bc800 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Thu, 5 May 2016 11:37:04 -0400 Subject: [PATCH 1/3] remove _obs_getcensus and make demographicsnapshot use obs_get directly --- src/pg/sql/41_observatory_augmentation.sql | 380 +++--------------- .../sql/41_observatory_augmentation_test.sql | 42 -- 2 files changed, 62 insertions(+), 360 deletions(-) diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index bd5ac1c..d53679f 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -28,18 +28,18 @@ AS $$ DECLARE target_cols text[]; BEGIN - target_cols := Array['total_pop', - 'male_pop', - 'female_pop', - 'median_age', - 'white_pop', - 'black_pop', - 'asian_pop', - 'hispanic_pop', - 'amerindian_pop', - 'other_race_pop', - 'two_or_more_races_pop', - 'not_hispanic_pop', + target_cols := Array['us.census.acs.B01001001', + 'us.census.acs.B01001002', + 'us.census.acs.B01001026', + 'us.census.acs.B01002001', + 'us.census.acs.B03002003', + 'us.census.acs.B03002004', + 'us.census.acs.B03002006', + 'us.census.acs.B03002012', + 'us.census.acs.B03002005', + 'us.census.acs.B03002008', + 'us.census.acs.B03002009', + 'us.census.acs.B03002002', --'not_us_citizen_pop', --'workers_16_and_over', --'commuters_by_car_truck_van', @@ -51,339 +51,83 @@ AS $$ --'walked_to_work', --'worked_at_home', --'children', - 'households', + 'us.census.acs.B11001001', --'population_3_years_over', --'in_school', --'in_grades_1_to_4', --'in_grades_5_to_8', --'in_grades_9_to_12', --'in_undergrad_college', - 'pop_25_years_over', - 'high_school_diploma', - 'less_one_year_college', - 'one_year_more_college', - 'associates_degree', - 'bachelors_degree', - 'masters_degree', + 'us.census.acs.B15003001', + 'us.census.acs.B15003017', + 'us.census.acs.B15003019', + 'us.census.acs.B15003020', + 'us.census.acs.B15003021', + 'us.census.acs.B15003022', + 'us.census.acs.B15003023', --'pop_5_years_over', --'speak_only_english_at_home', --'speak_spanish_at_home', --'pop_determined_poverty_status', --'poverty', - 'median_income', - 'gini_index', - 'income_per_capita', - 'housing_units', - 'vacant_housing_units', - 'vacant_housing_units_for_rent', - 'vacant_housing_units_for_sale', - 'median_rent', - 'percent_income_spent_on_rent', - 'owner_occupied_housing_units', - 'million_dollar_housing_units', - 'mortgaged_housing_units', + 'us.census.acs.B19013001', + 'us.census.acs.B19083001', + 'us.census.acs.B19301001', + 'us.census.acs.B25001001', + 'us.census.acs.B25002003', + 'us.census.acs.B25004002', + 'us.census.acs.B25004004', + 'us.census.acs.B25058001', + 'us.census.acs.B25071001', + 'us.census.acs.B25075001', + 'us.census.acs.B25075025', + 'us.census.acs.B25081002', --'pop_15_and_over', --'pop_never_married', --'pop_now_married', --'pop_separated', --'pop_widowed', --'pop_divorced', - 'commuters_16_over', - 'commute_less_10_mins', - 'commute_10_14_mins', - 'commute_15_19_mins', - 'commute_20_24_mins', - 'commute_25_29_mins', - 'commute_30_34_mins', - 'commute_35_44_mins', - 'commute_45_59_mins', - 'commute_60_more_mins', - 'aggregate_travel_time_to_work', - 'income_less_10000', - 'income_10000_14999', - 'income_15000_19999', - 'income_20000_24999', - 'income_25000_29999', - 'income_30000_34999', - 'income_35000_39999', - 'income_40000_44999', - 'income_45000_49999', - 'income_50000_59999', - 'income_60000_74999', - 'income_75000_99999', - 'income_100000_124999', - 'income_125000_149999', - 'income_150000_199999', - 'income_200000_or_more', - 'land_area']; + 'us.census.acs.B08134001', + 'us.census.acs.B08134002', + 'us.census.acs.B08134003', + 'us.census.acs.B08134004', + 'us.census.acs.B08134005', + 'us.census.acs.B08134006', + 'us.census.acs.B08134007', + 'us.census.acs.B08134008', + 'us.census.acs.B08134009', + 'us.census.acs.B08134010', + 'us.census.acs.B08135001', + 'us.census.acs.B19001002', + 'us.census.acs.B19001003', + 'us.census.acs.B19001004', + 'us.census.acs.B19001005', + 'us.census.acs.B19001006', + 'us.census.acs.B19001007', + 'us.census.acs.B19001008', + 'us.census.acs.B19001009', + 'us.census.acs.B19001010', + 'us.census.acs.B19001011', + 'us.census.acs.B19001012', + 'us.census.acs.B19001013', + 'us.census.acs.B19001014', + 'us.census.acs.B19001015', + 'us.census.acs.B19001016', + 'us.census.acs.B19001017', + NULL]; -- land_area RETURN QUERY EXECUTE - 'select * from cdb_observatory._OBS_GetCensus($1, $2 )' - USING geom, target_cols + 'select * from cdb_observatory._OBS_Get($1, $2, $3, $4 )' + USING geom, target_cols, time_span, geometry_level RETURN; END; $$ LANGUAGE plpgsql; --- CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetDemographicSnapshot(geom geometry, time_span text default '2009 - 2013', geometry_level text default 'us.census.tiger.block_group' ) --- RETURNS TABLE( --- total_pop NUMERIC, --- male_pop NUMERIC, --- female_pop NUMERIC, --- median_age NUMERIC, --- white_pop NUMERIC, --- black_pop NUMERIC, --- asian_pop NUMERIC, --- hispanic_pop NUMERIC, --- amerindian_pop NUMERIC, --- other_race_pop NUMERIC, --- two_or_more_races_pop NUMERIC, --- not_hispanic_pop NUMERIC, --- --not_us_citizen_pop NUMERIC, --- --workers_16_and_over NUMERIC, --- --commuters_by_car_truck_van NUMERIC, --- --commuters_drove_alone NUMERIC, --- --commuters_by_carpool NUMERIC, --- --commuters_by_public_transportation NUMERIC, --- --commuters_by_bus NUMERIC, --- --commuters_by_subway_or_elevated NUMERIC, --- --walked_to_work NUMERIC, --- --worked_at_home NUMERIC, --- --children NUMERIC, -- TODO we should be able to get this at BG --- households NUMERIC, --- --population_3_years_over NUMERIC, --- --in_school NUMERIC, --- --in_grades_1_to_4 NUMERIC, --- --in_grades_5_to_8 NUMERIC, --- --in_grades_9_to_12 NUMERIC, --- --in_undergrad_college NUMERIC, --- pop_25_years_over NUMERIC, --- high_school_diploma NUMERIC, --- less_one_year_college NUMERIC, --- one_year_more_college NUMERIC, --- associates_degree NUMERIC, --- bachelors_degree NUMERIC, --- masters_degree NUMERIC, --- --pop_5_years_over NUMERIC, --- --speak_only_english_at_home NUMERIC, --- --speak_spanish_at_home NUMERIC, --- --pop_determined_poverty_status NUMERIC, --- --poverty NUMERIC, --- median_income NUMERIC, --- gini_index NUMERIC, --- income_per_capita NUMERIC, --- housing_units NUMERIC, --- vacant_housing_units NUMERIC, --- vacant_housing_units_for_rent NUMERIC, --- vacant_housing_units_for_sale NUMERIC, --- median_rent NUMERIC, --- percent_income_spent_on_rent NUMERIC, --- owner_occupied_housing_units NUMERIC, --- million_dollar_housing_units NUMERIC, --- mortgaged_housing_units NUMERIC, --- --pop_15_and_over NUMERIC, --- --pop_never_married NUMERIC, --- --pop_now_married NUMERIC, --- --pop_separated NUMERIC, --- --pop_widowed NUMERIC, --- --pop_divorced NUMERIC, --- commuters_16_over NUMERIC, --- commute_less_10_mins NUMERIC, --- commute_10_14_mins NUMERIC, --- commute_15_19_mins NUMERIC, --- commute_20_24_mins NUMERIC, --- commute_25_29_mins NUMERIC, --- commute_30_34_mins NUMERIC, --- commute_35_44_mins NUMERIC, --- commute_45_59_mins NUMERIC, --- commute_60_more_mins NUMERIC, --- aggregate_travel_time_to_work NUMERIC, --- income_less_10000 NUMERIC, --- income_10000_14999 NUMERIC, --- income_15000_19999 NUMERIC, --- income_20000_24999 NUMERIC, --- income_25000_29999 NUMERIC, --- income_30000_34999 NUMERIC, --- income_35000_39999 NUMERIC, --- income_40000_44999 NUMERIC, --- income_45000_49999 NUMERIC, --- income_50000_59999 NUMERIC, --- income_60000_74999 NUMERIC, --- income_75000_99999 NUMERIC, --- income_100000_124999 NUMERIC, --- income_125000_149999 NUMERIC, --- income_150000_199999 NUMERIC, --- income_200000_or_more NUMERIC, --- land_area NUMERIC) --- AS $$ --- DECLARE --- target_cols text[]; --- names text[]; --- vals NUMERIC[]; --- q text; --- BEGIN --- target_cols := Array['total_pop', --- 'male_pop', --- 'female_pop', --- 'median_age', --- 'white_pop', --- 'black_pop', --- 'asian_pop', --- 'hispanic_pop', --- 'amerindian_pop', --- 'other_race_pop', --- 'two_or_more_races_pop', --- 'not_hispanic_pop', --- --'not_us_citizen_pop', --- --'workers_16_and_over', --- --'commuters_by_car_truck_van', --- --'commuters_drove_alone', --- --'commuters_by_carpool', --- --'commuters_by_public_transportation', --- --'commuters_by_bus', --- --'commuters_by_subway_or_elevated', --- --'walked_to_work', --- --'worked_at_home', --- --'children', --- 'households', --- --'population_3_years_over', --- --'in_school', --- --'in_grades_1_to_4', --- --'in_grades_5_to_8', --- --'in_grades_9_to_12', --- --'in_undergrad_college', --- 'pop_25_years_over', --- 'high_school_diploma', --- 'less_one_year_college', --- 'one_year_more_college', --- 'associates_degree', --- 'bachelors_degree', --- 'masters_degree', --- --'pop_5_years_over', --- --'speak_only_english_at_home', --- --'speak_spanish_at_home', --- --'pop_determined_poverty_status', --- --'poverty', --- 'median_income', --- 'gini_index', --- 'income_per_capita', --- 'housing_units', --- 'vacant_housing_units', --- 'vacant_housing_units_for_rent', --- 'vacant_housing_units_for_sale', --- 'median_rent', --- 'percent_income_spent_on_rent', --- 'owner_occupied_housing_units', --- 'million_dollar_housing_units', --- 'mortgaged_housing_units', --- --'pop_15_and_over', --- --'pop_never_married', --- --'pop_now_married', --- --'pop_separated', --- --'pop_widowed', --- --'pop_divorced', --- 'commuters_16_over', --- 'commute_less_10_mins', --- 'commute_10_14_mins', --- 'commute_15_19_mins', --- 'commute_20_24_mins', --- 'commute_25_29_mins', --- 'commute_30_34_mins', --- 'commute_35_44_mins', --- 'commute_45_59_mins', --- 'commute_60_more_mins', --- 'aggregate_travel_time_to_work', --- 'income_less_10000', --- 'income_10000_14999', --- 'income_15000_19999', --- 'income_20000_24999', --- 'income_25000_29999', --- 'income_30000_34999', --- 'income_35000_39999', --- 'income_40000_44999', --- 'income_45000_49999', --- 'income_50000_59999', --- 'income_60000_74999', --- 'income_75000_99999', --- 'income_100000_124999', --- 'income_125000_149999', --- 'income_150000_199999', --- 'income_200000_or_more', --- 'land_area']; --- --- q := --- $query$ --- WITH a As ( --- SELECT --- array_agg(_OBS_GetCensusJ->>'name') As names, --- array_agg(_OBS_GetCensusJ->>'value') As vals --- FROM cdb_observatory._OBS_GetCensusJ($1,$2,$3,$4) --- )$query$ || --- cdb_observatory._OBS_BuildSnapshotQuery(target_cols) || --- ' FROM a' --- ; --- --- RETURN QUERY --- EXECUTE --- q --- USING geom, target_cols, time_span, geometry_level; --- --- RETURN; --- END; --- $$ LANGUAGE plpgsql; - --Base functions for performing augmentation ---------------------------------------------------------------------------------------- - ---Returns arrays of values for the given census dimension names for a given ---point or polygon -CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetCensus( - geom geometry, - dimension_names text[], - time_span text DEFAULT '2009 - 2013', - geometry_level text DEFAULT 'us.census.tiger.block_group' -) -RETURNS SETOF JSON -AS $$ -DECLARE - ids text[]; -BEGIN - - ids := cdb_observatory._OBS_LookupCensusHuman(dimension_names); - - RETURN QUERY - SELECT * FROM cdb_observatory._OBS_Get(geom, ids, time_span, geometry_level); -END; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetCensus( - geom geometry, - dimension_name text, - time_span text DEFAULT '2009 - 2013', - geometry_level text DEFAULT 'us.census.tiger.block_group' -) -RETURNS NUMERIC -AS $$ -DECLARE - ids Text[]; - result_json json; - result Numeric; -BEGIN - - ids := cdb_observatory._OBS_LookupCensusHuman(Array[dimension_name]); - result_json := (SELECT a FROM cdb_observatory._OBS_Get(geom, ids, time_span, geometry_level) as a limit 1); - EXECUTE - format('select $1::numeric as "%s"', result_json->>'name') - INTO result - USING - result_json->>'value'; - - return result; -END; -$$ LANGUAGE plpgsql; - - -- Base augmentation fucntion. CREATE OR REPLACE FUNCTION cdb_observatory._OBS_Get( geom geometry, diff --git a/src/pg/test/sql/41_observatory_augmentation_test.sql b/src/pg/test/sql/41_observatory_augmentation_test.sql index 11609df..5fab313 100644 --- a/src/pg/test/sql/41_observatory_augmentation_test.sql +++ b/src/pg/test/sql/41_observatory_augmentation_test.sql @@ -5,48 +5,6 @@ WITH result as( FROM cdb_observatory.OBS_GetDemographicSnapshot(cdb_observatory._TestPoint()) ) select expected_columns ='58' as OBS_GetDemographicSnapshot_test_no_returns FROM result; --- --- dimension | dimension_value --- ----------|---------------- --- total_pop | 9516.27915900609 --- male_pop | 6152.51885204623 - -WITH result as ( - SELECT array_agg(_obs_getcensus->>'value') as b - FROM( select * from - cdb_observatory._OBS_GetCensus( - cdb_observatory._TestPoint(), - Array['total_pop','male_pop']::text[] - )) a -) -select b='{9516.27915900609,6152.51885204623}' - as test_obsGetCensusWithTestPointAnd2Variables - from result; --- what happens on null island? --- expect nulls back: {female_pop, male_pop} | {NULL, NULL} - -WITH result as ( - SELECT count(vals) non_null - FROM( select _OBS_GetCensus->>'value' vals from - cdb_observatory._OBS_GetCensus( - ST_Buffer(ST_SetSRID(ST_Point(0, 0), 4326)::geography, 5000)::geometry, - Array['total_pop','male_pop']::text[] - )) a -) -SELECT non_null = 0 as test_obsGetCensusWithNullIslandArea -FROM result; - --- expect nulls back {female_pop, male_pop} | {NULL, NULL} -WITH result as ( - SELECT count(vals) non_null - FROM( select _OBS_GetCensus->>'value' vals from - cdb_observatory._OBS_GetCensus( - ST_SetSRID(ST_Point(0, 0), 4326), - Array['total_pop','male_pop']::text[] - )) a -) -SELECT non_null = 0 as test_obsGetCensusWithNullIsland -FROM result; -- -- names | vals From 5d20bd0804879acd77c9a3f4752ccc4de3f57700 Mon Sep 17 00:00:00 2001 From: John Krauss Date: Thu, 5 May 2016 11:43:16 -0400 Subject: [PATCH 2/3] remove test expectations --- .../expected/41_observatory_augmentation_test.out | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/pg/test/expected/41_observatory_augmentation_test.out b/src/pg/test/expected/41_observatory_augmentation_test.out index 4ad4fdc..23ef1ec 100644 --- a/src/pg/test/expected/41_observatory_augmentation_test.out +++ b/src/pg/test/expected/41_observatory_augmentation_test.out @@ -32,21 +32,6 @@ Done. t (1 row) - test_obsgetcensuswithtestpointand2variables ---------------------------------------------- - t -(1 row) - - test_obsgetcensuswithnullislandarea -------------------------------------- - t -(1 row) - - test_obsgetcensuswithnullisland ---------------------------------- - t -(1 row) - obs_get_gini_index_at_test_point ---------------------------------- t From 3299bb013b1778def962acfed82040fc1d527f1b Mon Sep 17 00:00:00 2001 From: John Krauss Date: Thu, 5 May 2016 11:46:04 -0400 Subject: [PATCH 3/3] kill obs_lookupcensushuman --- src/pg/sql/40_observatory_utility.sql | 31 ------------------- .../expected/40_observatory_utility_test.out | 21 ------------- .../test/sql/40_observatory_utility_test.sql | 13 -------- 3 files changed, 65 deletions(-) diff --git a/src/pg/sql/40_observatory_utility.sql b/src/pg/sql/40_observatory_utility.sql index 17dd50d..cb5611d 100644 --- a/src/pg/sql/40_observatory_utility.sql +++ b/src/pg/sql/40_observatory_utility.sql @@ -79,37 +79,6 @@ BEGIN END; $$ LANGUAGE plpgsql; ---Gets the column id for a census variable given a human readable version of it --- Old: OBS_LOOKUP_CENSUS_HUMAN - -CREATE OR REPLACE FUNCTION cdb_observatory._OBS_LookupCensusHuman( - column_names text[], - -- TODO: change variable name table_name to table_id - table_name text DEFAULT 'us.census.acs.extract_block_group_5yr_2013_69b156927c' -) -RETURNS text[] as $$ -DECLARE - column_id text; - result text; -BEGIN - EXECUTE format(' - WITH col_names AS ( - select row_number() over() as no, a.column_name as column_name from( - select unnest($1) as column_name - ) a - ) - select array_agg(column_id order by col_names.no) - FROM observatory.OBS_column_table,col_names - where colname = col_names.column_name - and table_id = %L limit 1 - ', table_name) - INTO result - using column_names; - RETURN result; -END -$$ LANGUAGE plpgsql; - - --Test point cause Stuart always seems to make random points in the water CREATE OR REPLACE FUNCTION cdb_observatory._TestPoint() RETURNS geometry diff --git a/src/pg/test/expected/40_observatory_utility_test.out b/src/pg/test/expected/40_observatory_utility_test.out index 93beccc..2895652 100644 --- a/src/pg/test/expected/40_observatory_utility_test.out +++ b/src/pg/test/expected/40_observatory_utility_test.out @@ -94,27 +94,6 @@ from result; t (1 row) --- OBS_LookupCensusHuman --- should give back: {"us.census.acs.B19083001"} -SELECT - cdb_observatory._OBS_LookupCensusHuman( - Array['gini_index'] - ); - _obs_lookupcensushuman ---------------------------- - {us.census.acs.B19083001} -(1 row) - --- should be empty array -SELECT - cdb_observatory._OBS_LookupCensusHuman( - Array['cookies'] - ); - _obs_lookupcensushuman ------------------------- - -(1 row) - -- OBS_BuildSnapshotQuery -- Should give back: SELECT vals[1] As total_pop, vals[2] As male_pop, vals[3] As female_pop, vals[4] As median_age SELECT diff --git a/src/pg/test/sql/40_observatory_utility_test.sql b/src/pg/test/sql/40_observatory_utility_test.sql index 3410336..2fe41bd 100644 --- a/src/pg/test/sql/40_observatory_utility_test.sql +++ b/src/pg/test/sql/40_observatory_utility_test.sql @@ -55,19 +55,6 @@ SELECT select expected is null as OBS_GetColumnData_missing_measure from result; --- OBS_LookupCensusHuman --- should give back: {"us.census.acs.B19083001"} -SELECT - cdb_observatory._OBS_LookupCensusHuman( - Array['gini_index'] - ); - --- should be empty array -SELECT - cdb_observatory._OBS_LookupCensusHuman( - Array['cookies'] - ); - -- OBS_BuildSnapshotQuery -- Should give back: SELECT vals[1] As total_pop, vals[2] As male_pop, vals[3] As female_pop, vals[4] As median_age SELECT