cartodb4/observatory-extension
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #48 from CartoDB/kill_get_census

Browse Source 
Kill get census
This commit is contained in:
john krauss
2016-05-05 12:07:29 -04:00
parent ab75d2ecd6 3299bb013b
commit a792cb5f4f
6 changed files with 62 additions and 440 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
src/pg/sql/40_observatory_utility.sql
View File

@@ -79,37 +79,6 @@ BEGIN
END;
$$ LANGUAGE plpgsql;
--Gets the column id for a census variable given a human readable version of it
-- Old: OBS_LOOKUP_CENSUS_HUMAN
CREATE OR REPLACE FUNCTION cdb_observatory._OBS_LookupCensusHuman(
column_names text[],
-- TODO: change variable name table_name to table_id
table_name text DEFAULT 'us.census.acs.extract_block_group_5yr_2013_69b156927c'
)
RETURNS text[] as $$
DECLARE
column_id text;
result text;
BEGIN
EXECUTE format('
WITH col_names AS (
select row_number() over() as no, a.column_name as column_name from(
select unnest($1) as column_name
) a
)
select array_agg(column_id order by col_names.no)
FROM observatory.OBS_column_table,col_names
where colname = col_names.column_name
and table_id = %L limit 1
', table_name)
INTO result
using column_names;
RETURN result;
END
$$ LANGUAGE plpgsql;
--Test point cause Stuart always seems to make random points in the water
CREATE OR REPLACE FUNCTION cdb_observatory._TestPoint()
RETURNS geometry

380
src/pg/sql/41_observatory_augmentation.sql
View File

@@ -28,18 +28,18 @@ AS $$
DECLARE
target_cols text[];
BEGIN
target_cols := Array['total_pop',
'male_pop',
'female_pop',
'median_age',
'white_pop',
'black_pop',
'asian_pop',
'hispanic_pop',
'amerindian_pop',
'other_race_pop',
'two_or_more_races_pop',
'not_hispanic_pop',
target_cols := Array['us.census.acs.B01001001',
'us.census.acs.B01001002',
'us.census.acs.B01001026',
'us.census.acs.B01002001',
'us.census.acs.B03002003',
'us.census.acs.B03002004',
'us.census.acs.B03002006',
'us.census.acs.B03002012',
'us.census.acs.B03002005',
'us.census.acs.B03002008',
'us.census.acs.B03002009',
'us.census.acs.B03002002',
--'not_us_citizen_pop',
--'workers_16_and_over',
--'commuters_by_car_truck_van',
@@ -51,339 +51,83 @@ AS $$
--'walked_to_work',
--'worked_at_home',
--'children',
'households',
'us.census.acs.B11001001',
--'population_3_years_over',
--'in_school',
--'in_grades_1_to_4',
--'in_grades_5_to_8',
--'in_grades_9_to_12',
--'in_undergrad_college',
'pop_25_years_over',
'high_school_diploma',
'less_one_year_college',
'one_year_more_college',
'associates_degree',
'bachelors_degree',
'masters_degree',
'us.census.acs.B15003001',
'us.census.acs.B15003017',
'us.census.acs.B15003019',
'us.census.acs.B15003020',
'us.census.acs.B15003021',
'us.census.acs.B15003022',
'us.census.acs.B15003023',
--'pop_5_years_over',
--'speak_only_english_at_home',
--'speak_spanish_at_home',
--'pop_determined_poverty_status',
--'poverty',
'median_income',
'gini_index',
'income_per_capita',
'housing_units',
'vacant_housing_units',
'vacant_housing_units_for_rent',
'vacant_housing_units_for_sale',
'median_rent',
'percent_income_spent_on_rent',
'owner_occupied_housing_units',
'million_dollar_housing_units',
'mortgaged_housing_units',
'us.census.acs.B19013001',
'us.census.acs.B19083001',
'us.census.acs.B19301001',
'us.census.acs.B25001001',
'us.census.acs.B25002003',
'us.census.acs.B25004002',
'us.census.acs.B25004004',
'us.census.acs.B25058001',
'us.census.acs.B25071001',
'us.census.acs.B25075001',
'us.census.acs.B25075025',
'us.census.acs.B25081002',
--'pop_15_and_over',
--'pop_never_married',
--'pop_now_married',
--'pop_separated',
--'pop_widowed',
--'pop_divorced',
'commuters_16_over',
'commute_less_10_mins',
'commute_10_14_mins',
'commute_15_19_mins',
'commute_20_24_mins',
'commute_25_29_mins',
'commute_30_34_mins',
'commute_35_44_mins',
'commute_45_59_mins',
'commute_60_more_mins',
'aggregate_travel_time_to_work',
'income_less_10000',
'income_10000_14999',
'income_15000_19999',
'income_20000_24999',
'income_25000_29999',
'income_30000_34999',
'income_35000_39999',
'income_40000_44999',
'income_45000_49999',
'income_50000_59999',
'income_60000_74999',
'income_75000_99999',
'income_100000_124999',
'income_125000_149999',
'income_150000_199999',
'income_200000_or_more',
'land_area'];
'us.census.acs.B08134001',
'us.census.acs.B08134002',
'us.census.acs.B08134003',
'us.census.acs.B08134004',
'us.census.acs.B08134005',
'us.census.acs.B08134006',
'us.census.acs.B08134007',
'us.census.acs.B08134008',
'us.census.acs.B08134009',
'us.census.acs.B08134010',
'us.census.acs.B08135001',
'us.census.acs.B19001002',
'us.census.acs.B19001003',
'us.census.acs.B19001004',
'us.census.acs.B19001005',
'us.census.acs.B19001006',
'us.census.acs.B19001007',
'us.census.acs.B19001008',
'us.census.acs.B19001009',
'us.census.acs.B19001010',
'us.census.acs.B19001011',
'us.census.acs.B19001012',
'us.census.acs.B19001013',
'us.census.acs.B19001014',
'us.census.acs.B19001015',
'us.census.acs.B19001016',
'us.census.acs.B19001017',
NULL]; -- land_area
RETURN QUERY
EXECUTE
'select * from cdb_observatory._OBS_GetCensus($1, $2 )'
USING geom, target_cols
'select * from cdb_observatory._OBS_Get($1, $2, $3, $4 )'
USING geom, target_cols, time_span, geometry_level
RETURN;
END;
$$ LANGUAGE plpgsql;
-- CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetDemographicSnapshot(geom geometry, time_span text default '2009 - 2013', geometry_level text default 'us.census.tiger.block_group' )
-- RETURNS TABLE(
-- total_pop NUMERIC,
-- male_pop NUMERIC,
-- female_pop NUMERIC,
-- median_age NUMERIC,
-- white_pop NUMERIC,
-- black_pop NUMERIC,
-- asian_pop NUMERIC,
-- hispanic_pop NUMERIC,
-- amerindian_pop NUMERIC,
-- other_race_pop NUMERIC,
-- two_or_more_races_pop NUMERIC,
-- not_hispanic_pop NUMERIC,
-- --not_us_citizen_pop NUMERIC,
-- --workers_16_and_over NUMERIC,
-- --commuters_by_car_truck_van NUMERIC,
-- --commuters_drove_alone NUMERIC,
-- --commuters_by_carpool NUMERIC,
-- --commuters_by_public_transportation NUMERIC,
-- --commuters_by_bus NUMERIC,
-- --commuters_by_subway_or_elevated NUMERIC,
-- --walked_to_work NUMERIC,
-- --worked_at_home NUMERIC,
-- --children NUMERIC, -- TODO we should be able to get this at BG
-- households NUMERIC,
-- --population_3_years_over NUMERIC,
-- --in_school NUMERIC,
-- --in_grades_1_to_4 NUMERIC,
-- --in_grades_5_to_8 NUMERIC,
-- --in_grades_9_to_12 NUMERIC,
-- --in_undergrad_college NUMERIC,
-- pop_25_years_over NUMERIC,
-- high_school_diploma NUMERIC,
-- less_one_year_college NUMERIC,
-- one_year_more_college NUMERIC,
-- associates_degree NUMERIC,
-- bachelors_degree NUMERIC,
-- masters_degree NUMERIC,
-- --pop_5_years_over NUMERIC,
-- --speak_only_english_at_home NUMERIC,
-- --speak_spanish_at_home NUMERIC,
-- --pop_determined_poverty_status NUMERIC,
-- --poverty NUMERIC,
-- median_income NUMERIC,
-- gini_index NUMERIC,
-- income_per_capita NUMERIC,
-- housing_units NUMERIC,
-- vacant_housing_units NUMERIC,
-- vacant_housing_units_for_rent NUMERIC,
-- vacant_housing_units_for_sale NUMERIC,
-- median_rent NUMERIC,
-- percent_income_spent_on_rent NUMERIC,
-- owner_occupied_housing_units NUMERIC,
-- million_dollar_housing_units NUMERIC,
-- mortgaged_housing_units NUMERIC,
-- --pop_15_and_over NUMERIC,
-- --pop_never_married NUMERIC,
-- --pop_now_married NUMERIC,
-- --pop_separated NUMERIC,
-- --pop_widowed NUMERIC,
-- --pop_divorced NUMERIC,
-- commuters_16_over NUMERIC,
-- commute_less_10_mins NUMERIC,
-- commute_10_14_mins NUMERIC,
-- commute_15_19_mins NUMERIC,
-- commute_20_24_mins NUMERIC,
-- commute_25_29_mins NUMERIC,
-- commute_30_34_mins NUMERIC,
-- commute_35_44_mins NUMERIC,
-- commute_45_59_mins NUMERIC,
-- commute_60_more_mins NUMERIC,
-- aggregate_travel_time_to_work NUMERIC,
-- income_less_10000 NUMERIC,
-- income_10000_14999 NUMERIC,
-- income_15000_19999 NUMERIC,
-- income_20000_24999 NUMERIC,
-- income_25000_29999 NUMERIC,
-- income_30000_34999 NUMERIC,
-- income_35000_39999 NUMERIC,
-- income_40000_44999 NUMERIC,
-- income_45000_49999 NUMERIC,
-- income_50000_59999 NUMERIC,
-- income_60000_74999 NUMERIC,
-- income_75000_99999 NUMERIC,
-- income_100000_124999 NUMERIC,
-- income_125000_149999 NUMERIC,
-- income_150000_199999 NUMERIC,
-- income_200000_or_more NUMERIC,
-- land_area NUMERIC)
-- AS $$
-- DECLARE
-- target_cols text[];
-- names text[];
-- vals NUMERIC[];
-- q text;
-- BEGIN
-- target_cols := Array['total_pop',
-- 'male_pop',
-- 'female_pop',
-- 'median_age',
-- 'white_pop',
-- 'black_pop',
-- 'asian_pop',
-- 'hispanic_pop',
-- 'amerindian_pop',
-- 'other_race_pop',
-- 'two_or_more_races_pop',
-- 'not_hispanic_pop',
-- --'not_us_citizen_pop',
-- --'workers_16_and_over',
-- --'commuters_by_car_truck_van',
-- --'commuters_drove_alone',
-- --'commuters_by_carpool',
-- --'commuters_by_public_transportation',
-- --'commuters_by_bus',
-- --'commuters_by_subway_or_elevated',
-- --'walked_to_work',
-- --'worked_at_home',
-- --'children',
-- 'households',
-- --'population_3_years_over',
-- --'in_school',
-- --'in_grades_1_to_4',
-- --'in_grades_5_to_8',
-- --'in_grades_9_to_12',
-- --'in_undergrad_college',
-- 'pop_25_years_over',
-- 'high_school_diploma',
-- 'less_one_year_college',
-- 'one_year_more_college',
-- 'associates_degree',
-- 'bachelors_degree',
-- 'masters_degree',
-- --'pop_5_years_over',
-- --'speak_only_english_at_home',
-- --'speak_spanish_at_home',
-- --'pop_determined_poverty_status',
-- --'poverty',
-- 'median_income',
-- 'gini_index',
-- 'income_per_capita',
-- 'housing_units',
-- 'vacant_housing_units',
-- 'vacant_housing_units_for_rent',
-- 'vacant_housing_units_for_sale',
-- 'median_rent',
-- 'percent_income_spent_on_rent',
-- 'owner_occupied_housing_units',
-- 'million_dollar_housing_units',
-- 'mortgaged_housing_units',
-- --'pop_15_and_over',
-- --'pop_never_married',
-- --'pop_now_married',
-- --'pop_separated',
-- --'pop_widowed',
-- --'pop_divorced',
-- 'commuters_16_over',
-- 'commute_less_10_mins',
-- 'commute_10_14_mins',
-- 'commute_15_19_mins',
-- 'commute_20_24_mins',
-- 'commute_25_29_mins',
-- 'commute_30_34_mins',
-- 'commute_35_44_mins',
-- 'commute_45_59_mins',
-- 'commute_60_more_mins',
-- 'aggregate_travel_time_to_work',
-- 'income_less_10000',
-- 'income_10000_14999',
-- 'income_15000_19999',
-- 'income_20000_24999',
-- 'income_25000_29999',
-- 'income_30000_34999',
-- 'income_35000_39999',
-- 'income_40000_44999',
-- 'income_45000_49999',
-- 'income_50000_59999',
-- 'income_60000_74999',
-- 'income_75000_99999',
-- 'income_100000_124999',
-- 'income_125000_149999',
-- 'income_150000_199999',
-- 'income_200000_or_more',
-- 'land_area'];
--
-- q :=
-- $query$
-- WITH a As (
-- SELECT
-- array_agg(_OBS_GetCensusJ->>'name') As names,
-- array_agg(_OBS_GetCensusJ->>'value') As vals
-- FROM cdb_observatory._OBS_GetCensusJ($1,$2,$3,$4)
-- )$query$ ||
-- cdb_observatory._OBS_BuildSnapshotQuery(target_cols) ||
-- ' FROM a'
-- ;
--
-- RETURN QUERY
-- EXECUTE
-- q
-- USING geom, target_cols, time_span, geometry_level;
--
-- RETURN;
-- END;
-- $$ LANGUAGE plpgsql;
--Base functions for performing augmentation
----------------------------------------------------------------------------------------
--Returns arrays of values for the given census dimension names for a given
--point or polygon
CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetCensus(
geom geometry,
dimension_names text[],
time_span text DEFAULT '2009 - 2013',
geometry_level text DEFAULT 'us.census.tiger.block_group'
)
RETURNS SETOF JSON
AS $$
DECLARE
ids text[];
BEGIN
ids := cdb_observatory._OBS_LookupCensusHuman(dimension_names);
RETURN QUERY
SELECT * FROM cdb_observatory._OBS_Get(geom, ids, time_span, geometry_level);
END;
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION cdb_observatory._OBS_GetCensus(
geom geometry,
dimension_name text,
time_span text DEFAULT '2009 - 2013',
geometry_level text DEFAULT 'us.census.tiger.block_group'
)
RETURNS NUMERIC
AS $$
DECLARE
ids Text[];
result_json json;
result Numeric;
BEGIN
ids := cdb_observatory._OBS_LookupCensusHuman(Array[dimension_name]);
result_json := (SELECT a FROM cdb_observatory._OBS_Get(geom, ids, time_span, geometry_level) as a limit 1);
EXECUTE
format('select $1::numeric as "%s"', result_json->>'name')
INTO result
USING
result_json->>'value';
return result;
END;
$$ LANGUAGE plpgsql;
-- Base augmentation fucntion.
CREATE OR REPLACE FUNCTION cdb_observatory._OBS_Get(
geom geometry,

21
src/pg/test/expected/40_observatory_utility_test.out
View File

@@ -94,27 +94,6 @@ from result;
t
(1 row)
-- OBS_LookupCensusHuman
-- should give back: {"us.census.acs.B19083001"}
SELECT
cdb_observatory._OBS_LookupCensusHuman(
Array['gini_index']
);
_obs_lookupcensushuman
---------------------------
{us.census.acs.B19083001}
(1 row)
-- should be empty array
SELECT
cdb_observatory._OBS_LookupCensusHuman(
Array['cookies']
);
_obs_lookupcensushuman
------------------------
(1 row)
-- OBS_BuildSnapshotQuery
-- Should give back: SELECT vals[1] As total_pop, vals[2] As male_pop, vals[3] As female_pop, vals[4] As median_age
SELECT

15
src/pg/test/expected/41_observatory_augmentation_test.out
View File

@@ -32,21 +32,6 @@ Done.
t
(1 row)
test_obsgetcensuswithtestpointand2variables
---------------------------------------------
t
(1 row)
test_obsgetcensuswithnullislandarea
-------------------------------------
t
(1 row)
test_obsgetcensuswithnullisland
---------------------------------
t
(1 row)
obs_get_gini_index_at_test_point
----------------------------------
t

13
src/pg/test/sql/40_observatory_utility_test.sql
View File

@@ -55,19 +55,6 @@ SELECT
select expected is null as OBS_GetColumnData_missing_measure
from result;
-- OBS_LookupCensusHuman
-- should give back: {"us.census.acs.B19083001"}
SELECT
cdb_observatory._OBS_LookupCensusHuman(
Array['gini_index']
);
-- should be empty array
SELECT
cdb_observatory._OBS_LookupCensusHuman(
Array['cookies']
);
-- OBS_BuildSnapshotQuery
-- Should give back: SELECT vals[1] As total_pop, vals[2] As male_pop, vals[3] As female_pop, vals[4] As median_age
SELECT

42
src/pg/test/sql/41_observatory_augmentation_test.sql
View File

@@ -5,48 +5,6 @@ WITH result as(
FROM cdb_observatory.OBS_GetDemographicSnapshot(cdb_observatory._TestPoint())
) select expected_columns ='58' as OBS_GetDemographicSnapshot_test_no_returns
FROM result;
--
-- dimension | dimension_value
-- ----------|----------------
-- total_pop | 9516.27915900609
-- male_pop | 6152.51885204623
WITH result as (
SELECT array_agg(_obs_getcensus->>'value') as b
FROM( select * from
cdb_observatory._OBS_GetCensus(
cdb_observatory._TestPoint(),
Array['total_pop','male_pop']::text[]
)) a
)
select b='{9516.27915900609,6152.51885204623}'
as test_obsGetCensusWithTestPointAnd2Variables
from result;
-- what happens on null island?
-- expect nulls back: {female_pop, male_pop} | {NULL, NULL}
WITH result as (
SELECT count(vals) non_null
FROM( select _OBS_GetCensus->>'value' vals from
cdb_observatory._OBS_GetCensus(
ST_Buffer(ST_SetSRID(ST_Point(0, 0), 4326)::geography, 5000)::geometry,
Array['total_pop','male_pop']::text[]
)) a
)
SELECT non_null = 0 as test_obsGetCensusWithNullIslandArea
FROM result;
-- expect nulls back {female_pop, male_pop} | {NULL, NULL}
WITH result as (
SELECT count(vals) non_null
FROM( select _OBS_GetCensus->>'value' vals from
cdb_observatory._OBS_GetCensus(
ST_SetSRID(ST_Point(0, 0), 4326),
Array['total_pop','male_pop']::text[]
)) a
)
SELECT non_null = 0 as test_obsGetCensusWithNullIsland
FROM result;
--
-- names | vals